[WDR] fixed parsing of playlists

[youtube-dl] / youtube_dl / extractor / wdr.py

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index fddcbf190767ab97450708467a2c67dd1ac13f92..1e729cb7cb1917d5e4c80a30d98ed9cd167f7d2e 100644 (file)
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -4,12 +4,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_parse_qs,
-    compat_urlparse,
-)
 from ..utils import (
     determine_ext,
+    js_to_json,
     strip_jsonp,
     unified_strdate,
     ExtractorError,
@@ -17,12 +14,10 @@ from ..utils import (
 
 
 class WDRIE(InfoExtractor):
-    _CURRENT_MAUS_URL = r'https?://www.wdrmaus.de/aktuelle-sendung/(wdr|index).php5'
+    _CURRENT_MAUS_URL = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/){1,2}[^/?#]+\.php5'
     _PAGE_REGEX = r'/mediathek/(?P<media_type>[^/]+)/(?P<type>[^/]+)/(?P<display_id>.+)\.html'
     _VALID_URL = r'(?P<page_url>https?://(?:www\d\.)?wdr\d?\.de)' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL
 
-    _JS_URL_REGEX = r'(https?://deviceids-medp.wdr.de/ondemand/\d+/\d+\.js)'
-
     _TESTS = [
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/doku-am-freitag/video-geheimnis-aachener-dom-100.html',
@@ -77,7 +72,7 @@ class WDRIE(InfoExtractor):
         },
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html',
-            'playlist_mincount': 10,
+            'playlist_mincount': 8,
             'info_dict': {
                 'id': 'aktuelle-stunde/aktuelle-stunde-120',
             },
@@ -93,6 +88,20 @@ class WDRIE(InfoExtractor):
             },
             'skip': 'The id changes from week to week because of the new episode'
         },
+        {
+            'url': 'http://www.wdrmaus.de/sachgeschichten/sachgeschichten/achterbahn.php5',
+            'md5': 'ca365705551e4bd5217490f3b0591290',
+            'info_dict': {
+                'id': 'mdb-186083',
+                'ext': 'flv',
+                'upload_date': '20130919',
+                'title': 'Sachgeschichte - Achterbahn ',
+                'description': '- Die Sendung mit der Maus -',
+            },
+            'params': {
+                'skip_download': True,  # the file has different versions :(
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -102,13 +111,17 @@ class WDRIE(InfoExtractor):
         display_id = mobj.group('display_id')
         webpage = self._download_webpage(url, display_id)
 
-        js_url = self._search_regex(self._JS_URL_REGEX, webpage, 'js_url', default=None)
+        # for wdr.de the data-extension is in a tag with the class "mediaLink"
+        # for wdrmaus its in a link to the page in a multiline "videoLink"-tag
+        json_metadata = self._html_search_regex(
+            r'class=(?:"mediaLink\b[^"]*"[^>]+|"videoLink\b[^"]*"[\s]*>\n[^\n]*)data-extension="([^"]+)"',
+            webpage, 'media link', default=None, flags=re.MULTILINE)
 
-        if not js_url:
+        if not json_metadata:
             entries = [
                 self.url_result(page_url + href[0], 'WDR')
                 for href in re.findall(
-                    r'<a href="(%s)"' % self._PAGE_REGEX,
+                    r'<a href="(%s)"[^>]+data-extension=' % self._PAGE_REGEX,
                     webpage)
             ]
 
@@ -117,8 +130,12 @@ class WDRIE(InfoExtractor):
 
             raise ExtractorError('No downloadable streams found', expected=True)
 
+        media_link_obj = self._parse_json(json_metadata, display_id,
+                                          transform_source=js_to_json)
+        jsonp_url = media_link_obj['mediaObj']['url']
+
         metadata = self._download_json(
-            js_url, 'metadata', transform_source=strip_jsonp)
+            jsonp_url, 'metadata', transform_source=strip_jsonp)
 
         metadata_tracker_data = metadata['trackerData']
         metadata_media_resource = metadata['mediaResource']
@@ -215,72 +232,3 @@ class WDRMobileIE(InfoExtractor):
                 'User-Agent': 'mobile',
             },
         }
-
-
-class WDRMausIE(InfoExtractor):
-    _VALID_URL = 'https?://(?:www\.)?wdrmaus\.de/(?:[^/]+/){,2}(?P<id>[^/?#]+)((?<!index)\.php5|/(?:$|[?#]))'
-    IE_DESC = 'Sendung mit der Maus'
-    _TESTS = [{
-        'url': 'http://www.wdrmaus.de/sachgeschichten/sachgeschichten/achterbahn.php5',
-        'md5': '178b432d002162a14ccb3e0876741095',
-        'info_dict': {
-            'id': 'achterbahn',
-            'ext': 'mp4',
-            'thumbnail': 're:^http://.+\.jpg',
-            'upload_date': '20131001',
-            'title': '19.09.2013 - Achterbahn',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-        param_code = self._html_search_regex(
-            r'<a href="\?startVideo=1&amp;([^"]+)"', webpage, 'parameters')
-
-        title_date = self._search_regex(
-            r'<div class="sendedatum"><p>Sendedatum:\s*([0-9\.]+)</p>',
-            webpage, 'air date')
-        title_str = self._html_search_regex(
-            r'<h1>(.*?)</h1>', webpage, 'title')
-        title = '%s - %s' % (title_date, title_str)
-        upload_date = unified_strdate(
-            self._html_search_meta('dc.date', webpage))
-
-        fields = compat_parse_qs(param_code)
-        video_url = fields['firstVideo'][0]
-        thumbnail = compat_urlparse.urljoin(url, fields['startPicture'][0])
-
-        formats = [{
-            'format_id': 'rtmp',
-            'url': video_url,
-        }]
-
-        jscode = self._download_webpage(
-            'http://www.wdrmaus.de/codebase/js/extended-medien.min.js',
-            video_id, fatal=False,
-            note='Downloading URL translation table',
-            errnote='Could not download URL translation table')
-        if jscode:
-            for m in re.finditer(
-                    r"stream:\s*'dslSrc=(?P<stream>[^']+)',\s*download:\s*'(?P<dl>[^']+)'\s*\}",
-                    jscode):
-                if video_url.startswith(m.group('stream')):
-                    http_url = video_url.replace(
-                        m.group('stream'), m.group('dl'))
-                    formats.append({
-                        'format_id': 'http',
-                        'url': http_url,
-                    })
-                    break
-
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
-            'upload_date': upload_date,
-        }