Merge pull request #7833 from remitamine/ooyala

[ooyala] improve extraction

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 7819a91dcb2f5108b0c6690f05cc6579d2d4362c..aaf0c49c8cb474e397a71988502848ed8351336b 100644 (file)
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -292,7 +292,7 @@ class F4mFD(FragmentFD):
         # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
         # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244
         # and https://github.com/rg3/youtube-dl/issues/7823)
-        manifest = fix_xml_ampersands(urlh.read()).strip()
+        manifest = fix_xml_ampersands(urlh.read().decode('utf-8', 'ignore')).strip()
 
         doc = compat_etree_fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 571289421e2d63e9e55b036a54ee952e98ccfead..02dfd36f4ace366ccffe18e3cf211eb9fe050c95 100644 (file)
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -3,7 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_etree_fromstring
+from ..compat import (
+    compat_etree_fromstring,
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -67,6 +70,17 @@ class VevoIE(InfoExtractor):
         'params': {
             'skip_download': 'true',
         }
+    }, {
+        'note': 'No video_info',
+        'url': 'http://www.vevo.com/watch/k-camp-1/Till-I-Die/USUV71503000',
+        'md5': '8b83cc492d72fc9cf74a02acee7dc1b0',
+        'info_dict': {
+            'id': 'USUV71503000',
+            'ext': 'mp4',
+            'title': 'Till I Die - K Camp ft. T.I.',
+            'duration': 193,
+        },
+        'expected_warnings': ['Unable to download SMIL file'],
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
 
@@ -81,11 +95,17 @@ class VevoIE(InfoExtractor):
         if webpage is False:
             self._oauth_token = None
         else:
+            if 'THIS PAGE IS CURRENTLY UNAVAILABLE IN YOUR REGION' in webpage:
+                raise ExtractorError('%s said: This page is currently unavailable in your region.' % self.IE_NAME, expected=True)
+
             self._oauth_token = self._search_regex(
                 r'access_token":\s*"([^"]+)"',
                 webpage, 'access token', fatal=False)
 
     def _formats_from_json(self, video_info):
+        if not video_info:
+            return []
+
         last_version = {'version': -1}
         for version in video_info['videoVersions']:
             # These are the HTTP downloads, other types are for different manifests
@@ -110,9 +130,8 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
-    def _formats_from_smil(self, smil_xml):
+    def _formats_from_smil(self, smil_doc):
         formats = []
-        smil_doc = compat_etree_fromstring(smil_xml.encode('utf-8'))
         els = smil_doc.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
         for el in els:
             src = el.attrib['src']
@@ -145,14 +164,14 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
-    def _download_api_formats(self, video_id):
+    def _download_api_formats(self, video_id, video_url):
         if not self._oauth_token:
             self._downloader.report_warning(
                 'No oauth token available, skipping API HLS download')
             return []
 
-        api_url = 'https://apiv2.vevo.com/video/%s/streams/hls?token=%s' % (
-            video_id, self._oauth_token)
+        api_url = compat_urlparse.urljoin(video_url, '//apiv2.vevo.com/video/%s/streams/hls?token=%s' % (
+            video_id, self._oauth_token))
         api_data = self._download_json(
             api_url, video_id,
             note='Downloading HLS formats',
@@ -166,18 +185,26 @@ class VevoIE(InfoExtractor):
             preference=0)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+
+        webpage = None
 
         json_url = 'http://videoplayer.vevo.com/VideoService/AuthenticateVideo?isrc=%s' % video_id
         response = self._download_json(json_url, video_id)
-        video_info = response['video']
+        video_info = response['video'] or {}
 
-        if not video_info:
+        if not video_info and response.get('statusCode') != 909:
             if 'statusMessage' in response:
                 raise ExtractorError('%s said: %s' % (self.IE_NAME, response['statusMessage']), expected=True)
             raise ExtractorError('Unable to extract videos')
 
+        if not video_info:
+            if url.startswith('vevo:'):
+                raise ExtractorError('Please specify full Vevo URL for downloading', expected=True)
+            webpage = self._download_webpage(url, video_id)
+
+        title = video_info.get('title') or self._og_search_title(webpage)
+
         formats = self._formats_from_json(video_info)
 
         is_explicit = video_info.get('isExplicit')
@@ -189,11 +216,11 @@ class VevoIE(InfoExtractor):
             age_limit = None
 
         # Download via HLS API
-        formats.extend(self._download_api_formats(video_id))
+        formats.extend(self._download_api_formats(video_id, url))
 
         # Download SMIL
         smil_blocks = sorted((
-            f for f in video_info['videoVersions']
+            f for f in video_info.get('videoVersions', [])
             if f['sourceType'] == 13),
             key=lambda f: f['version'])
         smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
@@ -205,23 +232,26 @@ class VevoIE(InfoExtractor):
             if smil_url_m is not None:
                 smil_url = smil_url_m
         if smil_url:
-            smil_xml = self._download_webpage(
-                smil_url, video_id, 'Downloading SMIL info', fatal=False)
-            if smil_xml:
-                formats.extend(self._formats_from_smil(smil_xml))
+            smil_doc = self._download_smil(smil_url, video_id, fatal=False)
+            if smil_doc:
+                formats.extend(self._formats_from_smil(smil_doc))
 
         self._sort_formats(formats)
-        timestamp_ms = int_or_none(self._search_regex(
+        timestamp = int_or_none(self._search_regex(
             r'/Date\((\d+)\)/',
-            video_info['launchDate'], 'launch date', fatal=False))
+            video_info['launchDate'], 'launch date', fatal=False),
+            scale=1000) if video_info else None
+
+        duration = video_info.get('duration') or int_or_none(
+            self._html_search_meta('video:duration', webpage))
 
         return {
             'id': video_id,
-            'title': video_info['title'],
+            'title': title,
             'formats': formats,
-            'thumbnail': video_info['imageUrl'],
-            'timestamp': timestamp_ms // 1000,
-            'uploader': video_info['mainArtists'][0]['artistName'],
-            'duration': video_info['duration'],
+            'thumbnail': video_info.get('imageUrl'),
+            'timestamp': timestamp,
+            'uploader': video_info['mainArtists'][0]['artistName'] if video_info else None,
+            'duration': duration,
             'age_limit': age_limit,
         }

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index b468023060d3476e0a0ede8e36ea23b25ffeb298..ef096cbd248def4c67a1046c326ce7d1fd6a8796 100644 (file)
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -10,8 +10,8 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
-    determine_ext,
     unified_strdate,
+    qualities,
 )
 
 
@@ -33,6 +33,7 @@ class WDRIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'Page Not Found',
         },
         {
             'url': 'http://www1.wdr.de/themen/av/videomargaspiegelisttot101-videoplayer.html',
@@ -47,6 +48,7 @@ class WDRIE(InfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'Page Not Found',
         },
         {
             'url': 'http://www1.wdr.de/themen/kultur/audioerlebtegeschichtenmargaspiegel100-audioplayer.html',
@@ -71,6 +73,7 @@ class WDRIE(InfoExtractor):
                 'upload_date': '20140717',
                 'is_live': False
             },
+            'skip': 'Page Not Found',
         },
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100.html',
@@ -83,10 +86,10 @@ class WDRIE(InfoExtractor):
             'url': 'http://www1.wdr.de/mediathek/video/livestream/index.html',
             'info_dict': {
                 'id': 'mdb-103364',
-                'title': 're:^WDR Fernsehen [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'title': 're:^WDR Fernsehen Live [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
                 'description': 'md5:ae2ff888510623bf8d4b115f95a9b7c9',
                 'ext': 'flv',
-                'upload_date': '20150212',
+                'upload_date': '20150101',
                 'is_live': True
             },
             'params': {
@@ -150,25 +153,52 @@ class WDRIE(InfoExtractor):
         if upload_date:
             upload_date = unified_strdate(upload_date)
 
+        formats = []
+        preference = qualities(['S', 'M', 'L', 'XL'])
+
         if video_url.endswith('.f4m'):
-            video_url += '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18'
-            ext = 'flv'
+            f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18', page_id, f4m_id='hds', fatal=False)
+            if f4m_formats:
+                formats.extend(f4m_formats)
         elif video_url.endswith('.smil'):
-            fmt = self._extract_smil_formats(video_url, page_id)[0]
-            video_url = fmt['url']
-            sep = '&' if '?' in video_url else '?'
-            video_url += sep
-            video_url += 'hdcore=3.3.0&plugin=aasp-3.3.0.99.43'
-            ext = fmt['ext']
+            smil_formats = self._extract_smil_formats(video_url, page_id, False, {
+                'hdcore': '3.3.0',
+                'plugin': 'aasp-3.3.0.99.43',
+            })
+            if smil_formats:
+                formats.extend(smil_formats)
         else:
-            ext = determine_ext(video_url)
+            formats.append({
+                'url': video_url,
+                'http_headers': {
+                    'User-Agent': 'mobile',
+                },
+            })
+
+        m3u8_url = self._search_regex(r'rel="adaptiv"[^>]+href="([^"]+)"', webpage, 'm3u8 url', default=None)
+        if m3u8_url:
+            m3u8_formats = self._extract_m3u8_formats(m3u8_url, page_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+            if m3u8_formats:
+                formats.extend(m3u8_formats)
+
+        direct_urls = re.findall(r'rel="web(S|M|L|XL)"[^>]+href="([^"]+)"', webpage)
+        if direct_urls:
+            for quality, video_url in direct_urls:
+                formats.append({
+                    'url': video_url,
+                    'preference': preference(quality),
+                    'http_headers': {
+                        'User-Agent': 'mobile',
+                    },
+                })
+
+        self._sort_formats(formats)
 
         description = self._html_search_meta('Description', webpage, 'description')
 
         return {
             'id': page_id,
-            'url': video_url,
-            'ext': ext,
+            'formats': formats,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,