X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Ftheplatform.py;h=9a57b49df66422a7a7828f2a12ff1385016951ef;hb=8477a6928314cff7957fde55e56a05b2acf76871;hp=adaec337579e0bdca194b0b6cf44cefd918edd31;hpb=aa6cd05ed82b14af0e3827b2ff43eed02087b574;p=youtube-dl diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py index adaec3375..9a57b49df 100644 --- a/youtube_dl/extractor/theplatform.py +++ b/youtube_dl/extractor/theplatform.py @@ -2,7 +2,6 @@ from __future__ import unicode_literals import re -import json import time import hmac import binascii @@ -17,11 +16,13 @@ from ..compat import ( from ..utils import ( determine_ext, ExtractorError, - xpath_with_ns, - unsmuggle_url, - int_or_none, - url_basename, float_or_none, + int_or_none, + sanitized_Request, + unsmuggle_url, + xpath_with_ns, + mimetype2ext, + find_xpath_attr, ) default_ns = 'http://www.w3.org/2005/SMIL21/Language' @@ -29,17 +30,13 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns}) class ThePlatformBaseIE(InfoExtractor): - def _extract_theplatform_smil_formats(self, smil_url, video_id, note='Downloading SMIL data'): + def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'): meta = self._download_xml(smil_url, video_id, note=note) - try: - error_msg = next( - n.attrib['abstract'] - for n in meta.findall(_x('.//smil:ref')) - if n.attrib.get('title') == 'Geographic Restriction' or n.attrib.get('title') == 'Expired') - except StopIteration: - pass - else: - raise ExtractorError(error_msg, expected=True) + error_element = find_xpath_attr( + meta, _x('.//smil:ref'), 'src', + 'http://link.theplatform.com/s/errorFiles/Unavailable.mp4') + if error_element is not None: + raise ExtractorError(error_element.attrib['abstract'], expected=True) formats = self._parse_smil_formats( meta, smil_url, video_id, namespace=default_ns, @@ -55,12 +52,13 @@ class ThePlatformBaseIE(InfoExtractor): self._sort_formats(formats) - return formats + subtitles = self._parse_smil_subtitles(meta, default_ns) + + return formats, subtitles def get_metadata(self, path, video_id): info_url = 'http://link.theplatform.com/s/%s?format=preview' % path - info_json = self._download_webpage(info_url, video_id) - info = json.loads(info_json) + info = self._download_json(info_url, video_id) subtitles = {} captions = info.get('captions') @@ -68,7 +66,7 @@ class ThePlatformBaseIE(InfoExtractor): for caption in captions: lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type') subtitles[lang] = [{ - 'ext': 'srt' if mime == 'text/srt' else 'ttml', + 'ext': mimetype2ext(mime), 'url': src, }] @@ -84,7 +82,7 @@ class ThePlatformBaseIE(InfoExtractor): class ThePlatformIE(ThePlatformBaseIE): _VALID_URL = r'''(?x) (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P[^/]+)/ - (?:(?P(?:[^/]+/)+select/media/)|(?P(?:[^/\?]+/(?:swf|config)|onsite)/select/))? + (?:(?P(?:(?:[^/]+/)+select/)?media/)|(?P(?:[^/\?]+/(?:swf|config)|onsite)/select/))? |theplatform:)(?P[^/\?&]+)''' _TESTS = [{ @@ -139,6 +137,11 @@ class ThePlatformIE(ThePlatformBaseIE): 'upload_date': '20150701', 'categories': ['Today/Shows/Orange Room', 'Today/Sections/Money', 'Today/Topics/Tech', "Today/Topics/Editor's picks"], }, + }, { + # From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1 + # geo-restricted (US), HLS encrypted with AES-128 + 'url': 'http://player.theplatform.com/p/NnzsPC/onsite_universal/select/media/guid/2410887629/2928790?fwsitesection=nbc_the_blacklist_video_library&autoPlay=true&carouselID=137781', + 'only_matching': True, }] @staticmethod @@ -182,8 +185,12 @@ class ThePlatformIE(ThePlatformBaseIE): # Seems there's no pattern for the interested script filename, so # I try one by one for script in reversed(scripts): - feed_script = self._download_webpage(script, video_id, 'Downloading feed script') - feed_id = self._search_regex(r'defaultFeedId\s*:\s*"([^"]+)"', feed_script, 'default feed id', default=None) + feed_script = self._download_webpage( + self._proto_relative_url(script, 'http:'), + video_id, 'Downloading feed script') + feed_id = self._search_regex( + r'defaultFeedId\s*:\s*"([^"]+)"', feed_script, + 'default feed id', default=None) if feed_id is not None: break if feed_id is None: @@ -193,6 +200,20 @@ class ThePlatformIE(ThePlatformBaseIE): if smuggled_data.get('force_smil_url', False): smil_url = url + # Explicitly specified SMIL (see https://github.com/rg3/youtube-dl/issues/7385) + elif '/guid/' in url: + headers = {} + source_url = smuggled_data.get('source_url') + if source_url: + headers['Referer'] = source_url + request = sanitized_Request(url, headers=headers) + webpage = self._download_webpage(request, video_id) + smil_url = self._search_regex( + r']+href=(["\'])(?P.+?)\1[^>]+type=["\']application/smil\+xml', + webpage, 'smil url', group='url') + path = self._search_regex( + r'link\.theplatform\.com/s/((?:[^/?#&]+/)+[^/?#&]+)', smil_url, 'path') + smil_url += '?' if '?' not in smil_url else '&' + 'formats=m3u,mpeg4&format=SMIL' elif mobj.group('config'): config_url = url + '&form=json' config_url = config_url.replace('swf/', 'config/') @@ -210,12 +231,14 @@ class ThePlatformIE(ThePlatformBaseIE): if sig: smil_url = self._sign_url(smil_url, sig['key'], sig['secret']) - formats = self._extract_theplatform_smil_formats(smil_url, video_id) + formats, subtitles = self._extract_theplatform_smil(smil_url, video_id) ret = self.get_metadata(path, video_id) + combined_subtitles = self._merge_subtitles(ret.get('subtitles', {}), subtitles) ret.update({ 'id': video_id, 'formats': formats, + 'subtitles': combined_subtitles, }) return ret @@ -253,15 +276,18 @@ class ThePlatformFeedIE(ThePlatformBaseIE): entry = feed['entries'][0] formats = [] + subtitles = {} first_video_id = None duration = None for item in entry['media$content']: - smil_url = item['plfile$url'] + '&format=SMIL&Tracking=true&Embedded=true&formats=MPEG4,F4M' - cur_video_id = url_basename(smil_url) + smil_url = item['plfile$url'] + '&format=SMIL&mbr=true' + cur_video_id = ThePlatformIE._match_id(smil_url) if first_video_id is None: first_video_id = cur_video_id duration = float_or_none(item.get('plfile$duration')) - formats.extend(self._extract_theplatform_smil_formats(smil_url, video_id, 'Downloading SMIL data for %s' % cur_video_id)) + cur_formats, cur_subtitles = self._extract_theplatform_smil(smil_url, video_id, 'Downloading SMIL data for %s' % cur_video_id) + formats.extend(cur_formats) + subtitles = self._merge_subtitles(subtitles, cur_subtitles) self._sort_formats(formats) @@ -275,9 +301,11 @@ class ThePlatformFeedIE(ThePlatformBaseIE): categories = [item['media$name'] for item in entry.get('media$categories', [])] ret = self.get_metadata('%s/%s' % (provider_id, first_video_id), video_id) + subtitles = self._merge_subtitles(subtitles, ret['subtitles']) ret.update({ 'id': video_id, 'formats': formats, + 'subtitles': subtitles, 'thumbnails': thumbnails, 'duration': duration, 'timestamp': timestamp,