X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Fcanvas.py;h=8667a0d0457cccfc145cc52bc1eb1c7816aa04b8;hb=HEAD;hp=3faa76076318813d8b16f005f9a87ce5df986034;hpb=b8c6ffc51823a9ed6af69da0f9777c6504c1d027;p=youtube-dl diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py index 3faa76076..8667a0d04 100644 --- a/youtube_dl/extractor/canvas.py +++ b/youtube_dl/extractor/canvas.py @@ -11,49 +11,78 @@ from ..utils import ( strip_or_none, float_or_none, int_or_none, + merge_dicts, parse_iso8601, + str_or_none, + url_or_none, ) class CanvasIE(InfoExtractor): - _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?Pcanvas|een|ketnet|vrtvideo)/assets/(?P[^/?#&]+)' + _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?Pcanvas|een|ketnet|vrt(?:video|nieuws)|sporza)/assets/(?P[^/?#&]+)' _TESTS = [{ 'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', - 'md5': '90139b746a0a9bd7bb631283f6e2a64e', + 'md5': '68993eda72ef62386a15ea2cf3c93107', 'info_dict': { 'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', 'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'Nachtwacht: De Greystook', - 'description': 'md5:1db3f5dc4c7109c821261e7512975be7', + 'description': 'Nachtwacht: De Greystook', 'thumbnail': r're:^https?://.*\.jpg$', - 'duration': 1468.03, + 'duration': 1468.04, }, 'expected_warnings': ['is not a supported codec', 'Unknown MIME type'], }, { 'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e', 'only_matching': True, }] + _HLS_ENTRY_PROTOCOLS_MAP = { + 'HLS': 'm3u8_native', + 'HLS_AES': 'm3u8', + } + _REST_API_BASE = 'https://media-services-public.vrt.be/vualto-video-aggregator-web/rest/external/v1' def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) site_id, video_id = mobj.group('site_id'), mobj.group('id') + # Old API endpoint, serves more formats but may fail for some videos data = self._download_json( 'https://mediazone.vrt.be/api/v1/%s/assets/%s' - % (site_id, video_id), video_id) + % (site_id, video_id), video_id, 'Downloading asset JSON', + 'Unable to download asset JSON', fatal=False) + + # New API endpoint + if not data: + token = self._download_json( + '%s/tokens' % self._REST_API_BASE, video_id, + 'Downloading token', data=b'', + headers={'Content-Type': 'application/json'})['vrtPlayerToken'] + data = self._download_json( + '%s/videos/%s' % (self._REST_API_BASE, video_id), + video_id, 'Downloading video JSON', fatal=False, query={ + 'vrtPlayerToken': token, + 'client': '%s@PROD' % site_id, + }, expected_status=400) + message = data.get('message') + if message and not data.get('title'): + if data.get('code') == 'AUTHENTICATION_REQUIRED': + self.raise_login_required(message) + raise ExtractorError(message, expected=True) title = data['title'] description = data.get('description') formats = [] for target in data['targetUrls']: - format_url, format_type = target.get('url'), target.get('type') + format_url, format_type = url_or_none(target.get('url')), str_or_none(target.get('type')) if not format_url or not format_type: continue - if format_type == 'HLS': + format_type = format_type.upper() + if format_type in self._HLS_ENTRY_PROTOCOLS_MAP: formats.extend(self._extract_m3u8_formats( - format_url, video_id, 'mp4', entry_protocol='m3u8_native', + format_url, video_id, 'mp4', self._HLS_ENTRY_PROTOCOLS_MAP[format_type], m3u8_id=format_type, fatal=False)) elif format_type == 'HDS': formats.extend(self._extract_f4m_formats( @@ -129,20 +158,20 @@ class CanvasEenIE(InfoExtractor): }, 'skip': 'Pagina niet gevonden', }, { - 'url': 'https://www.een.be/sorry-voor-alles/herbekijk-sorry-voor-alles', + 'url': 'https://www.een.be/thuis/emma-pakt-thilly-aan', 'info_dict': { - 'id': 'mz-ast-11a587f8-b921-4266-82e2-0bce3e80d07f', - 'display_id': 'herbekijk-sorry-voor-alles', + 'id': 'md-ast-3a24ced2-64d7-44fb-b4ed-ed1aafbf90b8', + 'display_id': 'emma-pakt-thilly-aan', 'ext': 'mp4', - 'title': 'Herbekijk Sorry voor alles', - 'description': 'md5:8bb2805df8164e5eb95d6a7a29dc0dd3', + 'title': 'Emma pakt Thilly aan', + 'description': 'md5:c5c9b572388a99b2690030afa3f3bad7', 'thumbnail': r're:^https?://.*\.jpg$', - 'duration': 3788.06, + 'duration': 118.24, }, 'params': { 'skip_download': True, }, - 'skip': 'Episode no longer available', + 'expected_warnings': ['is not a supported codec'], }, { 'url': 'https://www.canvas.be/check-point/najaar-2016/de-politie-uw-vriend', 'only_matching': True, @@ -178,19 +207,44 @@ class VrtNUIE(GigyaBaseIE): IE_DESC = 'VrtNU.be' _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?Pvrtnu)/(?:[^/]+/)*(?P[^/?#&]+)' _TESTS = [{ + # Available via old API endpoint 'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1/postbus-x-s1a1/', 'info_dict': { 'id': 'pbs-pub-2e2d8c27-df26-45c9-9dc6-90c78153044d$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de', - 'ext': 'flv', + 'ext': 'mp4', 'title': 'De zwarte weduwe', - 'description': 'md5:d90c21dced7db869a85db89a623998d4', + 'description': 'md5:db1227b0f318c849ba5eab1fef895ee4', 'duration': 1457.04, 'thumbnail': r're:^https?://.*\.jpg$', - 'season': '1', + 'season': 'Season 1', 'season_number': 1, 'episode_number': 1, }, - 'skip': 'This video is only available for registered users' + 'skip': 'This video is only available for registered users', + 'params': { + 'username': '', + 'password': '', + }, + 'expected_warnings': ['is not a supported codec'], + }, { + # Only available via new API endpoint + 'url': 'https://www.vrt.be/vrtnu/a-z/kamp-waes/1/kamp-waes-s1a5/', + 'info_dict': { + 'id': 'pbs-pub-0763b56c-64fb-4d38-b95b-af60bf433c71$vid-ad36a73c-4735-4f1f-b2c0-a38e6e6aa7e1', + 'ext': 'mp4', + 'title': 'Aflevering 5', + 'description': 'Wie valt door de mand tijdens een missie?', + 'duration': 2967.06, + 'season': 'Season 1', + 'season_number': 1, + 'episode_number': 5, + }, + 'skip': 'This video is only available for registered users', + 'params': { + 'username': '', + 'password': '', + }, + 'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'], }] _NETRC_MACHINE = 'vrtnu' _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy' @@ -246,11 +300,15 @@ class VrtNUIE(GigyaBaseIE): def _real_extract(self, url): display_id = self._match_id(url) - webpage = self._download_webpage(url, display_id) + webpage, urlh = self._download_webpage_handle(url, display_id) + + info = self._search_json_ld(webpage, display_id, default={}) - title = self._html_search_regex( + # title is optional here since it may be extracted by extractor + # that is delegated from here + title = strip_or_none(self._html_search_regex( r'(?ms)

(.+?)

', - webpage, 'title').strip() + webpage, 'title', default=None)) description = self._html_search_regex( r'(?ms)
(.+?)
', @@ -276,7 +334,7 @@ class VrtNUIE(GigyaBaseIE): webpage, 'release_date', default=None)) # If there's a ? or a # in the URL, remove them and everything after - clean_url = url.split('?')[0].split('#')[0].strip('/') + clean_url = urlh.geturl().split('?')[0].split('#')[0].strip('/') securevideo_url = clean_url + '.mssecurevideo.json' try: @@ -295,7 +353,7 @@ class VrtNUIE(GigyaBaseIE): # the first one video_id = list(video.values())[0].get('videoid') - return { + return merge_dicts(info, { '_type': 'url_transparent', 'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id, 'ie_key': CanvasIE.ie_key(), @@ -307,4 +365,4 @@ class VrtNUIE(GigyaBaseIE): 'season_number': season_number, 'episode_number': episode_number, 'release_date': release_date, - } + })