X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Fpbs.py;h=143a7669639770e0cdfddc55e0b9395893301736;hb=05a976cd99ef2a0eb0b301cd4f98e1aec927968c;hp=ec95d070411f97e1dad2fde881c9e5f847caafc8;hpb=8d7d9d34521bcfa1b846964f404e8769d2e85ba0;p=youtube-dl diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py index ec95d0704..143a76696 100644 --- a/youtube_dl/extractor/pbs.py +++ b/youtube_dl/extractor/pbs.py @@ -4,6 +4,10 @@ import re from .common import InfoExtractor from ..utils import ( + ExtractorError, + determine_ext, + int_or_none, + unified_strdate, US_RATINGS, ) @@ -11,28 +15,85 @@ from ..utils import ( class PBSIE(InfoExtractor): _VALID_URL = r'''(?x)https?:// (?: - # Direct video URL - video\.pbs\.org/(?:viralplayer|video)/(?P[0-9]+)/? | - # Article with embedded player - (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P[^/]+)/?(?:$|[?\#]) | + # Direct video URL + video\.pbs\.org/(?:viralplayer|video)/(?P[0-9]+)/? | + # Article with embedded player (or direct video) + (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P[^/]+?)(?:\.html)?/?(?:$|[?\#]) | # Player video\.pbs\.org/(?:widget/)?partnerplayer/(?P[^/]+)/ ) ''' - _TEST = { - 'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/', - 'md5': 'ce1888486f0908d555a8093cac9a7362', - 'info_dict': { - 'id': '2365006249', - 'ext': 'mp4', - 'title': 'A More Perfect Union', - 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a', - 'duration': 3190, + _TESTS = [ + { + 'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/', + 'md5': 'ce1888486f0908d555a8093cac9a7362', + 'info_dict': { + 'id': '2365006249', + 'ext': 'mp4', + 'title': 'A More Perfect Union', + 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a', + 'duration': 3190, + }, }, - } + { + 'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/', + 'md5': '143c98aa54a346738a3d78f54c925321', + 'info_dict': { + 'id': '2365297690', + 'ext': 'mp4', + 'title': 'Losing Iraq', + 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e', + 'duration': 5050, + }, + }, + { + 'url': 'http://www.pbs.org/newshour/bb/education-jan-june12-cyberschools_02-23/', + 'md5': 'b19856d7f5351b17a5ab1dc6a64be633', + 'info_dict': { + 'id': '2201174722', + 'ext': 'mp4', + 'title': 'Cyber Schools Gain Popularity, but Quality Questions Persist', + 'description': 'md5:5871c15cba347c1b3d28ac47a73c7c28', + 'duration': 801, + }, + }, + { + 'url': 'http://www.pbs.org/wnet/gperf/dudamel-conducts-verdi-requiem-hollywood-bowl-full-episode/3374/', + 'md5': 'c62859342be2a0358d6c9eb306595978', + 'info_dict': { + 'id': '2365297708', + 'ext': 'mp4', + 'description': 'md5:68d87ef760660eb564455eb30ca464fe', + 'title': 'Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full', + 'duration': 6559, + 'thumbnail': 're:^https?://.*\.jpg$', + } + }, + { + 'url': 'http://www.pbs.org/wgbh/nova/earth/killer-typhoon.html', + 'md5': '908f3e5473a693b266b84e25e1cf9703', + 'info_dict': { + 'id': '2365160389', + 'display_id': 'killer-typhoon', + 'ext': 'mp4', + 'description': 'md5:c741d14e979fc53228c575894094f157', + 'title': 'Killer Typhoon', + 'duration': 3172, + 'thumbnail': 're:^https?://.*\.jpg$', + 'upload_date': '20140122', + } + }, + { + 'url': 'http://www.pbs.org/wgbh/pages/frontline/united-states-of-secrets/', + 'info_dict': { + 'id': 'united-states-of-secrets', + }, + 'playlist_count': 2, + } + ] - def _extract_ids(self, url): + def _extract_webpage(self, url): mobj = re.match(self._VALID_URL, url) presumptive_id = mobj.group('presumptive_id') @@ -40,15 +101,29 @@ class PBSIE(InfoExtractor): if presumptive_id: webpage = self._download_webpage(url, display_id) - # frontline video embed + upload_date = unified_strdate(self._search_regex( + r']+class="videotab[^"]*"[^>]+vid="(\d+)"', webpage) + if tabbed_videos: + return tabbed_videos, presumptive_id, upload_date + + MEDIA_ID_REGEXES = [ + r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'", # frontline video embed + r'class="coveplayerid">([^<]+)<', # coveplayer + r'', # jwplayer + ] + media_id = self._search_regex( - r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'", - webpage, 'frontline video ID', fatal=False, default=None) + MEDIA_ID_REGEXES, webpage, 'media ID', fatal=False, default=None) if media_id: - return media_id, presumptive_id + return media_id, presumptive_id, upload_date url = self._search_regex( - r'', + r'', webpage, 'player URL') mobj = re.match(self._VALID_URL, url) @@ -65,13 +140,56 @@ class PBSIE(InfoExtractor): video_id = mobj.group('id') display_id = video_id - return video_id, display_id + return video_id, display_id, None def _real_extract(self, url): - video_id, display_id = self._extract_ids(url) + video_id, display_id, upload_date = self._extract_webpage(url) + + if isinstance(video_id, list): + entries = [self.url_result( + 'http://video.pbs.org/video/%s' % vid_id, 'PBS', vid_id) + for vid_id in video_id] + return self.playlist_result(entries, display_id) + + info = self._download_json( + 'http://video.pbs.org/videoInfo/%s?format=json&type=partner' % video_id, + display_id) + + formats = [] + for encoding_name in ('recommended_encoding', 'alternate_encoding'): + redirect = info.get(encoding_name) + if not redirect: + continue + redirect_url = redirect.get('url') + if not redirect_url: + continue + + redirect_info = self._download_json( + redirect_url + '?format=json', display_id, + 'Downloading %s video url info' % encoding_name) + + if redirect_info['status'] == 'error': + if redirect_info['http_code'] == 403: + message = ( + 'The video is not available in your region due to ' + 'right restrictions') + else: + message = redirect_info['message'] + raise ExtractorError(message, expected=True) + + format_url = redirect_info.get('url') + if not format_url: + continue - info_url = 'http://video.pbs.org/videoInfo/%s?format=json' % video_id - info = self._download_json(info_url, display_id) + if determine_ext(format_url) == 'm3u8': + formats.extend(self._extract_m3u8_formats( + format_url, display_id, 'mp4', preference=1, m3u8_id='hls')) + else: + formats.append({ + 'url': format_url, + 'format_id': redirect.get('eeid'), + }) + self._sort_formats(formats) rating_str = info.get('rating') if rating_str is not None: @@ -80,11 +198,12 @@ class PBSIE(InfoExtractor): return { 'id': video_id, + 'display_id': display_id, 'title': info['title'], - 'url': info['alternate_encoding']['url'], - 'ext': 'mp4', 'description': info['program'].get('description'), 'thumbnail': info.get('image_url'), - 'duration': info.get('duration'), + 'duration': int_or_none(info.get('duration')), 'age_limit': age_limit, + 'upload_date': upload_date, + 'formats': formats, }