X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Fyoutube.py;h=f65052a8901e962222de32c96d5796c6cca746fb;hb=a2dafe2887acc745b70eff811a2932098048cd64;hp=57b8fdff7204b1fbb5dc6d88742704638995b0eb;hpb=65697b3bf3bf6eaeb91a34e5308a6d2239118071;p=youtube-dl diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 57b8fdff7..f65052a89 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -29,18 +29,18 @@ from ..utils import ( ExtractorError, int_or_none, PagedList, - RegexNotFoundError, unescapeHTML, unified_strdate, orderedSet, write_json_file, + uppercase_escape, ) class YoutubeBaseInfoExtractor(InfoExtractor): """Provide base functions for Youtube extractors""" _LOGIN_URL = 'https://accounts.google.com/ServiceLogin' _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1' - _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' + _AGE_URL = 'https://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' _NETRC_MACHINE = 'youtube' # If True it will raise an error if no login info is provided _LOGIN_REQUIRED = False @@ -111,7 +111,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'next_url': '/', 'action_confirm': 'Confirm', } - req = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form)) + req = compat_urllib_request.Request(self._AGE_URL, + compat_urllib_parse.urlencode(age_form).encode('ascii')) self._download_webpage( req, None, @@ -135,14 +136,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): (?:https?://|//)? # http(s):// or protocol-independent URL (optional) (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/| (?:www\.)?deturl\.com/www\.youtube\.com/| - (?:www\.)?pwnyoutube\.com| + (?:www\.)?pwnyoutube\.com/| + (?:www\.)?yourepeat\.com/| tube\.majestyc\.net/| youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains (?:.*?\#/)? # handle anchor (#/) redirect urls (?: # the various things that can precede the ID: (?:(?:v|embed|e)/) # v/ or embed/ or e/ |(?: # or the v= param in all its forms - (?:(?:watch|movie)(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx) + (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx) (?:\?|\#!?) # the params delimiter ? or # or #! (?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx) v= @@ -207,6 +209,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 256, 'preference': -50}, # Dash webm + '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'VP8', 'acodec': 'none', 'preference': -40}, + '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'VP8', 'acodec': 'none', 'preference': -40}, + '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'VP8', 'acodec': 'none', 'preference': -40}, + '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'VP8', 'acodec': 'none', 'preference': -40}, + '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'VP8', 'acodec': 'none', 'preference': -40}, + '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'VP8', 'acodec': 'none', 'preference': -40}, '242': {'ext': 'webm', 'height': 240, 'resolution': '240p', 'format_note': 'DASH webm', 'preference': -40}, '243': {'ext': 'webm', 'height': 360, 'resolution': '360p', 'format_note': 'DASH webm', 'preference': -40}, '244': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40}, @@ -276,16 +284,34 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): u"url": u"http://www.youtube.com/watch?v=a9LDPn-MO4I", u"file": u"a9LDPn-MO4I.m4a", u"note": u"256k DASH audio (format 141) via DASH manifest", - u"params": { - u"format": "141" - }, u"info_dict": { u"upload_date": "20121002", u"uploader_id": "8KVIDEO", u"description": "No description available.", u"uploader": "8KVIDEO", u"title": "UHDTV TEST 8K VIDEO.mp4" - } + }, + u"params": { + u"youtube_include_dash_manifest": True, + u"format": "141", + }, + }, + # DASH manifest with encrypted signature + { + u'url': u'https://www.youtube.com/watch?v=IB3lcPjvWLA', + u'info_dict': { + u'id': u'IB3lcPjvWLA', + u'ext': u'm4a', + u'title': u'Afrojack - The Spark ft. Spree Wilson', + u'description': u'md5:3199ed45ee8836572865580804d7ac0f', + u'uploader': u'AfrojackVEVO', + u'uploader_id': u'AfrojackVEVO', + u'upload_date': u'20131011', + }, + u"params": { + u'youtube_include_dash_manifest': True, + u'format': '141', + }, }, ] @@ -494,7 +520,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): return a % b m = re.match( - r'^(?P[a-zA-Z]+)\((?P[a-z0-9,]+)\)$', expr) + r'^(?P[a-zA-Z$]+)\((?P[a-z0-9,]+)\)$', expr) if m: fname = m.group('func') if fname not in functions: @@ -1007,7 +1033,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): def _get_available_subtitles(self, video_id, webpage): try: sub_list = self._download_webpage( - 'http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id, + 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id, video_id, note=False) except ExtractorError as err: self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err)) @@ -1023,7 +1049,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): 'fmt': self._downloader.params.get('subtitlesformat', 'srt'), 'name': unescapeHTML(l[0]).encode('utf-8'), }) - url = u'http://www.youtube.com/api/timedtext?' + params + url = u'https://www.youtube.com/api/timedtext?' + params sub_lang_list[lang] = url if not sub_lang_list: self._downloader.report_warning(u'video doesn\'t have subtitles') @@ -1077,8 +1103,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): self._downloader.report_warning(err_msg) return {} - def _extract_id(self, url): - mobj = re.match(self._VALID_URL, url, re.VERBOSE) + @classmethod + def extract_id(cls, url): + mobj = re.match(cls._VALID_URL, url, re.VERBOSE) if mobj is None: raise ExtractorError(u'Invalid URL: %s' % url) video_id = mobj.group(2) @@ -1107,7 +1134,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): mobj = re.search(self._NEXT_URL_RE, url) if mobj: url = 'https://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/') - video_id = self._extract_id(url) + video_id = self.extract_id(url) # Get video webpage url = 'https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id @@ -1261,8 +1288,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): mobj = re.search(r';ytplayer.config = ({.*?});', video_webpage) if not mobj: raise ValueError('Could not find vevo ID') - info = json.loads(mobj.group(1)) - args = info['args'] + ytplayer_config = json.loads(mobj.group(1)) + args = ytplayer_config['args'] # Easy way to know if the 's' value is in url_encoded_fmt_stream_map # this signatures are encrypted if 'url_encoded_fmt_stream_map' not in args: @@ -1289,7 +1316,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): 'url': video_real_url, 'player_url': player_url, } - dct.update(self._formats[itag]) + if itag in self._formats: + dct.update(self._formats[itag]) formats.append(dct) return formats @@ -1354,11 +1382,24 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): raise ExtractorError(u'no conn, hlsvp or url_encoded_fmt_stream_map information found in video info') # Look for the DASH manifest - dash_manifest_url_lst = video_info.get('dashmpd') - if dash_manifest_url_lst and dash_manifest_url_lst[0]: + if (self._downloader.params.get('youtube_include_dash_manifest', False)): try: + # The DASH manifest used needs to be the one from the original video_webpage. + # The one found in get_video_info seems to be using different signatures. + # However, in the case of an age restriction there won't be any embedded dashmpd in the video_webpage. + # Luckily, it seems, this case uses some kind of default signature (len == 86), so the + # combination of get_video_info and the _static_decrypt_signature() decryption fallback will work here. + if age_gate: + dash_manifest_url = video_info.get('dashmpd')[0] + else: + dash_manifest_url = ytplayer_config['args']['dashmpd'] + def decrypt_sig(mobj): + s = mobj.group(1) + dec_s = self._decrypt_signature(s, video_id, player_url, age_gate) + return '/signature/%s' % dec_s + dash_manifest_url = re.sub(r'/s/([\w\.]+)', decrypt_sig, dash_manifest_url) dash_doc = self._download_xml( - dash_manifest_url_lst[0], video_id, + dash_manifest_url, video_id, note=u'Downloading DASH manifest', errnote=u'Could not download DASH manifest') for r in dash_doc.findall(u'.//{urn:mpeg:DASH:schema:MPD:2011}Representation'): @@ -1412,7 +1453,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): class YoutubePlaylistIE(YoutubeBaseInfoExtractor): IE_DESC = u'YouTube.com playlists' - _VALID_URL = r"""(?: + _VALID_URL = r"""(?x)(?: (?:https?://)? (?:\w+\.)? youtube\.com/ @@ -1421,21 +1462,20 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): \? (?:.*?&)*? (?:p|a|list)= | p/ ) - ((?:PL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,}) + ( + (?:PL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,} + # Top tracks, they can also include dots + |(?:MC)[\w\.]* + ) .* | ((?:PL|EC|UU|FL|RD)[0-9A-Za-z-_]{10,}) )""" - _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&page=%s' + _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s' _MORE_PAGES_INDICATOR = r'data-link-type="next"' - _VIDEO_RE = r'href="/watch\?v=(?P[0-9A-Za-z_-]{11})&[^"]*?index=(?P\d+)' + _VIDEO_RE = r'href="\s*/watch\?v=(?P[0-9A-Za-z_-]{11})&[^"]*?index=(?P\d+)' IE_NAME = u'youtube:playlist' - @classmethod - def suitable(cls, url): - """Receives a URL and returns True if suitable for this IE.""" - return re.match(cls._VALID_URL, url, re.VERBOSE) is not None - def _real_initialize(self): self._login() @@ -1448,18 +1488,22 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): # the id of the playlist is just 'RD' + video_id url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id) webpage = self._download_webpage(url, playlist_id, u'Downloading Youtube mix') - title_span = (get_element_by_attribute('class', 'title long-title', webpage) or - get_element_by_attribute('class', 'title ', webpage)) + search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage) + title_span = (search_title('playlist-title') or + search_title('title long-title') or search_title('title')) title = clean_html(title_span) - video_re = r'data-index="\d+".*?href="/watch\?v=([0-9A-Za-z_-]{11})&[^"]*?list=%s' % re.escape(playlist_id) - ids = orderedSet(re.findall(video_re, webpage)) + video_re = r'''(?x)data-video-username="(.*?)".*? + href="/watch\?v=([0-9A-Za-z_-]{11})&[^"]*?list=%s''' % re.escape(playlist_id) + matches = orderedSet(re.findall(video_re, webpage, flags=re.DOTALL)) + # Some of the videos may have been deleted, their username field is empty + ids = [video_id for (username, video_id) in matches if username] url_results = self._ids_to_results(ids) return self.playlist_result(url_results, playlist_id, title) def _real_extract(self, url): # Extract playlist id - mobj = re.match(self._VALID_URL, url, re.VERBOSE) + mobj = re.match(self._VALID_URL, url) if mobj is None: raise ExtractorError(u'Invalid URL: %s' % url) playlist_id = mobj.group(1) or mobj.group(2) @@ -1481,29 +1525,31 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): raise ExtractorError(u'For downloading YouTube.com top lists, use ' u'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True) + url = self._TEMPLATE_URL % playlist_id + page = self._download_webpage(url, playlist_id) + more_widget_html = content_html = page + # Extract the video ids from the playlist pages ids = [] for page_num in itertools.count(1): - url = self._TEMPLATE_URL % (playlist_id, page_num) - page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num) - matches = re.finditer(self._VIDEO_RE, page) + matches = re.finditer(self._VIDEO_RE, content_html) # We remove the duplicates and the link with index 0 # (it's not the first video of the playlist) new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0') ids.extend(new_ids) - if re.search(self._MORE_PAGES_INDICATOR, page) is None: + mobj = re.search(r'data-uix-load-more-href="/?(?P[^"]+)"', more_widget_html) + if not mobj: break - try: - playlist_title = self._og_search_title(page) - except RegexNotFoundError: - self.report_warning( - u'Playlist page is missing OpenGraph title, falling back ...', - playlist_id) - playlist_title = self._html_search_regex( - r'

(.*?)

', page, u'title') + more = self._download_json( + 'https://youtube.com/%s' % mobj.group('more'), playlist_id, 'Downloading page #%s' % page_num) + content_html = more['content_html'] + more_widget_html = more['load_more_widget_html'] + + playlist_title = self._html_search_regex( + r'

\s*(.*?)\s*

', page, u'title') url_results = self._ids_to_results(ids) return self.playlist_result(url_results, playlist_id, playlist_title) @@ -1520,7 +1566,7 @@ class YoutubeTopListIE(YoutubePlaylistIE): channel = mobj.group('chann') title = mobj.group('title') query = compat_urllib_parse.urlencode({'title': title}) - playlist_re = 'href="([^"]+?%s[^"]+?)"' % re.escape(query) + playlist_re = 'href="([^"]+?%s.*?)"' % re.escape(query) channel_page = self._download_webpage('https://www.youtube.com/%s' % channel, title) link = self._html_search_regex(playlist_re, channel_page, u'list') url = compat_urlparse.urljoin('https://www.youtube.com/', link) @@ -1545,7 +1591,7 @@ class YoutubeChannelIE(InfoExtractor): IE_DESC = u'YouTube.com channels' _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)" _MORE_PAGES_INDICATOR = 'yt-uix-load-more' - _MORE_PAGES_URL = 'http://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s' + _MORE_PAGES_URL = 'https://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s' IE_NAME = u'youtube:channel' def extract_videos_from_page(self, page): @@ -1580,11 +1626,10 @@ class YoutubeChannelIE(InfoExtractor): # Download all channel pages using the json-based channel_ajax query for pagenum in itertools.count(1): url = self._MORE_PAGES_URL % (pagenum, channel_id) - page = self._download_webpage(url, channel_id, - u'Downloading page #%s' % pagenum) - - page = json.loads(page) - + page = self._download_json( + url, channel_id, note=u'Downloading page #%s' % pagenum, + transform_source=uppercase_escape) + ids_in_page = self.extract_videos_from_page(page['content_html']) video_ids.extend(ids_in_page) @@ -1601,9 +1646,9 @@ class YoutubeChannelIE(InfoExtractor): class YoutubeUserIE(InfoExtractor): IE_DESC = u'YouTube.com user videos (URL or "ytuser" keyword)' _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)' - _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s' + _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/users/%s' _GDATA_PAGE_SIZE = 50 - _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json' + _GDATA_URL = 'https://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json' IE_NAME = u'youtube:user' @classmethod @@ -1652,7 +1697,7 @@ class YoutubeUserIE(InfoExtractor): '_type': 'url', 'url': video_id, 'ie_key': 'Youtube', - 'id': 'video_id', + 'id': video_id, 'title': title, } url_results = PagedList(download_page, self._GDATA_PAGE_SIZE) @@ -1684,7 +1729,8 @@ class YoutubeSearchIE(SearchInfoExtractor): api_response = data['data'] if 'items' not in api_response: - raise ExtractorError(u'[youtube] No video results') + raise ExtractorError( + u'[youtube] No video results', expected=True) new_ids = list(video['id'] for video in api_response['items']) video_ids += new_ids @@ -1734,7 +1780,7 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor): action = 'action_load_system_feed' if self._PERSONAL_FEED: action = 'action_load_personal_feed' - return 'http://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME) + return 'https://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME) @property def IE_NAME(self): @@ -1803,7 +1849,10 @@ class YoutubeFavouritesIE(YoutubeBaseInfoExtractor): class YoutubeTruncatedURLIE(InfoExtractor): IE_NAME = 'youtube:truncated_url' IE_DESC = False # Do not list - _VALID_URL = r'(?:https?://)?[^/]+/watch\?feature=[a-z_]+$' + _VALID_URL = r'''(?x) + (?:https?://)?[^/]+/watch\?(?:feature=[a-z_]+)?$| + (?:https?://)?(?:www\.)?youtube\.com/attribution_link\?a=[^&]+$ + ''' def _real_extract(self, url): raise ExtractorError(