3 from __future__ import unicode_literals
14 from .common import InfoExtractor, SearchInfoExtractor
15 from ..jsinterp import JSInterpreter
16 from ..swfinterp import SWFInterpreter
17 from ..compat import (
20 compat_urllib_parse_unquote,
21 compat_urllib_parse_unquote_plus,
22 compat_urllib_parse_urlencode,
23 compat_urllib_parse_urlparse,
32 get_element_by_attribute,
54 class YoutubeBaseInfoExtractor(InfoExtractor):
55 """Provide base functions for Youtube extractors"""
56 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
57 _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
58 _PASSWORD_CHALLENGE_URL = 'https://accounts.google.com/signin/challenge/sl/password'
59 _NETRC_MACHINE = 'youtube'
60 # If True it will raise an error if no login info is provided
61 _LOGIN_REQUIRED = False
63 def _set_language(self):
65 '.youtube.com', 'PREF', 'f1=50000000&hl=en',
66 # YouTube sets the expire time to about two months
67 expire_time=time.time() + 2 * 30 * 24 * 3600)
69 def _ids_to_results(self, ids):
71 self.url_result(vid_id, 'Youtube', video_id=vid_id)
76 Attempt to log in to YouTube.
77 True is returned if successful or skipped.
78 False is returned if login failed.
80 If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
82 (username, password) = self._get_login_info()
83 # No authentication to be performed
85 if self._LOGIN_REQUIRED:
86 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
89 login_page = self._download_webpage(
90 self._LOGIN_URL, None,
91 note='Downloading login page',
92 errnote='unable to fetch login page', fatal=False)
93 if login_page is False:
96 login_form = self._hidden_inputs(login_page)
99 'checkConnection': 'youtube',
104 login_results = self._download_webpage(
105 self._PASSWORD_CHALLENGE_URL, None,
106 note='Logging in', errnote='unable to log in', fatal=False,
107 data=urlencode_postdata(login_form))
108 if login_results is False:
111 error_msg = self._html_search_regex(
112 r'<[^>]+id="errormsg_0_Passwd"[^>]*>([^<]+)<',
113 login_results, 'error message', default=None)
115 raise ExtractorError('Unable to login: %s' % error_msg, expected=True)
117 if re.search(r'id="errormsg_0_Passwd"', login_results) is not None:
118 raise ExtractorError('Please use your account password and a two-factor code instead of an application-specific password.', expected=True)
121 # TODO add SMS and phone call support - these require making a request and then prompting the user
123 if re.search(r'(?i)<form[^>]+id="challenge"', login_results) is not None:
124 tfa_code = self._get_tfa_info('2-step verification code')
127 self._downloader.report_warning(
128 'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
129 '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
132 tfa_code = remove_start(tfa_code, 'G-')
134 tfa_form_strs = self._form_hidden_inputs('challenge', login_results)
136 tfa_form_strs.update({
141 tfa_data = urlencode_postdata(tfa_form_strs)
143 tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
144 tfa_results = self._download_webpage(
146 note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)
148 if tfa_results is False:
151 if re.search(r'(?i)<form[^>]+id="challenge"', tfa_results) is not None:
152 self._downloader.report_warning('Two-factor code expired or invalid. Please try again, or use a one-use backup code instead.')
154 if re.search(r'(?i)<form[^>]+id="gaia_loginform"', tfa_results) is not None:
155 self._downloader.report_warning('unable to log in - did the page structure change?')
157 if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None:
158 self._downloader.report_warning('Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
161 if re.search(r'(?i)<form[^>]+id="gaia_loginform"', login_results) is not None:
162 self._downloader.report_warning('unable to log in: bad username or password')
166 def _real_initialize(self):
167 if self._downloader is None:
170 if not self._login():
174 class YoutubeEntryListBaseInfoExtractor(YoutubeBaseInfoExtractor):
175 # Extract entries from page with "Load more" button
176 def _entries(self, page, playlist_id):
177 more_widget_html = content_html = page
178 for page_num in itertools.count(1):
179 for entry in self._process_page(content_html):
182 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
186 more = self._download_json(
187 'https://youtube.com/%s' % mobj.group('more'), playlist_id,
188 'Downloading page #%s' % page_num,
189 transform_source=uppercase_escape)
190 content_html = more['content_html']
191 if not content_html.strip():
192 # Some webpages show a "Load more" button but they don't
195 more_widget_html = more['load_more_widget_html']
198 class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
199 def _process_page(self, content):
200 for video_id, video_title in self.extract_videos_from_page(content):
201 yield self.url_result(video_id, 'Youtube', video_id, video_title)
203 def extract_videos_from_page(self, page):
206 for mobj in re.finditer(self._VIDEO_RE, page):
207 # The link with index 0 is not the first video of the playlist (not sure if still actual)
208 if 'index' in mobj.groupdict() and mobj.group('id') == '0':
210 video_id = mobj.group('id')
211 video_title = unescapeHTML(mobj.group('title'))
213 video_title = video_title.strip()
215 idx = ids_in_page.index(video_id)
216 if video_title and not titles_in_page[idx]:
217 titles_in_page[idx] = video_title
219 ids_in_page.append(video_id)
220 titles_in_page.append(video_title)
221 return zip(ids_in_page, titles_in_page)
224 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
225 def _process_page(self, content):
226 for playlist_id in orderedSet(re.findall(
227 r'<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*><a[^>]+href="/?playlist\?list=([0-9A-Za-z-_]{10,})"',
229 yield self.url_result(
230 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
232 def _real_extract(self, url):
233 playlist_id = self._match_id(url)
234 webpage = self._download_webpage(url, playlist_id)
235 title = self._og_search_title(webpage, fatal=False)
236 return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
239 class YoutubeIE(YoutubeBaseInfoExtractor):
240 IE_DESC = 'YouTube.com'
241 _VALID_URL = r"""(?x)^
243 (?:https?://|//) # http(s):// or protocol-independent URL
244 (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
245 (?:www\.)?deturl\.com/www\.youtube\.com/|
246 (?:www\.)?pwnyoutube\.com/|
247 (?:www\.)?yourepeat\.com/|
248 tube\.majestyc\.net/|
249 youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
250 (?:.*?\#/)? # handle anchor (#/) redirect urls
251 (?: # the various things that can precede the ID:
252 (?:(?:v|embed|e)/(?!videoseries)) # v/ or embed/ or e/
253 |(?: # or the v= param in all its forms
254 (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
255 (?:\?|\#!?) # the params delimiter ? or # or #!
256 (?:.*?[&;])?? # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&v=V36LpHqtcDY)
261 youtu\.be| # just youtu.be/xxxx
262 vid\.plus| # or vid.plus/xxxx
263 zwearz\.com/watch| # or zwearz.com/watch/xxxx
265 |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
267 )? # all until now is optional -> you can pass the naked ID
268 ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
269 (?!.*?\blist=) # combined list/video URLs are handled by the playlist IE
270 (?(1).+)? # if we found the ID, everything can follow
272 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
274 '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
275 '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
276 '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
277 '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
278 '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
279 '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
280 '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
281 '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
282 # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
283 '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
284 '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
285 '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
286 '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
287 '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
288 '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
289 '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
290 '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
291 '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
295 '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
296 '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
297 '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
298 '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
299 '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
300 '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
301 '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
303 # Apple HTTP Live Streaming
304 '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
305 '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
306 '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
307 '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
308 '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
309 '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
310 '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
311 '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
314 '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
315 '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
316 '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
317 '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
318 '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
319 '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40}, # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
320 '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
321 '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
322 '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
323 '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
324 '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60, 'preference': -40},
325 '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264', 'preference': -40},
328 '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
329 '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
330 '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
331 '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
332 '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'preference': -50, 'container': 'm4a_dash'},
333 '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'preference': -50, 'container': 'm4a_dash'},
334 '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'preference': -50, 'container': 'm4a_dash'},
337 '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
338 '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
339 '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
340 '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
341 '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
342 '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
343 '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9', 'preference': -40},
344 '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
345 '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
346 '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
347 '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
348 '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
349 '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
350 '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
351 '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
352 # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
353 '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
354 '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
355 '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
356 '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
357 '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'preference': -40},
358 '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60, 'preference': -40},
361 '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
362 '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
364 # Dash webm audio with opus inside
365 '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
366 '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
367 '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
370 '_rtmp': {'protocol': 'rtmp'},
372 _SUBTITLE_FORMATS = ('ttml', 'vtt')
377 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
381 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
382 'uploader': 'Philipp Hagemeister',
383 'uploader_id': 'phihag',
384 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
385 'upload_date': '20121002',
386 'license': 'Standard YouTube License',
387 'description': 'test chars: "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
388 'categories': ['Science & Technology'],
389 'tags': ['youtube-dl'],
392 'dislike_count': int,
398 'url': 'https://www.youtube.com/watch?v=UxxajLWwzqY',
399 'note': 'Test generic use_cipher_signature video (#897)',
403 'upload_date': '20120506',
404 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
405 'alt_title': 'I Love It (feat. Charli XCX)',
406 'description': 'md5:f3ceb5ef83a08d95b9d146f973157cc8',
407 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
408 'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
409 'iconic ep', 'iconic', 'love', 'it'],
411 'uploader': 'Icona Pop',
412 'uploader_id': 'IconaPop',
413 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IconaPop',
414 'license': 'Standard YouTube License',
415 'creator': 'Icona Pop',
419 'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
420 'note': 'Test VEVO video with age protection (#956)',
424 'upload_date': '20130703',
425 'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
426 'alt_title': 'Tunnel Vision',
427 'description': 'md5:64249768eec3bc4276236606ea996373',
429 'uploader': 'justintimberlakeVEVO',
430 'uploader_id': 'justintimberlakeVEVO',
431 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
432 'license': 'Standard YouTube License',
433 'creator': 'Justin Timberlake',
438 'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
439 'note': 'Embed-only video (#1746)',
443 'upload_date': '20120608',
444 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
445 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
446 'uploader': 'SET India',
447 'uploader_id': 'setindia',
448 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
449 'license': 'Standard YouTube License',
454 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
455 'note': 'Use the first video ID in the URL',
459 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
460 'uploader': 'Philipp Hagemeister',
461 'uploader_id': 'phihag',
462 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
463 'upload_date': '20121002',
464 'license': 'Standard YouTube License',
465 'description': 'test chars: "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
466 'categories': ['Science & Technology'],
467 'tags': ['youtube-dl'],
470 'dislike_count': int,
473 'skip_download': True,
477 'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
478 'note': '256k DASH audio (format 141) via DASH manifest',
482 'upload_date': '20121002',
483 'uploader_id': '8KVIDEO',
484 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
486 'uploader': '8KVIDEO',
487 'license': 'Standard YouTube License',
488 'title': 'UHDTV TEST 8K VIDEO.mp4'
491 'youtube_include_dash_manifest': True,
494 'skip': 'format 141 not served anymore',
496 # DASH manifest with encrypted signature
498 'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
502 'title': 'Afrojack, Spree Wilson - The Spark ft. Spree Wilson',
503 'description': 'md5:12e7067fa6735a77bdcbb58cb1187d2d',
505 'uploader': 'AfrojackVEVO',
506 'uploader_id': 'AfrojackVEVO',
507 'upload_date': '20131011',
508 'license': 'Standard YouTube License',
511 'youtube_include_dash_manifest': True,
512 'format': '141/bestaudio[ext=m4a]',
515 # JS player signature function name containing $
517 'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
521 'title': 'Taylor Swift - Shake It Off',
522 'alt_title': 'Shake It Off',
523 'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
525 'uploader': 'TaylorSwiftVEVO',
526 'uploader_id': 'TaylorSwiftVEVO',
527 'upload_date': '20140818',
528 'license': 'Standard YouTube License',
529 'creator': 'Taylor Swift',
532 'youtube_include_dash_manifest': True,
533 'format': '141/bestaudio[ext=m4a]',
538 'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
543 'upload_date': '20100909',
544 'uploader': 'The Amazing Atheist',
545 'uploader_id': 'TheAmazingAtheist',
546 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
547 'license': 'Standard YouTube License',
548 'title': 'Burning Everyone\'s Koran',
549 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
552 # Normal age-gate video (No vevo, embed allowed)
554 'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
558 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
559 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
561 'uploader': 'The Witcher',
562 'uploader_id': 'WitcherGame',
563 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
564 'upload_date': '20140605',
565 'license': 'Standard YouTube License',
569 # Age-gate video with encrypted signature
571 'url': 'https://www.youtube.com/watch?v=6kLq3WMV1nU',
575 'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
576 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
578 'uploader': 'LloydVEVO',
579 'uploader_id': 'LloydVEVO',
580 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
581 'upload_date': '20110629',
582 'license': 'Standard YouTube License',
586 # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
588 'url': '__2ABJjxzNo',
593 'upload_date': '20100430',
594 'uploader_id': 'deadmau5',
595 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
596 'creator': 'deadmau5',
597 'description': 'md5:12c56784b8032162bb936a5f76d55360',
598 'uploader': 'deadmau5',
599 'license': 'Standard YouTube License',
600 'title': 'Deadmau5 - Some Chords (HD)',
601 'alt_title': 'Some Chords',
603 'expected_warnings': [
604 'DASH manifest missing',
607 # Olympics (https://github.com/rg3/youtube-dl/issues/4431)
609 'url': 'lqQg6PlCWgI',
614 'upload_date': '20150827',
615 'uploader_id': 'olympic',
616 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
617 'license': 'Standard YouTube License',
618 'description': 'HO09 - Women - GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
619 'uploader': 'Olympic',
620 'title': 'Hockey - Women - GER-AUS - London 2012 Olympic Games',
623 'skip_download': 'requires avconv',
628 'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
632 'stretched_ratio': 16 / 9.,
634 'upload_date': '20110310',
635 'uploader_id': 'AllenMeow',
636 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
637 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
639 'license': 'Standard YouTube License',
640 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
643 # url_encoded_fmt_stream_map is empty string
645 'url': 'qEJwOuvDf7I',
649 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
651 'upload_date': '20150404',
652 'uploader_id': 'spbelect',
653 'uploader': 'Наблюдатели Петербурга',
656 'skip_download': 'requires avconv',
658 'skip': 'This live event has ended.',
660 # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
662 'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
666 'title': 'md5:7b81415841e02ecd4313668cde88737a',
667 'description': 'md5:116377fd2963b81ec4ce64b542173306',
669 'upload_date': '20150625',
670 'uploader_id': 'dorappi2000',
671 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
672 'uploader': 'dorappi2000',
673 'license': 'Standard YouTube License',
674 'formats': 'mincount:32',
677 # DASH manifest with segment_list
679 'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
680 'md5': '8ce563a1d667b599d21064e982ab9e31',
684 'upload_date': '20150501', # According to '<meta itemprop="datePublished"', but in other places it's 20150510
685 'uploader': 'Airtek',
686 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
687 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
688 'license': 'Standard YouTube License',
689 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
692 'youtube_include_dash_manifest': True,
693 'format': '135', # bestvideo
695 'skip': 'This live event has ended.',
698 # Multifeed videos (multiple cameras), URL is for Main Camera
699 'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
702 'title': 'teamPGP: Rocket League Noob Stream',
703 'description': 'md5:dc7872fb300e143831327f1bae3af010',
709 'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
710 'description': 'md5:dc7872fb300e143831327f1bae3af010',
712 'upload_date': '20150721',
713 'uploader': 'Beer Games Beer',
714 'uploader_id': 'beergamesbeer',
715 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
716 'license': 'Standard YouTube License',
722 'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
723 'description': 'md5:dc7872fb300e143831327f1bae3af010',
725 'upload_date': '20150721',
726 'uploader': 'Beer Games Beer',
727 'uploader_id': 'beergamesbeer',
728 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
729 'license': 'Standard YouTube License',
735 'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
736 'description': 'md5:dc7872fb300e143831327f1bae3af010',
738 'upload_date': '20150721',
739 'uploader': 'Beer Games Beer',
740 'uploader_id': 'beergamesbeer',
741 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
742 'license': 'Standard YouTube License',
748 'title': 'teamPGP: Rocket League Noob Stream (zim)',
749 'description': 'md5:dc7872fb300e143831327f1bae3af010',
751 'upload_date': '20150721',
752 'uploader': 'Beer Games Beer',
753 'uploader_id': 'beergamesbeer',
754 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
755 'license': 'Standard YouTube License',
759 'skip_download': True,
763 # Multifeed video with comma in title (see https://github.com/rg3/youtube-dl/issues/8536)
764 'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
767 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
770 'skip': 'Not multifeed anymore',
773 'url': 'https://vid.plus/FlRa-iH7PGw',
774 'only_matching': True,
777 'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
778 'only_matching': True,
781 # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
782 # Also tests cut-off URL expansion in video description (see
783 # https://github.com/rg3/youtube-dl/issues/1892,
784 # https://github.com/rg3/youtube-dl/issues/8164)
785 'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
789 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
790 'alt_title': 'Dark Walk',
791 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
793 'upload_date': '20151119',
794 'uploader_id': 'IronSoulElf',
795 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
796 'uploader': 'IronSoulElf',
797 'license': 'Standard YouTube License',
798 'creator': 'Todd Haberman, Daniel Law Heath & Aaron Kaplan',
801 'skip_download': True,
805 # Tags with '};' (see https://github.com/rg3/youtube-dl/issues/7468)
806 'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
807 'only_matching': True,
810 # Video with yt:stretch=17:0
811 'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
815 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
816 'description': 'md5:ee18a25c350637c8faff806845bddee9',
817 'upload_date': '20151107',
818 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
819 'uploader': 'CH GAMER DROID',
822 'skip_download': True,
824 'skip': 'This video does not exist.',
827 # Video licensed under Creative Commons
828 'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
832 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
833 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
835 'upload_date': '20150127',
836 'uploader_id': 'BerkmanCenter',
837 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
838 'uploader': 'The Berkman Klein Center for Internet & Society',
839 'license': 'Creative Commons Attribution license (reuse allowed)',
842 'skip_download': True,
846 # Channel-like uploader_url
847 'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
851 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
852 'description': 'md5:dda0d780d5a6e120758d1711d062a867',
854 'upload_date': '20151119',
855 'uploader': 'Bernie 2016',
856 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
857 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
858 'license': 'Creative Commons Attribution license (reuse allowed)',
861 'skip_download': True,
865 'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;v=V36LpHqtcDY',
866 'only_matching': True,
869 # YouTube Red paid video (https://github.com/rg3/youtube-dl/issues/10059)
870 'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
871 'only_matching': True,
874 # Rental video preview
875 'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
879 'title': 'Piku - Trailer',
880 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
881 'upload_date': '20150811',
882 'uploader': 'FlixMatrix',
883 'uploader_id': 'FlixMatrixKaravan',
884 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
885 'license': 'Standard YouTube License',
888 'skip_download': True,
892 # YouTube Red video with episode data
893 'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
897 'title': 'Isolation - Mind Field (Ep 1)',
898 'description': 'md5:8013b7ddea787342608f63a13ddc9492',
900 'upload_date': '20170118',
901 'uploader': 'Vsauce',
902 'uploader_id': 'Vsauce',
903 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
904 'license': 'Standard YouTube License',
905 'series': 'Mind Field',
910 'skip_download': True,
912 'expected_warnings': [
913 'Skipping DASH manifest',
918 'url': '1t24XAntNCY',
919 'only_matching': True,
923 def __init__(self, *args, **kwargs):
924 super(YoutubeIE, self).__init__(*args, **kwargs)
925 self._player_cache = {}
927 def report_video_info_webpage_download(self, video_id):
928 """Report attempt to download video info webpage."""
929 self.to_screen('%s: Downloading video info webpage' % video_id)
931 def report_information_extraction(self, video_id):
932 """Report attempt to extract video information."""
933 self.to_screen('%s: Extracting video information' % video_id)
935 def report_unavailable_format(self, video_id, format):
936 """Report extracted video URL."""
937 self.to_screen('%s: Format %s not available' % (video_id, format))
939 def report_rtmp_download(self):
940 """Indicate the download will use the RTMP protocol."""
941 self.to_screen('RTMP download detected')
943 def _signature_cache_id(self, example_sig):
944 """ Return a string representation of a signature """
945 return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
947 def _extract_signature_function(self, video_id, player_url, example_sig):
949 r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|/base)?\.(?P<ext>[a-z]+)$',
952 raise ExtractorError('Cannot identify player %r' % player_url)
953 player_type = id_m.group('ext')
954 player_id = id_m.group('id')
956 # Read from filesystem cache
957 func_id = '%s_%s_%s' % (
958 player_type, player_id, self._signature_cache_id(example_sig))
959 assert os.path.basename(func_id) == func_id
961 cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
962 if cache_spec is not None:
963 return lambda s: ''.join(s[i] for i in cache_spec)
966 'Downloading player %s' % player_url
967 if self._downloader.params.get('verbose') else
968 'Downloading %s player %s' % (player_type, player_id)
970 if player_type == 'js':
971 code = self._download_webpage(
972 player_url, video_id,
974 errnote='Download of %s failed' % player_url)
975 res = self._parse_sig_js(code)
976 elif player_type == 'swf':
977 urlh = self._request_webpage(
978 player_url, video_id,
980 errnote='Download of %s failed' % player_url)
982 res = self._parse_sig_swf(code)
984 assert False, 'Invalid player type %r' % player_type
986 test_string = ''.join(map(compat_chr, range(len(example_sig))))
987 cache_res = res(test_string)
988 cache_spec = [ord(c) for c in cache_res]
990 self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
993 def _print_sig_code(self, func, example_sig):
994 def gen_sig_code(idxs):
995 def _genslice(start, end, step):
996 starts = '' if start == 0 else str(start)
997 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
998 steps = '' if step == 1 else (':%d' % step)
999 return 's[%s%s%s]' % (starts, ends, steps)
1002 # Quelch pyflakes warnings - start will be set when step is set
1003 start = '(Never used)'
1004 for i, prev in zip(idxs[1:], idxs[:-1]):
1005 if step is not None:
1006 if i - prev == step:
1008 yield _genslice(start, prev, step)
1011 if i - prev in [-1, 1]:
1016 yield 's[%d]' % prev
1020 yield _genslice(start, i, step)
1022 test_string = ''.join(map(compat_chr, range(len(example_sig))))
1023 cache_res = func(test_string)
1024 cache_spec = [ord(c) for c in cache_res]
1025 expr_code = ' + '.join(gen_sig_code(cache_spec))
1026 signature_id_tuple = '(%s)' % (
1027 ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
1028 code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
1029 ' return %s\n') % (signature_id_tuple, expr_code)
1030 self.to_screen('Extracted signature function:\n' + code)
1032 def _parse_sig_js(self, jscode):
1033 funcname = self._search_regex(
1034 (r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1035 r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\('),
1036 jscode, 'Initial JS player signature function name', group='sig')
1038 jsi = JSInterpreter(jscode)
1039 initial_function = jsi.extract_function(funcname)
1040 return lambda s: initial_function([s])
1042 def _parse_sig_swf(self, file_contents):
1043 swfi = SWFInterpreter(file_contents)
1044 TARGET_CLASSNAME = 'SignatureDecipher'
1045 searched_class = swfi.extract_class(TARGET_CLASSNAME)
1046 initial_function = swfi.extract_function(searched_class, 'decipher')
1047 return lambda s: initial_function([s])
1049 def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
1050 """Turn the encrypted s field into a working signature"""
1052 if player_url is None:
1053 raise ExtractorError('Cannot decrypt signature without player_url')
1055 if player_url.startswith('//'):
1056 player_url = 'https:' + player_url
1057 elif not re.match(r'https?://', player_url):
1058 player_url = compat_urlparse.urljoin(
1059 'https://www.youtube.com', player_url)
1061 player_id = (player_url, self._signature_cache_id(s))
1062 if player_id not in self._player_cache:
1063 func = self._extract_signature_function(
1064 video_id, player_url, s
1066 self._player_cache[player_id] = func
1067 func = self._player_cache[player_id]
1068 if self._downloader.params.get('youtube_print_sig_code'):
1069 self._print_sig_code(func, s)
1071 except Exception as e:
1072 tb = traceback.format_exc()
1073 raise ExtractorError(
1074 'Signature extraction failed: ' + tb, cause=e)
1076 def _get_subtitles(self, video_id, webpage):
1078 subs_doc = self._download_xml(
1079 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
1080 video_id, note=False)
1081 except ExtractorError as err:
1082 self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
1086 for track in subs_doc.findall('track'):
1087 lang = track.attrib['lang_code']
1088 if lang in sub_lang_list:
1091 for ext in self._SUBTITLE_FORMATS:
1092 params = compat_urllib_parse_urlencode({
1096 'name': track.attrib['name'].encode('utf-8'),
1098 sub_formats.append({
1099 'url': 'https://www.youtube.com/api/timedtext?' + params,
1102 sub_lang_list[lang] = sub_formats
1103 if not sub_lang_list:
1104 self._downloader.report_warning('video doesn\'t have subtitles')
1106 return sub_lang_list
1108 def _get_ytplayer_config(self, video_id, webpage):
1110 # User data may contain arbitrary character sequences that may affect
1111 # JSON extraction with regex, e.g. when '};' is contained the second
1112 # regex won't capture the whole JSON. Yet working around by trying more
1113 # concrete regex first keeping in mind proper quoted string handling
1114 # to be implemented in future that will replace this workaround (see
1115 # https://github.com/rg3/youtube-dl/issues/7468,
1116 # https://github.com/rg3/youtube-dl/pull/7599)
1117 r';ytplayer\.config\s*=\s*({.+?});ytplayer',
1118 r';ytplayer\.config\s*=\s*({.+?});',
1120 config = self._search_regex(
1121 patterns, webpage, 'ytplayer.config', default=None)
1123 return self._parse_json(
1124 uppercase_escape(config), video_id, fatal=False)
1126 def _get_automatic_captions(self, video_id, webpage):
1127 """We need the webpage for getting the captions url, pass it as an
1128 argument to speed up the process."""
1129 self.to_screen('%s: Looking for automatic captions' % video_id)
1130 player_config = self._get_ytplayer_config(video_id, webpage)
1131 err_msg = 'Couldn\'t find automatic captions for %s' % video_id
1132 if not player_config:
1133 self._downloader.report_warning(err_msg)
1136 args = player_config['args']
1137 caption_url = args.get('ttsurl')
1139 timestamp = args['timestamp']
1140 # We get the available subtitles
1141 list_params = compat_urllib_parse_urlencode({
1146 list_url = caption_url + '&' + list_params
1147 caption_list = self._download_xml(list_url, video_id)
1148 original_lang_node = caption_list.find('track')
1149 if original_lang_node is None:
1150 self._downloader.report_warning('Video doesn\'t have automatic captions')
1152 original_lang = original_lang_node.attrib['lang_code']
1153 caption_kind = original_lang_node.attrib.get('kind', '')
1156 for lang_node in caption_list.findall('target'):
1157 sub_lang = lang_node.attrib['lang_code']
1159 for ext in self._SUBTITLE_FORMATS:
1160 params = compat_urllib_parse_urlencode({
1161 'lang': original_lang,
1165 'kind': caption_kind,
1167 sub_formats.append({
1168 'url': caption_url + '&' + params,
1171 sub_lang_list[sub_lang] = sub_formats
1172 return sub_lang_list
1174 # Some videos don't provide ttsurl but rather caption_tracks and
1175 # caption_translation_languages (e.g. 20LmZk1hakA)
1176 caption_tracks = args['caption_tracks']
1177 caption_translation_languages = args['caption_translation_languages']
1178 caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
1179 parsed_caption_url = compat_urllib_parse_urlparse(caption_url)
1180 caption_qs = compat_parse_qs(parsed_caption_url.query)
1183 for lang in caption_translation_languages.split(','):
1184 lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
1185 sub_lang = lang_qs.get('lc', [None])[0]
1189 for ext in self._SUBTITLE_FORMATS:
1191 'tlang': [sub_lang],
1194 sub_url = compat_urlparse.urlunparse(parsed_caption_url._replace(
1195 query=compat_urllib_parse_urlencode(caption_qs, True)))
1196 sub_formats.append({
1200 sub_lang_list[sub_lang] = sub_formats
1201 return sub_lang_list
1202 # An extractor error can be raise by the download process if there are
1203 # no automatic captions but there are subtitles
1204 except (KeyError, ExtractorError):
1205 self._downloader.report_warning(err_msg)
1208 def _mark_watched(self, video_id, video_info):
1209 playback_url = video_info.get('videostats_playback_base_url', [None])[0]
1210 if not playback_url:
1212 parsed_playback_url = compat_urlparse.urlparse(playback_url)
1213 qs = compat_urlparse.parse_qs(parsed_playback_url.query)
1215 # cpn generation algorithm is reverse engineered from base.js.
1216 # In fact it works even with dummy cpn.
1217 CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
1218 cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
1224 playback_url = compat_urlparse.urlunparse(
1225 parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
1227 self._download_webpage(
1228 playback_url, video_id, 'Marking watched',
1229 'Unable to mark watched', fatal=False)
1232 def extract_id(cls, url):
1233 mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
1235 raise ExtractorError('Invalid URL: %s' % url)
1236 video_id = mobj.group(2)
1239 def _extract_from_m3u8(self, manifest_url, video_id):
1242 def _get_urls(_manifest):
1243 lines = _manifest.split('\n')
1244 urls = filter(lambda l: l and not l.startswith('#'),
1247 manifest = self._download_webpage(manifest_url, video_id, 'Downloading formats manifest')
1248 formats_urls = _get_urls(manifest)
1249 for format_url in formats_urls:
1250 itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
1251 url_map[itag] = format_url
1254 def _extract_annotations(self, video_id):
1255 url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
1256 return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
1258 def _real_extract(self, url):
1259 url, smuggled_data = unsmuggle_url(url, {})
1262 'http' if self._downloader.params.get('prefer_insecure', False)
1267 parsed_url = compat_urllib_parse_urlparse(url)
1268 for component in [parsed_url.fragment, parsed_url.query]:
1269 query = compat_parse_qs(component)
1270 if start_time is None and 't' in query:
1271 start_time = parse_duration(query['t'][0])
1272 if start_time is None and 'start' in query:
1273 start_time = parse_duration(query['start'][0])
1274 if end_time is None and 'end' in query:
1275 end_time = parse_duration(query['end'][0])
1277 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1278 mobj = re.search(self._NEXT_URL_RE, url)
1280 url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
1281 video_id = self.extract_id(url)
1284 url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
1285 video_webpage = self._download_webpage(url, video_id)
1287 # Attempt to extract SWF player URL
1288 mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1289 if mobj is not None:
1290 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1296 def add_dash_mpd(video_info):
1297 dash_mpd = video_info.get('dashmpd')
1298 if dash_mpd and dash_mpd[0] not in dash_mpds:
1299 dash_mpds.append(dash_mpd[0])
1302 embed_webpage = None
1304 if re.search(r'player-age-gate-content">', video_webpage) is not None:
1306 # We simulate the access to the video from www.youtube.com/v/{video_id}
1307 # this can be viewed without login into Youtube
1308 url = proto + '://www.youtube.com/embed/%s' % video_id
1309 embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
1310 data = compat_urllib_parse_urlencode({
1311 'video_id': video_id,
1312 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1313 'sts': self._search_regex(
1314 r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
1316 video_info_url = proto + '://www.youtube.com/get_video_info?' + data
1317 video_info_webpage = self._download_webpage(
1318 video_info_url, video_id,
1319 note='Refetching age-gated info webpage',
1320 errnote='unable to download video info webpage')
1321 video_info = compat_parse_qs(video_info_webpage)
1322 add_dash_mpd(video_info)
1326 # Try looking directly into the video webpage
1327 ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
1329 args = ytplayer_config['args']
1330 if args.get('url_encoded_fmt_stream_map'):
1331 # Convert to the same format returned by compat_parse_qs
1332 video_info = dict((k, [v]) for k, v in args.items())
1333 add_dash_mpd(video_info)
1334 # Rental video is not rented but preview is available (e.g.
1335 # https://www.youtube.com/watch?v=yYr8q0y5Jfg,
1336 # https://github.com/rg3/youtube-dl/issues/10532)
1337 if not video_info and args.get('ypc_vid'):
1338 return self.url_result(
1339 args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
1340 if args.get('livestream') == '1' or args.get('live_playback') == 1:
1342 if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
1343 # We also try looking in get_video_info since it may contain different dashmpd
1344 # URL that points to a DASH manifest with possibly different itag set (some itags
1345 # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
1346 # manifest pointed by get_video_info's dashmpd).
1347 # The general idea is to take a union of itags of both DASH manifests (for example
1348 # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
1349 self.report_video_info_webpage_download(video_id)
1350 for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
1352 '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1353 % (proto, video_id, el_type))
1354 video_info_webpage = self._download_webpage(
1356 video_id, note=False,
1357 errnote='unable to download video info webpage')
1358 get_video_info = compat_parse_qs(video_info_webpage)
1359 if get_video_info.get('use_cipher_signature') != ['True']:
1360 add_dash_mpd(get_video_info)
1362 video_info = get_video_info
1363 if 'token' in get_video_info:
1364 # Different get_video_info requests may report different results, e.g.
1365 # some may report video unavailability, but some may serve it without
1366 # any complaint (see https://github.com/rg3/youtube-dl/issues/7362,
1367 # the original webpage as well as el=info and el=embedded get_video_info
1368 # requests report video unavailability due to geo restriction while
1369 # el=detailpage succeeds and returns valid data). This is probably
1370 # due to YouTube measures against IP ranges of hosting providers.
1371 # Working around by preferring the first succeeded video_info containing
1372 # the token if no such video_info yet was found.
1373 if 'token' not in video_info:
1374 video_info = get_video_info
1376 if 'token' not in video_info:
1377 if 'reason' in video_info:
1378 if 'The uploader has not made this video available in your country.' in video_info['reason']:
1379 regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
1381 raise ExtractorError('YouTube said: This video is available in %s only' % (
1382 ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
1384 raise ExtractorError(
1385 'YouTube said: %s' % video_info['reason'][0],
1386 expected=True, video_id=video_id)
1388 raise ExtractorError(
1389 '"token" parameter not in video info for unknown reason',
1393 if 'title' in video_info:
1394 video_title = video_info['title'][0]
1396 self._downloader.report_warning('Unable to extract video title')
1400 video_description = get_element_by_id("eow-description", video_webpage)
1401 if video_description:
1402 video_description = re.sub(r'''(?x)
1404 (?:[a-zA-Z-]+="[^"]*"\s+)*?
1405 (?:title|href)="([^"]+)"\s+
1406 (?:[a-zA-Z-]+="[^"]*"\s+)*?
1410 ''', r'\1', video_description)
1411 video_description = clean_html(video_description)
1413 fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
1415 video_description = unescapeHTML(fd_mobj.group(1))
1417 video_description = ''
1419 if 'multifeed_metadata_list' in video_info and not smuggled_data.get('force_singlefeed', False):
1420 if not self._downloader.params.get('noplaylist'):
1423 multifeed_metadata_list = video_info['multifeed_metadata_list'][0]
1424 for feed in multifeed_metadata_list.split(','):
1425 # Unquote should take place before split on comma (,) since textual
1426 # fields may contain comma as well (see
1427 # https://github.com/rg3/youtube-dl/issues/8536)
1428 feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
1430 '_type': 'url_transparent',
1431 'ie_key': 'Youtube',
1433 '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
1434 {'force_singlefeed': True}),
1435 'title': '%s (%s)' % (video_title, feed_data['title'][0]),
1437 feed_ids.append(feed_data['id'][0])
1439 'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
1440 % (', '.join(feed_ids), video_id))
1441 return self.playlist_result(entries, video_id, video_title, video_description)
1442 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1444 if 'view_count' in video_info:
1445 view_count = int(video_info['view_count'][0])
1449 # Check for "rental" videos
1450 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1451 raise ExtractorError('"rental" videos not supported')
1453 # Start extracting information
1454 self.report_information_extraction(video_id)
1457 if 'author' not in video_info:
1458 raise ExtractorError('Unable to extract uploader name')
1459 video_uploader = compat_urllib_parse_unquote_plus(video_info['author'][0])
1462 video_uploader_id = None
1463 video_uploader_url = None
1465 r'<link itemprop="url" href="(?P<uploader_url>https?://www.youtube.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
1467 if mobj is not None:
1468 video_uploader_id = mobj.group('uploader_id')
1469 video_uploader_url = mobj.group('uploader_url')
1471 self._downloader.report_warning('unable to extract uploader nickname')
1474 # We try first to get a high quality image:
1475 m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
1476 video_webpage, re.DOTALL)
1477 if m_thumb is not None:
1478 video_thumbnail = m_thumb.group(1)
1479 elif 'thumbnail_url' not in video_info:
1480 self._downloader.report_warning('unable to extract video thumbnail')
1481 video_thumbnail = None
1482 else: # don't panic if we can't find it
1483 video_thumbnail = compat_urllib_parse_unquote_plus(video_info['thumbnail_url'][0])
1486 upload_date = self._html_search_meta(
1487 'datePublished', video_webpage, 'upload date', default=None)
1489 upload_date = self._search_regex(
1490 [r'(?s)id="eow-date.*?>(.*?)</span>',
1491 r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.+?)</strong>'],
1492 video_webpage, 'upload date', default=None)
1494 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1495 upload_date = unified_strdate(upload_date)
1497 video_license = self._html_search_regex(
1498 r'<h4[^>]+class="title"[^>]*>\s*License\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li',
1499 video_webpage, 'license', default=None)
1501 m_music = re.search(
1502 r'<h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*<ul[^>]*>\s*<li>(?P<title>.+?) by (?P<creator>.+?)(?:\(.+?\))?</li',
1505 video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))
1506 video_creator = clean_html(m_music.group('creator'))
1508 video_alt_title = video_creator = None
1510 m_episode = re.search(
1511 r'<div[^>]+id="watch7-headline"[^>]*>\s*<span[^>]*>.*?>(?P<series>[^<]+)</a></b>\s*S(?P<season>\d+)\s*•\s*E(?P<episode>\d+)</span>',
1514 series = m_episode.group('series')
1515 season_number = int(m_episode.group('season'))
1516 episode_number = int(m_episode.group('episode'))
1518 series = season_number = episode_number = None
1520 m_cat_container = self._search_regex(
1521 r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
1522 video_webpage, 'categories', default=None)
1524 category = self._html_search_regex(
1525 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
1527 video_categories = None if category is None else [category]
1529 video_categories = None
1532 unescapeHTML(m.group('content'))
1533 for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
1535 def _extract_count(count_name):
1536 return str_to_int(self._search_regex(
1537 r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
1538 % re.escape(count_name),
1539 video_webpage, count_name, default=None))
1541 like_count = _extract_count('like')
1542 dislike_count = _extract_count('dislike')
1545 video_subtitles = self.extract_subtitles(video_id, video_webpage)
1546 automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
1548 video_duration = try_get(
1549 video_info, lambda x: int_or_none(x['length_seconds'][0]))
1550 if not video_duration:
1551 video_duration = parse_duration(self._html_search_meta(
1552 'duration', video_webpage, 'video duration'))
1555 video_annotations = None
1556 if self._downloader.params.get('writeannotations', False):
1557 video_annotations = self._extract_annotations(video_id)
1559 def _map_to_format_list(urlmap):
1561 for itag, video_real_url in urlmap.items():
1564 'url': video_real_url,
1565 'player_url': player_url,
1567 if itag in self._formats:
1568 dct.update(self._formats[itag])
1572 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1573 self.report_rtmp_download()
1575 'format_id': '_rtmp',
1577 'url': video_info['conn'][0],
1578 'player_url': player_url,
1580 elif len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1:
1581 encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
1582 if 'rtmpe%3Dyes' in encoded_url_map:
1583 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
1585 fmt_list = video_info.get('fmt_list', [''])[0]
1587 for fmt in fmt_list.split(','):
1588 spec = fmt.split('/')
1590 width_height = spec[1].split('x')
1591 if len(width_height) == 2:
1592 formats_spec[spec[0]] = {
1593 'resolution': spec[1],
1594 'width': int_or_none(width_height[0]),
1595 'height': int_or_none(width_height[1]),
1598 for url_data_str in encoded_url_map.split(','):
1599 url_data = compat_parse_qs(url_data_str)
1600 if 'itag' not in url_data or 'url' not in url_data:
1602 format_id = url_data['itag'][0]
1603 url = url_data['url'][0]
1605 if 'sig' in url_data:
1606 url += '&signature=' + url_data['sig'][0]
1607 elif 's' in url_data:
1608 encrypted_sig = url_data['s'][0]
1609 ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
1611 jsplayer_url_json = self._search_regex(
1613 embed_webpage if age_gate else video_webpage,
1614 'JS player URL (1)', default=None)
1615 if not jsplayer_url_json and not age_gate:
1616 # We need the embed website after all
1617 if embed_webpage is None:
1618 embed_url = proto + '://www.youtube.com/embed/%s' % video_id
1619 embed_webpage = self._download_webpage(
1620 embed_url, video_id, 'Downloading embed webpage')
1621 jsplayer_url_json = self._search_regex(
1622 ASSETS_RE, embed_webpage, 'JS player URL')
1624 player_url = json.loads(jsplayer_url_json)
1625 if player_url is None:
1626 player_url_json = self._search_regex(
1627 r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
1628 video_webpage, 'age gate player URL')
1629 player_url = json.loads(player_url_json)
1631 if self._downloader.params.get('verbose'):
1632 if player_url is None:
1633 player_version = 'unknown'
1634 player_desc = 'unknown'
1636 if player_url.endswith('swf'):
1637 player_version = self._search_regex(
1638 r'-(.+?)(?:/watch_as3)?\.swf$', player_url,
1639 'flash player', fatal=False)
1640 player_desc = 'flash player %s' % player_version
1642 player_version = self._search_regex(
1643 [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js', r'(?:www|player)-([^/]+)/base\.js'],
1645 'html5 player', fatal=False)
1646 player_desc = 'html5 player %s' % player_version
1648 parts_sizes = self._signature_cache_id(encrypted_sig)
1649 self.to_screen('{%s} signature length %s, %s' %
1650 (format_id, parts_sizes, player_desc))
1652 signature = self._decrypt_signature(
1653 encrypted_sig, video_id, player_url, age_gate)
1654 url += '&signature=' + signature
1655 if 'ratebypass' not in url:
1656 url += '&ratebypass=yes'
1659 'format_id': format_id,
1661 'player_url': player_url,
1663 if format_id in self._formats:
1664 dct.update(self._formats[format_id])
1665 if format_id in formats_spec:
1666 dct.update(formats_spec[format_id])
1668 # Some itags are not included in DASH manifest thus corresponding formats will
1669 # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).
1670 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
1671 mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
1672 width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
1675 'filesize': int_or_none(url_data.get('clen', [None])[0]),
1676 'tbr': float_or_none(url_data.get('bitrate', [None])[0], 1000),
1679 'fps': int_or_none(url_data.get('fps', [None])[0]),
1680 'format_note': url_data.get('quality_label', [None])[0] or url_data.get('quality', [None])[0],
1682 for key, value in more_fields.items():
1685 type_ = url_data.get('type', [None])[0]
1687 type_split = type_.split(';')
1688 kind_ext = type_split[0].split('/')
1689 if len(kind_ext) == 2:
1691 dct['ext'] = mimetype2ext(type_split[0])
1692 if kind in ('audio', 'video'):
1694 for mobj in re.finditer(
1695 r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
1696 if mobj.group('key') == 'codecs':
1697 codecs = mobj.group('val')
1700 dct.update(parse_codecs(codecs))
1702 elif video_info.get('hlsvp'):
1703 manifest_url = video_info['hlsvp'][0]
1704 url_map = self._extract_from_m3u8(manifest_url, video_id)
1705 formats = _map_to_format_list(url_map)
1706 # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
1707 for a_format in formats:
1708 a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
1710 unavailable_message = self._html_search_regex(
1711 r'(?s)<h1[^>]+id="unavailable-message"[^>]*>(.+?)</h1>',
1712 video_webpage, 'unavailable message', default=None)
1713 if unavailable_message:
1714 raise ExtractorError(unavailable_message, expected=True)
1715 raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
1717 # Look for the DASH manifest
1718 if self._downloader.params.get('youtube_include_dash_manifest', True):
1719 dash_mpd_fatal = True
1720 for mpd_url in dash_mpds:
1723 def decrypt_sig(mobj):
1725 dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
1726 return '/signature/%s' % dec_s
1728 mpd_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, mpd_url)
1730 for df in self._extract_mpd_formats(
1731 mpd_url, video_id, fatal=dash_mpd_fatal,
1732 formats_dict=self._formats):
1733 # Do not overwrite DASH format found in some previous DASH manifest
1734 if df['format_id'] not in dash_formats:
1735 dash_formats[df['format_id']] = df
1736 # Additional DASH manifests may end up in HTTP Error 403 therefore
1737 # allow them to fail without bug report message if we already have
1738 # some DASH manifest succeeded. This is temporary workaround to reduce
1739 # burst of bug reports until we figure out the reason and whether it
1740 # can be fixed at all.
1741 dash_mpd_fatal = False
1742 except (ExtractorError, KeyError) as e:
1743 self.report_warning(
1744 'Skipping DASH manifest: %r' % e, video_id)
1746 # Remove the formats we found through non-DASH, they
1747 # contain less info and it can be wrong, because we use
1748 # fixed values (for example the resolution). See
1749 # https://github.com/rg3/youtube-dl/issues/5774 for an
1751 formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
1752 formats.extend(dash_formats.values())
1754 # Check for malformed aspect ratio
1755 stretched_m = re.search(
1756 r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
1759 w = float(stretched_m.group('w'))
1760 h = float(stretched_m.group('h'))
1761 # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
1762 # We will only process correct ratios.
1766 if f.get('vcodec') != 'none':
1767 f['stretched_ratio'] = ratio
1769 self._sort_formats(formats)
1771 self.mark_watched(video_id, video_info)
1775 'uploader': video_uploader,
1776 'uploader_id': video_uploader_id,
1777 'uploader_url': video_uploader_url,
1778 'upload_date': upload_date,
1779 'license': video_license,
1780 'creator': video_creator,
1781 'title': video_title,
1782 'alt_title': video_alt_title,
1783 'thumbnail': video_thumbnail,
1784 'description': video_description,
1785 'categories': video_categories,
1787 'subtitles': video_subtitles,
1788 'automatic_captions': automatic_captions,
1789 'duration': video_duration,
1790 'age_limit': 18 if age_gate else 0,
1791 'annotations': video_annotations,
1792 'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
1793 'view_count': view_count,
1794 'like_count': like_count,
1795 'dislike_count': dislike_count,
1796 'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
1799 'start_time': start_time,
1800 'end_time': end_time,
1802 'season_number': season_number,
1803 'episode_number': episode_number,
1807 class YoutubeSharedVideoIE(InfoExtractor):
1808 _VALID_URL = r'(?:https?:)?//(?:www\.)?youtube\.com/shared\?.*\bci=(?P<id>[0-9A-Za-z_-]{11})'
1809 IE_NAME = 'youtube:shared'
1812 'url': 'https://www.youtube.com/shared?ci=1nEzmT-M4fU',
1814 'id': 'uPDB5I9wfp8',
1816 'title': 'Pocoyo: 90 minutos de episódios completos Português para crianças - PARTE 3',
1817 'description': 'md5:d9e4d9346a2dfff4c7dc4c8cec0f546d',
1818 'upload_date': '20160219',
1819 'uploader': 'Pocoyo - Português (BR)',
1820 'uploader_id': 'PocoyoBrazil',
1822 'add_ie': ['Youtube'],
1824 # There are already too many Youtube downloads
1825 'skip_download': True,
1829 def _real_extract(self, url):
1830 video_id = self._match_id(url)
1832 webpage = self._download_webpage(url, video_id)
1834 real_video_id = self._html_search_meta(
1835 'videoId', webpage, 'YouTube video id', fatal=True)
1837 return self.url_result(real_video_id, YoutubeIE.ie_key())
1840 class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
1841 IE_DESC = 'YouTube.com playlists'
1842 _VALID_URL = r"""(?x)(?:
1848 (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
1849 \? (?:.*?[&;])*? (?:p|a|list)=
1852 youtu\.be/[0-9A-Za-z_-]{11}\?.*?\blist=
1855 (?:PL|LL|EC|UU|FL|RD|UL|TL)?[0-9A-Za-z-_]{10,}
1856 # Top tracks, they can also include dots
1861 ((?:PL|LL|EC|UU|FL|RD|UL|TL)[0-9A-Za-z-_]{10,})
1863 _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&disable_polymer=true'
1864 _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&[^"]*?index=(?P<index>\d+)(?:[^>]+>(?P<title>[^<]+))?'
1865 IE_NAME = 'youtube:playlist'
1867 'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
1869 'title': 'ytdl test PL',
1870 'id': 'PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
1872 'playlist_count': 3,
1874 'url': 'https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
1876 'id': 'PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
1877 'title': 'YDL_Empty_List',
1879 'playlist_count': 0,
1880 'skip': 'This playlist is private',
1882 'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
1883 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
1885 'title': '29C3: Not my department',
1886 'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
1888 'playlist_count': 95,
1890 'note': 'issue #673',
1891 'url': 'PLBB231211A4F62143',
1893 'title': '[OLD]Team Fortress 2 (Class-based LP)',
1894 'id': 'PLBB231211A4F62143',
1896 'playlist_mincount': 26,
1898 'note': 'Large playlist',
1899 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
1901 'title': 'Uploads from Cauchemar',
1902 'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
1904 'playlist_mincount': 799,
1906 'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
1908 'title': 'YDL_safe_search',
1909 'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
1911 'playlist_count': 2,
1912 'skip': 'This playlist is private',
1915 'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
1916 'playlist_count': 4,
1919 'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
1922 'note': 'Embedded SWF player',
1923 'url': 'https://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
1924 'playlist_count': 4,
1927 'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
1930 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
1931 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
1933 'title': 'Uploads from Interstellar Movie',
1934 'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
1936 'playlist_mincount': 21,
1938 # Playlist URL that does not actually serve a playlist
1939 'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
1941 'id': 'FqZTN594JQw',
1943 'title': "Smiley's People 01 detective, Adventure Series, Action",
1944 'uploader': 'STREEM',
1945 'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
1946 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
1947 'upload_date': '20150526',
1948 'license': 'Standard YouTube License',
1949 'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
1950 'categories': ['People & Blogs'],
1953 'dislike_count': int,
1956 'skip_download': True,
1958 'add_ie': [YoutubeIE.ie_key()],
1960 'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
1962 'id': 'yeWKywCrFtk',
1964 'title': 'Small Scale Baler and Braiding Rugs',
1965 'uploader': 'Backus-Page House Museum',
1966 'uploader_id': 'backuspagemuseum',
1967 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
1968 'upload_date': '20161008',
1969 'license': 'Standard YouTube License',
1970 'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
1971 'categories': ['Nonprofits & Activism'],
1974 'dislike_count': int,
1978 'skip_download': True,
1981 'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
1982 'only_matching': True,
1984 'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
1985 'only_matching': True,
1988 def _real_initialize(self):
1991 def _extract_mix(self, playlist_id):
1992 # The mixes are generated from a single video
1993 # the id of the playlist is just 'RD' + video_id
1995 last_id = playlist_id[-11:]
1996 for n in itertools.count(1):
1997 url = 'https://youtube.com/watch?v=%s&list=%s' % (last_id, playlist_id)
1998 webpage = self._download_webpage(
1999 url, playlist_id, 'Downloading page {0} of Youtube mix'.format(n))
2000 new_ids = orderedSet(re.findall(
2001 r'''(?xs)data-video-username=".*?".*?
2002 href="/watch\?v=([0-9A-Za-z_-]{11})&[^"]*?list=%s''' % re.escape(playlist_id),
2004 # Fetch new pages until all the videos are repeated, it seems that
2005 # there are always 51 unique videos.
2006 new_ids = [_id for _id in new_ids if _id not in ids]
2012 url_results = self._ids_to_results(ids)
2014 search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
2016 search_title('playlist-title') or
2017 search_title('title long-title') or
2018 search_title('title'))
2019 title = clean_html(title_span)
2021 return self.playlist_result(url_results, playlist_id, title)
2023 def _extract_playlist(self, playlist_id):
2024 url = self._TEMPLATE_URL % playlist_id
2025 page = self._download_webpage(url, playlist_id)
2027 # the yt-alert-message now has tabindex attribute (see https://github.com/rg3/youtube-dl/issues/11604)
2028 for match in re.findall(r'<div class="yt-alert-message"[^>]*>([^<]+)</div>', page):
2029 match = match.strip()
2030 # Check if the playlist exists or is private
2031 mobj = re.match(r'[^<]*(?:The|This) playlist (?P<reason>does not exist|is private)[^<]*', match)
2033 reason = mobj.group('reason')
2034 message = 'This playlist %s' % reason
2035 if 'private' in reason:
2036 message += ', use --username or --netrc to access it'
2038 raise ExtractorError(message, expected=True)
2039 elif re.match(r'[^<]*Invalid parameters[^<]*', match):
2040 raise ExtractorError(
2041 'Invalid parameters. Maybe URL is incorrect.',
2043 elif re.match(r'[^<]*Choose your language[^<]*', match):
2046 self.report_warning('Youtube gives an alert message: ' + match)
2048 playlist_title = self._html_search_regex(
2049 r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
2050 page, 'title', default=None)
2054 if not playlist_title:
2056 # Some playlist URLs don't actually serve a playlist (e.g.
2057 # https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4)
2058 next(self._entries(page, playlist_id))
2059 except StopIteration:
2062 return has_videos, self.playlist_result(
2063 self._entries(page, playlist_id), playlist_id, playlist_title)
2065 def _check_download_just_video(self, url, playlist_id):
2066 # Check if it's a video-specific URL
2067 query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
2068 video_id = query_dict.get('v', [None])[0] or self._search_regex(
2069 r'(?:^|//)youtu\.be/([0-9A-Za-z_-]{11})', url,
2070 'video id', default=None)
2072 if self._downloader.params.get('noplaylist'):
2073 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
2074 return video_id, self.url_result(video_id, 'Youtube', video_id=video_id)
2076 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
2077 return video_id, None
2080 def _real_extract(self, url):
2081 # Extract playlist id
2082 mobj = re.match(self._VALID_URL, url)
2084 raise ExtractorError('Invalid URL: %s' % url)
2085 playlist_id = mobj.group(1) or mobj.group(2)
2087 video_id, video = self._check_download_just_video(url, playlist_id)
2091 if playlist_id.startswith(('RD', 'UL', 'PU')):
2092 # Mixes require a custom extraction process
2093 return self._extract_mix(playlist_id)
2095 has_videos, playlist = self._extract_playlist(playlist_id)
2096 if has_videos or not video_id:
2099 # Some playlist URLs don't actually serve a playlist (see
2100 # https://github.com/rg3/youtube-dl/issues/10537).
2101 # Fallback to plain video extraction if there is a video id
2102 # along with playlist id.
2103 return self.url_result(video_id, 'Youtube', video_id=video_id)
2106 class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
2107 IE_DESC = 'YouTube.com channels'
2108 _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
2109 _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
2110 _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
2111 IE_NAME = 'youtube:channel'
2113 'note': 'paginated channel',
2114 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
2115 'playlist_mincount': 91,
2117 'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
2118 'title': 'Uploads from lex will',
2121 'note': 'Age restricted channel',
2122 # from https://www.youtube.com/user/DeusExOfficial
2123 'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
2124 'playlist_mincount': 64,
2126 'id': 'UUs0ifCMCm1icqRbqhUINa0w',
2127 'title': 'Uploads from Deus Ex',
2132 def suitable(cls, url):
2133 return (False if YoutubePlaylistsIE.suitable(url) or YoutubeLiveIE.suitable(url)
2134 else super(YoutubeChannelIE, cls).suitable(url))
2136 def _build_template_url(self, url, channel_id):
2137 return self._TEMPLATE_URL % channel_id
2139 def _real_extract(self, url):
2140 channel_id = self._match_id(url)
2142 url = self._build_template_url(url, channel_id)
2144 # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
2145 # Workaround by extracting as a playlist if managed to obtain channel playlist URL
2146 # otherwise fallback on channel by page extraction
2147 channel_page = self._download_webpage(
2148 url + '?view=57', channel_id,
2149 'Downloading channel page', fatal=False)
2150 if channel_page is False:
2151 channel_playlist_id = False
2153 channel_playlist_id = self._html_search_meta(
2154 'channelId', channel_page, 'channel id', default=None)
2155 if not channel_playlist_id:
2156 channel_url = self._html_search_meta(
2157 ('al:ios:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad'),
2158 channel_page, 'channel url', default=None)
2160 channel_playlist_id = self._search_regex(
2161 r'vnd\.youtube://user/([0-9A-Za-z_-]+)',
2162 channel_url, 'channel id', default=None)
2163 if channel_playlist_id and channel_playlist_id.startswith('UC'):
2164 playlist_id = 'UU' + channel_playlist_id[2:]
2165 return self.url_result(
2166 compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')
2168 channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
2169 autogenerated = re.search(r'''(?x)
2171 channel-header-autogenerated-label|
2172 yt-channel-title-autogenerated
2173 )[^"]*"''', channel_page) is not None
2176 # The videos are contained in a single page
2177 # the ajax pages can't be used, they are empty
2180 video_id, 'Youtube', video_id=video_id,
2181 video_title=video_title)
2182 for video_id, video_title in self.extract_videos_from_page(channel_page)]
2183 return self.playlist_result(entries, channel_id)
2186 next(self._entries(channel_page, channel_id))
2187 except StopIteration:
2188 alert_message = self._html_search_regex(
2189 r'(?s)<div[^>]+class=(["\']).*?\byt-alert-message\b.*?\1[^>]*>(?P<alert>[^<]+)</div>',
2190 channel_page, 'alert', default=None, group='alert')
2192 raise ExtractorError('Youtube said: %s' % alert_message, expected=True)
2194 return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
2197 class YoutubeUserIE(YoutubeChannelIE):
2198 IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
2199 _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:(?P<user>user|c)/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
2200 _TEMPLATE_URL = 'https://www.youtube.com/%s/%s/videos'
2201 IE_NAME = 'youtube:user'
2204 'url': 'https://www.youtube.com/user/TheLinuxFoundation',
2205 'playlist_mincount': 320,
2207 'id': 'UUfX55Sx5hEFjoC3cNs6mCUQ',
2208 'title': 'Uploads from The Linux Foundation',
2211 # Only available via https://www.youtube.com/c/12minuteathlete/videos
2212 # but not https://www.youtube.com/user/12minuteathlete/videos
2213 'url': 'https://www.youtube.com/c/12minuteathlete/videos',
2214 'playlist_mincount': 249,
2216 'id': 'UUVjM-zV6_opMDx7WYxnjZiQ',
2217 'title': 'Uploads from 12 Minute Athlete',
2220 'url': 'ytuser:phihag',
2221 'only_matching': True,
2223 'url': 'https://www.youtube.com/c/gametrailers',
2224 'only_matching': True,
2226 'url': 'https://www.youtube.com/gametrailers',
2227 'only_matching': True,
2229 # This channel is not available.
2230 'url': 'https://www.youtube.com/user/kananishinoSMEJ/videos',
2231 'only_matching': True,
2235 def suitable(cls, url):
2236 # Don't return True if the url can be extracted with other youtube
2237 # extractor, the regex would is too permissive and it would match.
2238 other_yt_ies = iter(klass for (name, klass) in globals().items() if name.startswith('Youtube') and name.endswith('IE') and klass is not cls)
2239 if any(ie.suitable(url) for ie in other_yt_ies):
2242 return super(YoutubeUserIE, cls).suitable(url)
2244 def _build_template_url(self, url, channel_id):
2245 mobj = re.match(self._VALID_URL, url)
2246 return self._TEMPLATE_URL % (mobj.group('user') or 'user', mobj.group('id'))
2249 class YoutubeLiveIE(YoutubeBaseInfoExtractor):
2250 IE_DESC = 'YouTube.com live streams'
2251 _VALID_URL = r'(?P<base_url>https?://(?:\w+\.)?youtube\.com/(?:(?:user|channel|c)/)?(?P<id>[^/]+))/live'
2252 IE_NAME = 'youtube:live'
2255 'url': 'https://www.youtube.com/user/TheYoungTurks/live',
2257 'id': 'a48o2S1cPoo',
2259 'title': 'The Young Turks - Live Main Show',
2260 'uploader': 'The Young Turks',
2261 'uploader_id': 'TheYoungTurks',
2262 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
2263 'upload_date': '20150715',
2264 'license': 'Standard YouTube License',
2265 'description': 'md5:438179573adcdff3c97ebb1ee632b891',
2266 'categories': ['News & Politics'],
2267 'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
2269 'dislike_count': int,
2272 'skip_download': True,
2275 'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
2276 'only_matching': True,
2278 'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
2279 'only_matching': True,
2281 'url': 'https://www.youtube.com/TheYoungTurks/live',
2282 'only_matching': True,
2285 def _real_extract(self, url):
2286 mobj = re.match(self._VALID_URL, url)
2287 channel_id = mobj.group('id')
2288 base_url = mobj.group('base_url')
2289 webpage = self._download_webpage(url, channel_id, fatal=False)
2291 page_type = self._og_search_property(
2292 'type', webpage, 'page type', default=None)
2293 video_id = self._html_search_meta(
2294 'videoId', webpage, 'video id', default=None)
2295 if page_type == 'video' and video_id and re.match(r'^[0-9A-Za-z_-]{11}$', video_id):
2296 return self.url_result(video_id, YoutubeIE.ie_key())
2297 return self.url_result(base_url)
2300 class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
2301 IE_DESC = 'YouTube.com user/channel playlists'
2302 _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+)/playlists'
2303 IE_NAME = 'youtube:playlists'
2306 'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
2307 'playlist_mincount': 4,
2309 'id': 'ThirstForScience',
2310 'title': 'Thirst for Science',
2313 # with "Load more" button
2314 'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
2315 'playlist_mincount': 70,
2318 'title': 'Игорь Клейнер',
2321 'url': 'https://www.youtube.com/channel/UCiU1dHvZObB2iP6xkJ__Icw/playlists',
2322 'playlist_mincount': 17,
2324 'id': 'UCiU1dHvZObB2iP6xkJ__Icw',
2325 'title': 'Chem Player',
2330 class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
2331 IE_DESC = 'YouTube.com searches'
2332 # there doesn't appear to be a real limit, for example if you search for
2333 # 'python' you get more than 8.000.000 results
2334 _MAX_RESULTS = float('inf')
2335 IE_NAME = 'youtube:search'
2336 _SEARCH_KEY = 'ytsearch'
2337 _EXTRA_QUERY_ARGS = {}
2340 def _get_n_results(self, query, n):
2341 """Get a specified number of results for a query"""
2347 'search_query': query.encode('utf-8'),
2349 url_query.update(self._EXTRA_QUERY_ARGS)
2350 result_url = 'https://www.youtube.com/results?' + compat_urllib_parse_urlencode(url_query)
2352 for pagenum in itertools.count(1):
2353 data = self._download_json(
2354 result_url, video_id='query "%s"' % query,
2355 note='Downloading page %s' % pagenum,
2356 errnote='Unable to download API page',
2357 query={'spf': 'navigate'})
2358 html_content = data[1]['body']['content']
2360 if 'class="search-message' in html_content:
2361 raise ExtractorError(
2362 '[youtube] No video results', expected=True)
2364 new_videos = self._ids_to_results(orderedSet(re.findall(
2365 r'href="/watch\?v=(.{11})', html_content)))
2366 videos += new_videos
2367 if not new_videos or len(videos) > limit:
2369 next_link = self._html_search_regex(
2370 r'href="(/results\?[^"]*\bsp=[^"]+)"[^>]*>\s*<span[^>]+class="[^"]*\byt-uix-button-content\b[^"]*"[^>]*>Next',
2371 html_content, 'next link', default=None)
2372 if next_link is None:
2374 result_url = compat_urlparse.urljoin('https://www.youtube.com/', next_link)
2378 return self.playlist_result(videos, query)
2381 class YoutubeSearchDateIE(YoutubeSearchIE):
2382 IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
2383 _SEARCH_KEY = 'ytsearchdate'
2384 IE_DESC = 'YouTube.com searches, newest videos first'
2385 _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}
2388 class YoutubeSearchURLIE(YoutubePlaylistBaseInfoExtractor):
2389 IE_DESC = 'YouTube.com search URLs'
2390 IE_NAME = 'youtube:search_url'
2391 _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
2392 _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})(?:[^"]*"[^>]+\btitle="(?P<title>[^"]+))?'
2394 'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
2395 'playlist_mincount': 5,
2397 'title': 'youtube-dl test video',
2400 'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
2401 'only_matching': True,
2404 def _real_extract(self, url):
2405 mobj = re.match(self._VALID_URL, url)
2406 query = compat_urllib_parse_unquote_plus(mobj.group('query'))
2407 webpage = self._download_webpage(url, query)
2408 return self.playlist_result(self._process_page(webpage), playlist_title=query)
2411 class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
2412 IE_DESC = 'YouTube.com (multi-season) shows'
2413 _VALID_URL = r'https?://(?:www\.)?youtube\.com/show/(?P<id>[^?#]*)'
2414 IE_NAME = 'youtube:show'
2416 'url': 'https://www.youtube.com/show/airdisasters',
2417 'playlist_mincount': 5,
2419 'id': 'airdisasters',
2420 'title': 'Air Disasters',
2424 def _real_extract(self, url):
2425 playlist_id = self._match_id(url)
2426 return super(YoutubeShowIE, self)._real_extract(
2427 'https://www.youtube.com/show/%s/playlists' % playlist_id)
2430 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
2432 Base class for feed extractors
2433 Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
2435 _LOGIN_REQUIRED = True
2439 return 'youtube:%s' % self._FEED_NAME
2441 def _real_initialize(self):
2444 def _real_extract(self, url):
2445 page = self._download_webpage(
2446 'https://www.youtube.com/feed/%s' % self._FEED_NAME, self._PLAYLIST_TITLE)
2448 # The extraction process is the same as for playlists, but the regex
2449 # for the video ids doesn't contain an index
2451 more_widget_html = content_html = page
2452 for page_num in itertools.count(1):
2453 matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
2455 # 'recommended' feed has infinite 'load more' and each new portion spins
2456 # the same videos in (sometimes) slightly different order, so we'll check
2457 # for unicity and break when portion has no new videos
2458 new_ids = filter(lambda video_id: video_id not in ids, orderedSet(matches))
2464 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
2468 more = self._download_json(
2469 'https://youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
2470 'Downloading page #%s' % page_num,
2471 transform_source=uppercase_escape)
2472 content_html = more['content_html']
2473 more_widget_html = more['load_more_widget_html']
2475 return self.playlist_result(
2476 self._ids_to_results(ids), playlist_title=self._PLAYLIST_TITLE)
2479 class YoutubeWatchLaterIE(YoutubePlaylistIE):
2480 IE_NAME = 'youtube:watchlater'
2481 IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
2482 _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'
2485 'url': 'https://www.youtube.com/playlist?list=WL',
2486 'only_matching': True,
2488 'url': 'https://www.youtube.com/watch?v=bCNU9TrbiRk&index=1&list=WL',
2489 'only_matching': True,
2492 def _real_extract(self, url):
2493 _, video = self._check_download_just_video(url, 'WL')
2496 _, playlist = self._extract_playlist('WL')
2500 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
2501 IE_NAME = 'youtube:favorites'
2502 IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
2503 _VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
2504 _LOGIN_REQUIRED = True
2506 def _real_extract(self, url):
2507 webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
2508 playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, 'favourites playlist id')
2509 return self.url_result(playlist_id, 'YoutubePlaylist')
2512 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
2513 IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
2514 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/recommended|:ytrec(?:ommended)?'
2515 _FEED_NAME = 'recommended'
2516 _PLAYLIST_TITLE = 'Youtube Recommended videos'
2519 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
2520 IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
2521 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
2522 _FEED_NAME = 'subscriptions'
2523 _PLAYLIST_TITLE = 'Youtube Subscriptions'
2526 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
2527 IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
2528 _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/history|:ythistory'
2529 _FEED_NAME = 'history'
2530 _PLAYLIST_TITLE = 'Youtube History'
2533 class YoutubeTruncatedURLIE(InfoExtractor):
2534 IE_NAME = 'youtube:truncated_url'
2535 IE_DESC = False # Do not list
2536 _VALID_URL = r'''(?x)
2538 (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
2541 annotation_id=annotation_[^&]+|
2547 attribution_link\?a=[^&]+
2553 'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
2554 'only_matching': True,
2556 'url': 'https://www.youtube.com/watch?',
2557 'only_matching': True,
2559 'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
2560 'only_matching': True,
2562 'url': 'https://www.youtube.com/watch?feature=foo',
2563 'only_matching': True,
2565 'url': 'https://www.youtube.com/watch?hl=en-GB',
2566 'only_matching': True,
2568 'url': 'https://www.youtube.com/watch?t=2372',
2569 'only_matching': True,
2572 def _real_extract(self, url):
2573 raise ExtractorError(
2574 'Did you forget to quote the URL? Remember that & is a meta '
2575 'character in most shells, so you want to put the URL in quotes, '
2577 '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
2578 ' or simply youtube-dl BaW_jenozKc .',
2582 class YoutubeTruncatedIDIE(InfoExtractor):
2583 IE_NAME = 'youtube:truncated_id'
2584 IE_DESC = False # Do not list
2585 _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
2588 'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
2589 'only_matching': True,
2592 def _real_extract(self, url):
2593 video_id = self._match_id(url)
2594 raise ExtractorError(
2595 'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),