3 from __future__ import unicode_literals
13 from .common import InfoExtractor, SearchInfoExtractor
14 from ..jsinterp import JSInterpreter
15 from ..swfinterp import SWFInterpreter
16 from ..compat import (
20 compat_urllib_parse_unquote,
21 compat_urllib_parse_unquote_plus,
22 compat_urllib_parse_urlparse,
23 compat_urllib_request,
32 get_element_by_attribute,
48 class YoutubeBaseInfoExtractor(InfoExtractor):
49 """Provide base functions for Youtube extractors"""
50 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
51 _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
52 _NETRC_MACHINE = 'youtube'
53 # If True it will raise an error if no login info is provided
54 _LOGIN_REQUIRED = False
56 def _set_language(self):
58 '.youtube.com', 'PREF', 'f1=50000000&hl=en',
59 # YouTube sets the expire time to about two months
60 expire_time=time.time() + 2 * 30 * 24 * 3600)
62 def _ids_to_results(self, ids):
64 self.url_result(vid_id, 'Youtube', video_id=vid_id)
69 Attempt to log in to YouTube.
70 True is returned if successful or skipped.
71 False is returned if login failed.
73 If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
75 (username, password) = self._get_login_info()
76 # No authentication to be performed
78 if self._LOGIN_REQUIRED:
79 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
82 login_page = self._download_webpage(
83 self._LOGIN_URL, None,
84 note='Downloading login page',
85 errnote='unable to fetch login page', fatal=False)
86 if login_page is False:
89 galx = self._search_regex(r'(?s)<input.+?name="GALX".+?value="(.+?)"',
90 login_page, 'Login GALX parameter')
94 'continue': 'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
99 'PersistentCookie': 'yes',
101 'bgresponse': 'js_disabled',
102 'checkConnection': '',
103 'checkedDomains': 'youtube',
110 'service': 'youtube',
115 login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('ascii')
117 req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
118 login_results = self._download_webpage(
120 note='Logging in', errnote='unable to log in', fatal=False)
121 if login_results is False:
124 if re.search(r'id="errormsg_0_Passwd"', login_results) is not None:
125 raise ExtractorError('Please use your account password and a two-factor code instead of an application-specific password.', expected=True)
128 # TODO add SMS and phone call support - these require making a request and then prompting the user
130 if re.search(r'(?i)<form[^>]* id="challenge"', login_results) is not None:
131 tfa_code = self._get_tfa_info('2-step verification code')
134 self._downloader.report_warning(
135 'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
136 '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
139 tfa_code = remove_start(tfa_code, 'G-')
141 tfa_form_strs = self._form_hidden_inputs('challenge', login_results)
143 tfa_form_strs.update({
148 tfa_data = compat_urllib_parse.urlencode(encode_dict(tfa_form_strs)).encode('ascii')
150 tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data)
151 tfa_results = self._download_webpage(
153 note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)
155 if tfa_results is False:
158 if re.search(r'(?i)<form[^>]* id="challenge"', tfa_results) is not None:
159 self._downloader.report_warning('Two-factor code expired or invalid. Please try again, or use a one-use backup code instead.')
161 if re.search(r'(?i)<form[^>]* id="gaia_loginform"', tfa_results) is not None:
162 self._downloader.report_warning('unable to log in - did the page structure change?')
164 if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None:
165 self._downloader.report_warning('Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
168 if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
169 self._downloader.report_warning('unable to log in: bad username or password')
173 def _real_initialize(self):
174 if self._downloader is None:
177 if not self._login():
181 class YoutubeEntryListBaseInfoExtractor(InfoExtractor):
182 # Extract entries from page with "Load more" button
183 def _entries(self, page, playlist_id):
184 more_widget_html = content_html = page
185 for page_num in itertools.count(1):
186 for entry in self._process_page(content_html):
189 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
193 more = self._download_json(
194 'https://youtube.com/%s' % mobj.group('more'), playlist_id,
195 'Downloading page #%s' % page_num,
196 transform_source=uppercase_escape)
197 content_html = more['content_html']
198 if not content_html.strip():
199 # Some webpages show a "Load more" button but they don't
202 more_widget_html = more['load_more_widget_html']
205 class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
206 def _process_page(self, content):
207 for video_id, video_title in self.extract_videos_from_page(content):
208 yield self.url_result(video_id, 'Youtube', video_id, video_title)
210 def extract_videos_from_page(self, page):
213 for mobj in re.finditer(self._VIDEO_RE, page):
214 # The link with index 0 is not the first video of the playlist (not sure if still actual)
215 if 'index' in mobj.groupdict() and mobj.group('id') == '0':
217 video_id = mobj.group('id')
218 video_title = unescapeHTML(mobj.group('title'))
220 video_title = video_title.strip()
222 idx = ids_in_page.index(video_id)
223 if video_title and not titles_in_page[idx]:
224 titles_in_page[idx] = video_title
226 ids_in_page.append(video_id)
227 titles_in_page.append(video_title)
228 return zip(ids_in_page, titles_in_page)
231 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
232 def _process_page(self, content):
233 for playlist_id in re.findall(r'href="/?playlist\?list=(.+?)"', content):
234 yield self.url_result(
235 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
237 def _real_extract(self, url):
238 playlist_id = self._match_id(url)
239 webpage = self._download_webpage(url, playlist_id)
240 title = self._og_search_title(webpage, fatal=False)
241 return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
244 class YoutubeIE(YoutubeBaseInfoExtractor):
245 IE_DESC = 'YouTube.com'
246 _VALID_URL = r"""(?x)^
248 (?:https?://|//) # http(s):// or protocol-independent URL
249 (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
250 (?:www\.)?deturl\.com/www\.youtube\.com/|
251 (?:www\.)?pwnyoutube\.com/|
252 (?:www\.)?yourepeat\.com/|
253 tube\.majestyc\.net/|
254 youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
255 (?:.*?\#/)? # handle anchor (#/) redirect urls
256 (?: # the various things that can precede the ID:
257 (?:(?:v|embed|e)/(?!videoseries)) # v/ or embed/ or e/
258 |(?: # or the v= param in all its forms
259 (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
260 (?:\?|\#!?) # the params delimiter ? or # or #!
261 (?:.*?&)?? # any other preceding param (like /?s=tuff&v=xxxx)
266 youtu\.be| # just youtu.be/xxxx
267 vid\.plus # or vid.plus/xxxx
269 |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
271 )? # all until now is optional -> you can pass the naked ID
272 ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
273 (?!.*?&list=) # combined list/video URLs are handled by the playlist IE
274 (?(1).+)? # if we found the ID, everything can follow
276 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
278 '5': {'ext': 'flv', 'width': 400, 'height': 240},
279 '6': {'ext': 'flv', 'width': 450, 'height': 270},
280 '13': {'ext': '3gp'},
281 '17': {'ext': '3gp', 'width': 176, 'height': 144},
282 '18': {'ext': 'mp4', 'width': 640, 'height': 360},
283 '22': {'ext': 'mp4', 'width': 1280, 'height': 720},
284 '34': {'ext': 'flv', 'width': 640, 'height': 360},
285 '35': {'ext': 'flv', 'width': 854, 'height': 480},
286 '36': {'ext': '3gp', 'width': 320, 'height': 240},
287 '37': {'ext': 'mp4', 'width': 1920, 'height': 1080},
288 '38': {'ext': 'mp4', 'width': 4096, 'height': 3072},
289 '43': {'ext': 'webm', 'width': 640, 'height': 360},
290 '44': {'ext': 'webm', 'width': 854, 'height': 480},
291 '45': {'ext': 'webm', 'width': 1280, 'height': 720},
292 '46': {'ext': 'webm', 'width': 1920, 'height': 1080},
293 '59': {'ext': 'mp4', 'width': 854, 'height': 480},
294 '78': {'ext': 'mp4', 'width': 854, 'height': 480},
298 '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'preference': -20},
299 '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'preference': -20},
300 '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'preference': -20},
301 '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'preference': -20},
302 '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'preference': -20},
303 '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'preference': -20},
304 '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'preference': -20},
306 # Apple HTTP Live Streaming
307 '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'preference': -10},
308 '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'preference': -10},
309 '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'preference': -10},
310 '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'preference': -10},
311 '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'preference': -10},
312 '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'preference': -10},
313 '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'preference': -10},
316 '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
317 '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
318 '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
319 '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
320 '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
321 '138': {'ext': 'mp4', 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40}, # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
322 '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
323 '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
324 '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
325 '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
326 '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
329 '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
330 '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
331 '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
334 '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
335 '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
336 '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
337 '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
338 '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
339 '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
340 '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'vp9'},
341 '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
342 '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
343 '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
344 '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
345 '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
346 '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
347 '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
348 '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
349 '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
350 '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
351 '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
352 '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
353 '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'vp9'},
354 '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
357 '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
358 '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
360 # Dash webm audio with opus inside
361 '249': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
362 '250': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
363 '251': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
366 '_rtmp': {'protocol': 'rtmp'},
372 'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s&end=9',
376 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
377 'uploader': 'Philipp Hagemeister',
378 'uploader_id': 'phihag',
379 'upload_date': '20121002',
380 'description': 'test chars: "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
381 'categories': ['Science & Technology'],
382 'tags': ['youtube-dl'],
384 'dislike_count': int,
390 'url': 'http://www.youtube.com/watch?v=UxxajLWwzqY',
391 'note': 'Test generic use_cipher_signature video (#897)',
395 'upload_date': '20120506',
396 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
397 'description': 'md5:782e8651347686cba06e58f71ab51773',
398 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
399 'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
400 'iconic ep', 'iconic', 'love', 'it'],
401 'uploader': 'Icona Pop',
402 'uploader_id': 'IconaPop',
406 'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
407 'note': 'Test VEVO video with age protection (#956)',
411 'upload_date': '20130703',
412 'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
413 'description': 'md5:64249768eec3bc4276236606ea996373',
414 'uploader': 'justintimberlakeVEVO',
415 'uploader_id': 'justintimberlakeVEVO',
420 'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
421 'note': 'Embed-only video (#1746)',
425 'upload_date': '20120608',
426 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
427 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
428 'uploader': 'SET India',
429 'uploader_id': 'setindia'
433 'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&v=UxxajLWwzqY',
434 'note': 'Use the first video ID in the URL',
438 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
439 'uploader': 'Philipp Hagemeister',
440 'uploader_id': 'phihag',
441 'upload_date': '20121002',
442 'description': 'test chars: "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
443 'categories': ['Science & Technology'],
444 'tags': ['youtube-dl'],
446 'dislike_count': int,
449 'skip_download': True,
453 'url': 'http://www.youtube.com/watch?v=a9LDPn-MO4I',
454 'note': '256k DASH audio (format 141) via DASH manifest',
458 'upload_date': '20121002',
459 'uploader_id': '8KVIDEO',
461 'uploader': '8KVIDEO',
462 'title': 'UHDTV TEST 8K VIDEO.mp4'
465 'youtube_include_dash_manifest': True,
469 # DASH manifest with encrypted signature
471 'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
475 'title': 'Afrojack, Spree Wilson - The Spark ft. Spree Wilson',
476 'description': 'md5:12e7067fa6735a77bdcbb58cb1187d2d',
477 'uploader': 'AfrojackVEVO',
478 'uploader_id': 'AfrojackVEVO',
479 'upload_date': '20131011',
482 'youtube_include_dash_manifest': True,
486 # JS player signature function name containing $
488 'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
492 'title': 'Taylor Swift - Shake It Off',
493 'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
494 'uploader': 'TaylorSwiftVEVO',
495 'uploader_id': 'TaylorSwiftVEVO',
496 'upload_date': '20140818',
499 'youtube_include_dash_manifest': True,
505 'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
509 'upload_date': '20100909',
510 'uploader': 'The Amazing Atheist',
511 'uploader_id': 'TheAmazingAtheist',
512 'title': 'Burning Everyone\'s Koran',
513 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
516 # Normal age-gate video (No vevo, embed allowed)
518 'url': 'http://youtube.com/watch?v=HtVdAasjOgU',
522 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
523 'description': 're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
524 'uploader': 'The Witcher',
525 'uploader_id': 'WitcherGame',
526 'upload_date': '20140605',
530 # Age-gate video with encrypted signature
532 'url': 'http://www.youtube.com/watch?v=6kLq3WMV1nU',
536 'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
537 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
538 'uploader': 'LloydVEVO',
539 'uploader_id': 'LloydVEVO',
540 'upload_date': '20110629',
544 # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
546 'url': '__2ABJjxzNo',
550 'upload_date': '20100430',
551 'uploader_id': 'deadmau5',
552 'description': 'md5:12c56784b8032162bb936a5f76d55360',
553 'uploader': 'deadmau5',
554 'title': 'Deadmau5 - Some Chords (HD)',
556 'expected_warnings': [
557 'DASH manifest missing',
560 # Olympics (https://github.com/rg3/youtube-dl/issues/4431)
562 'url': 'lqQg6PlCWgI',
566 'upload_date': '20120724',
567 'uploader_id': 'olympic',
568 'description': 'HO09 - Women - GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
569 'uploader': 'Olympics',
570 'title': 'Hockey - Women - GER-AUS - London 2012 Olympic Games',
573 'skip_download': 'requires avconv',
578 'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
582 'stretched_ratio': 16 / 9.,
583 'upload_date': '20110310',
584 'uploader_id': 'AllenMeow',
585 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
587 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
590 # url_encoded_fmt_stream_map is empty string
592 'url': 'qEJwOuvDf7I',
596 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
598 'upload_date': '20150404',
599 'uploader_id': 'spbelect',
600 'uploader': 'Наблюдатели Петербурга',
603 'skip_download': 'requires avconv',
606 # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
608 'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
612 'title': 'md5:7b81415841e02ecd4313668cde88737a',
613 'description': 'md5:116377fd2963b81ec4ce64b542173306',
614 'upload_date': '20150625',
615 'uploader_id': 'dorappi2000',
616 'uploader': 'dorappi2000',
617 'formats': 'mincount:33',
620 # DASH manifest with segment_list
622 'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
623 'md5': '8ce563a1d667b599d21064e982ab9e31',
627 'upload_date': '20150501', # According to '<meta itemprop="datePublished"', but in other places it's 20150510
628 'uploader': 'Airtek',
629 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
630 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
631 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
634 'youtube_include_dash_manifest': True,
635 'format': '135', # bestvideo
639 # Multifeed videos (multiple cameras), URL is for Main Camera
640 'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
643 'title': 'teamPGP: Rocket League Noob Stream',
644 'description': 'md5:dc7872fb300e143831327f1bae3af010',
650 'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
651 'description': 'md5:dc7872fb300e143831327f1bae3af010',
652 'upload_date': '20150721',
653 'uploader': 'Beer Games Beer',
654 'uploader_id': 'beergamesbeer',
660 'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
661 'description': 'md5:dc7872fb300e143831327f1bae3af010',
662 'upload_date': '20150721',
663 'uploader': 'Beer Games Beer',
664 'uploader_id': 'beergamesbeer',
670 'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
671 'description': 'md5:dc7872fb300e143831327f1bae3af010',
672 'upload_date': '20150721',
673 'uploader': 'Beer Games Beer',
674 'uploader_id': 'beergamesbeer',
680 'title': 'teamPGP: Rocket League Noob Stream (zim)',
681 'description': 'md5:dc7872fb300e143831327f1bae3af010',
682 'upload_date': '20150721',
683 'uploader': 'Beer Games Beer',
684 'uploader_id': 'beergamesbeer',
688 'skip_download': True,
692 'url': 'http://vid.plus/FlRa-iH7PGw',
693 'only_matching': True,
696 # Title with JS-like syntax "};"
697 'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
701 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
702 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
703 'upload_date': '20151119',
704 'uploader_id': 'IronSoulElf',
705 'uploader': 'IronSoulElf',
708 'skip_download': True,
713 def __init__(self, *args, **kwargs):
714 super(YoutubeIE, self).__init__(*args, **kwargs)
715 self._player_cache = {}
717 def report_video_info_webpage_download(self, video_id):
718 """Report attempt to download video info webpage."""
719 self.to_screen('%s: Downloading video info webpage' % video_id)
721 def report_information_extraction(self, video_id):
722 """Report attempt to extract video information."""
723 self.to_screen('%s: Extracting video information' % video_id)
725 def report_unavailable_format(self, video_id, format):
726 """Report extracted video URL."""
727 self.to_screen('%s: Format %s not available' % (video_id, format))
729 def report_rtmp_download(self):
730 """Indicate the download will use the RTMP protocol."""
731 self.to_screen('RTMP download detected')
733 def _signature_cache_id(self, example_sig):
734 """ Return a string representation of a signature """
735 return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
737 def _extract_signature_function(self, video_id, player_url, example_sig):
739 r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|/base)?\.(?P<ext>[a-z]+)$',
742 raise ExtractorError('Cannot identify player %r' % player_url)
743 player_type = id_m.group('ext')
744 player_id = id_m.group('id')
746 # Read from filesystem cache
747 func_id = '%s_%s_%s' % (
748 player_type, player_id, self._signature_cache_id(example_sig))
749 assert os.path.basename(func_id) == func_id
751 cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
752 if cache_spec is not None:
753 return lambda s: ''.join(s[i] for i in cache_spec)
756 'Downloading player %s' % player_url
757 if self._downloader.params.get('verbose') else
758 'Downloading %s player %s' % (player_type, player_id)
760 if player_type == 'js':
761 code = self._download_webpage(
762 player_url, video_id,
764 errnote='Download of %s failed' % player_url)
765 res = self._parse_sig_js(code)
766 elif player_type == 'swf':
767 urlh = self._request_webpage(
768 player_url, video_id,
770 errnote='Download of %s failed' % player_url)
772 res = self._parse_sig_swf(code)
774 assert False, 'Invalid player type %r' % player_type
776 test_string = ''.join(map(compat_chr, range(len(example_sig))))
777 cache_res = res(test_string)
778 cache_spec = [ord(c) for c in cache_res]
780 self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
783 def _print_sig_code(self, func, example_sig):
784 def gen_sig_code(idxs):
785 def _genslice(start, end, step):
786 starts = '' if start == 0 else str(start)
787 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
788 steps = '' if step == 1 else (':%d' % step)
789 return 's[%s%s%s]' % (starts, ends, steps)
792 # Quelch pyflakes warnings - start will be set when step is set
793 start = '(Never used)'
794 for i, prev in zip(idxs[1:], idxs[:-1]):
798 yield _genslice(start, prev, step)
801 if i - prev in [-1, 1]:
810 yield _genslice(start, i, step)
812 test_string = ''.join(map(compat_chr, range(len(example_sig))))
813 cache_res = func(test_string)
814 cache_spec = [ord(c) for c in cache_res]
815 expr_code = ' + '.join(gen_sig_code(cache_spec))
816 signature_id_tuple = '(%s)' % (
817 ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
818 code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
819 ' return %s\n') % (signature_id_tuple, expr_code)
820 self.to_screen('Extracted signature function:\n' + code)
822 def _parse_sig_js(self, jscode):
823 funcname = self._search_regex(
824 r'\.sig\|\|([a-zA-Z0-9$]+)\(', jscode,
825 'Initial JS player signature function name')
827 jsi = JSInterpreter(jscode)
828 initial_function = jsi.extract_function(funcname)
829 return lambda s: initial_function([s])
831 def _parse_sig_swf(self, file_contents):
832 swfi = SWFInterpreter(file_contents)
833 TARGET_CLASSNAME = 'SignatureDecipher'
834 searched_class = swfi.extract_class(TARGET_CLASSNAME)
835 initial_function = swfi.extract_function(searched_class, 'decipher')
836 return lambda s: initial_function([s])
838 def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
839 """Turn the encrypted s field into a working signature"""
841 if player_url is None:
842 raise ExtractorError('Cannot decrypt signature without player_url')
844 if player_url.startswith('//'):
845 player_url = 'https:' + player_url
847 player_id = (player_url, self._signature_cache_id(s))
848 if player_id not in self._player_cache:
849 func = self._extract_signature_function(
850 video_id, player_url, s
852 self._player_cache[player_id] = func
853 func = self._player_cache[player_id]
854 if self._downloader.params.get('youtube_print_sig_code'):
855 self._print_sig_code(func, s)
857 except Exception as e:
858 tb = traceback.format_exc()
859 raise ExtractorError(
860 'Signature extraction failed: ' + tb, cause=e)
862 def _get_subtitles(self, video_id, webpage):
864 subs_doc = self._download_xml(
865 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
866 video_id, note=False)
867 except ExtractorError as err:
868 self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
872 for track in subs_doc.findall('track'):
873 lang = track.attrib['lang_code']
874 if lang in sub_lang_list:
877 for ext in ['sbv', 'vtt', 'srt']:
878 params = compat_urllib_parse.urlencode({
882 'name': track.attrib['name'].encode('utf-8'),
885 'url': 'https://www.youtube.com/api/timedtext?' + params,
888 sub_lang_list[lang] = sub_formats
889 if not sub_lang_list:
890 self._downloader.report_warning('video doesn\'t have subtitles')
894 def _get_ytplayer_config(self, video_id, webpage):
896 r';ytplayer\.config\s*=\s*({.+?});ytplayer',
897 r';ytplayer\.config\s*=\s*({.+?});',
899 config = self._search_regex(
900 patterns, webpage, 'ytplayer.config', default=None)
902 return self._parse_json(
903 uppercase_escape(config), video_id, fatal=False)
905 def _get_automatic_captions(self, video_id, webpage):
906 """We need the webpage for getting the captions url, pass it as an
907 argument to speed up the process."""
908 self.to_screen('%s: Looking for automatic captions' % video_id)
909 player_config = self._get_ytplayer_config(video_id, webpage)
910 err_msg = 'Couldn\'t find automatic captions for %s' % video_id
911 if not player_config:
912 self._downloader.report_warning(err_msg)
915 args = player_config['args']
916 caption_url = args['ttsurl']
917 timestamp = args['timestamp']
918 # We get the available subtitles
919 list_params = compat_urllib_parse.urlencode({
924 list_url = caption_url + '&' + list_params
925 caption_list = self._download_xml(list_url, video_id)
926 original_lang_node = caption_list.find('track')
927 if original_lang_node is None:
928 self._downloader.report_warning('Video doesn\'t have automatic captions')
930 original_lang = original_lang_node.attrib['lang_code']
931 caption_kind = original_lang_node.attrib.get('kind', '')
934 for lang_node in caption_list.findall('target'):
935 sub_lang = lang_node.attrib['lang_code']
937 for ext in ['sbv', 'vtt', 'srt']:
938 params = compat_urllib_parse.urlencode({
939 'lang': original_lang,
943 'kind': caption_kind,
946 'url': caption_url + '&' + params,
949 sub_lang_list[sub_lang] = sub_formats
951 # An extractor error can be raise by the download process if there are
952 # no automatic captions but there are subtitles
953 except (KeyError, ExtractorError):
954 self._downloader.report_warning(err_msg)
958 def extract_id(cls, url):
959 mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
961 raise ExtractorError('Invalid URL: %s' % url)
962 video_id = mobj.group(2)
965 def _extract_from_m3u8(self, manifest_url, video_id):
968 def _get_urls(_manifest):
969 lines = _manifest.split('\n')
970 urls = filter(lambda l: l and not l.startswith('#'),
973 manifest = self._download_webpage(manifest_url, video_id, 'Downloading formats manifest')
974 formats_urls = _get_urls(manifest)
975 for format_url in formats_urls:
976 itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
977 url_map[itag] = format_url
980 def _extract_annotations(self, video_id):
981 url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
982 return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
984 def _parse_dash_manifest(
985 self, video_id, dash_manifest_url, player_url, age_gate, fatal=True):
986 def decrypt_sig(mobj):
988 dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
989 return '/signature/%s' % dec_s
990 dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
991 dash_doc = self._download_xml(
992 dash_manifest_url, video_id,
993 note='Downloading DASH manifest',
994 errnote='Could not download DASH manifest',
997 if dash_doc is False:
1001 for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
1002 mime_type = a.attrib.get('mimeType')
1003 for r in a.findall('{urn:mpeg:DASH:schema:MPD:2011}Representation'):
1004 url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
1007 if mime_type == 'text/vtt':
1008 # TODO implement WebVTT downloading
1010 elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
1011 segment_list = r.find('{urn:mpeg:DASH:schema:MPD:2011}SegmentList')
1012 format_id = r.attrib['id']
1013 video_url = url_el.text
1014 filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
1016 'format_id': format_id,
1018 'width': int_or_none(r.attrib.get('width')),
1019 'height': int_or_none(r.attrib.get('height')),
1020 'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
1021 'asr': int_or_none(r.attrib.get('audioSamplingRate')),
1022 'filesize': filesize,
1023 'fps': int_or_none(r.attrib.get('frameRate')),
1025 if segment_list is not None:
1027 'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
1028 'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
1029 'protocol': 'http_dash_segments',
1032 existing_format = next(
1033 fo for fo in formats
1034 if fo['format_id'] == format_id)
1035 except StopIteration:
1036 full_info = self._formats.get(format_id, {}).copy()
1038 codecs = r.attrib.get('codecs')
1040 if full_info.get('acodec') == 'none' and 'vcodec' not in full_info:
1041 full_info['vcodec'] = codecs
1042 elif full_info.get('vcodec') == 'none' and 'acodec' not in full_info:
1043 full_info['acodec'] = codecs
1044 formats.append(full_info)
1046 existing_format.update(f)
1048 self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
1051 def _real_extract(self, url):
1052 url, smuggled_data = unsmuggle_url(url, {})
1055 'http' if self._downloader.params.get('prefer_insecure', False)
1060 parsed_url = compat_urllib_parse_urlparse(url)
1061 for component in [parsed_url.fragment, parsed_url.query]:
1062 query = compat_parse_qs(component)
1063 if start_time is None and 't' in query:
1064 start_time = parse_duration(query['t'][0])
1065 if start_time is None and 'start' in query:
1066 start_time = parse_duration(query['start'][0])
1067 if end_time is None and 'end' in query:
1068 end_time = parse_duration(query['end'][0])
1070 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1071 mobj = re.search(self._NEXT_URL_RE, url)
1073 url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
1074 video_id = self.extract_id(url)
1077 url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
1078 video_webpage = self._download_webpage(url, video_id)
1080 # Attempt to extract SWF player URL
1081 mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1082 if mobj is not None:
1083 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1089 def add_dash_mpd(video_info):
1090 dash_mpd = video_info.get('dashmpd')
1091 if dash_mpd and dash_mpd[0] not in dash_mpds:
1092 dash_mpds.append(dash_mpd[0])
1095 embed_webpage = None
1097 if re.search(r'player-age-gate-content">', video_webpage) is not None:
1099 # We simulate the access to the video from www.youtube.com/v/{video_id}
1100 # this can be viewed without login into Youtube
1101 url = proto + '://www.youtube.com/embed/%s' % video_id
1102 embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
1103 data = compat_urllib_parse.urlencode({
1104 'video_id': video_id,
1105 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1106 'sts': self._search_regex(
1107 r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
1109 video_info_url = proto + '://www.youtube.com/get_video_info?' + data
1110 video_info_webpage = self._download_webpage(
1111 video_info_url, video_id,
1112 note='Refetching age-gated info webpage',
1113 errnote='unable to download video info webpage')
1114 video_info = compat_parse_qs(video_info_webpage)
1115 add_dash_mpd(video_info)
1119 # Try looking directly into the video webpage
1120 ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
1122 args = ytplayer_config['args']
1123 if args.get('url_encoded_fmt_stream_map'):
1124 # Convert to the same format returned by compat_parse_qs
1125 video_info = dict((k, [v]) for k, v in args.items())
1126 add_dash_mpd(video_info)
1127 if args.get('livestream') == '1' or args.get('live_playback') == 1:
1129 if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
1130 # We also try looking in get_video_info since it may contain different dashmpd
1131 # URL that points to a DASH manifest with possibly different itag set (some itags
1132 # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
1133 # manifest pointed by get_video_info's dashmpd).
1134 # The general idea is to take a union of itags of both DASH manifests (for example
1135 # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
1136 self.report_video_info_webpage_download(video_id)
1137 for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
1139 '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1140 % (proto, video_id, el_type))
1141 video_info_webpage = self._download_webpage(
1143 video_id, note=False,
1144 errnote='unable to download video info webpage')
1145 get_video_info = compat_parse_qs(video_info_webpage)
1146 if get_video_info.get('use_cipher_signature') != ['True']:
1147 add_dash_mpd(get_video_info)
1149 video_info = get_video_info
1150 if 'token' in get_video_info:
1151 # Different get_video_info requests may report different results, e.g.
1152 # some may report video unavailability, but some may serve it without
1153 # any complaint (see https://github.com/rg3/youtube-dl/issues/7362,
1154 # the original webpage as well as el=info and el=embedded get_video_info
1155 # requests report video unavailability due to geo restriction while
1156 # el=detailpage succeeds and returns valid data). This is probably
1157 # due to YouTube measures against IP ranges of hosting providers.
1158 # Working around by preferring the first succeeded video_info containing
1159 # the token if no such video_info yet was found.
1160 if 'token' not in video_info:
1161 video_info = get_video_info
1163 if 'token' not in video_info:
1164 if 'reason' in video_info:
1165 if 'The uploader has not made this video available in your country.' in video_info['reason']:
1166 regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
1168 raise ExtractorError('YouTube said: This video is available in %s only' % (
1169 ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
1171 raise ExtractorError(
1172 'YouTube said: %s' % video_info['reason'][0],
1173 expected=True, video_id=video_id)
1175 raise ExtractorError(
1176 '"token" parameter not in video info for unknown reason',
1180 if 'title' in video_info:
1181 video_title = video_info['title'][0]
1183 self._downloader.report_warning('Unable to extract video title')
1187 video_description = get_element_by_id("eow-description", video_webpage)
1188 if video_description:
1189 video_description = re.sub(r'''(?x)
1191 (?:[a-zA-Z-]+="[^"]+"\s+)*?
1193 (?:[a-zA-Z-]+="[^"]+"\s+)*?
1194 class="yt-uix-redirect-link"\s*>
1197 ''', r'\1', video_description)
1198 video_description = clean_html(video_description)
1200 fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
1202 video_description = unescapeHTML(fd_mobj.group(1))
1204 video_description = ''
1206 if 'multifeed_metadata_list' in video_info and not smuggled_data.get('force_singlefeed', False):
1207 if not self._downloader.params.get('noplaylist'):
1210 multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
1211 for feed in multifeed_metadata_list.split(','):
1212 feed_data = compat_parse_qs(feed)
1214 '_type': 'url_transparent',
1215 'ie_key': 'Youtube',
1217 '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
1218 {'force_singlefeed': True}),
1219 'title': '%s (%s)' % (video_title, feed_data['title'][0]),
1221 feed_ids.append(feed_data['id'][0])
1223 'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
1224 % (', '.join(feed_ids), video_id))
1225 return self.playlist_result(entries, video_id, video_title, video_description)
1226 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1228 if 'view_count' in video_info:
1229 view_count = int(video_info['view_count'][0])
1233 # Check for "rental" videos
1234 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1235 raise ExtractorError('"rental" videos not supported')
1237 # Start extracting information
1238 self.report_information_extraction(video_id)
1241 if 'author' not in video_info:
1242 raise ExtractorError('Unable to extract uploader name')
1243 video_uploader = compat_urllib_parse_unquote_plus(video_info['author'][0])
1246 video_uploader_id = None
1247 mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
1248 if mobj is not None:
1249 video_uploader_id = mobj.group(1)
1251 self._downloader.report_warning('unable to extract uploader nickname')
1254 # We try first to get a high quality image:
1255 m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
1256 video_webpage, re.DOTALL)
1257 if m_thumb is not None:
1258 video_thumbnail = m_thumb.group(1)
1259 elif 'thumbnail_url' not in video_info:
1260 self._downloader.report_warning('unable to extract video thumbnail')
1261 video_thumbnail = None
1262 else: # don't panic if we can't find it
1263 video_thumbnail = compat_urllib_parse_unquote_plus(video_info['thumbnail_url'][0])
1266 upload_date = self._html_search_meta(
1267 'datePublished', video_webpage, 'upload date', default=None)
1269 upload_date = self._search_regex(
1270 [r'(?s)id="eow-date.*?>(.*?)</span>',
1271 r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.+?)</strong>'],
1272 video_webpage, 'upload date', default=None)
1274 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1275 upload_date = unified_strdate(upload_date)
1277 m_cat_container = self._search_regex(
1278 r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
1279 video_webpage, 'categories', default=None)
1281 category = self._html_search_regex(
1282 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
1284 video_categories = None if category is None else [category]
1286 video_categories = None
1289 unescapeHTML(m.group('content'))
1290 for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
1292 def _extract_count(count_name):
1293 return str_to_int(self._search_regex(
1294 r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
1295 % re.escape(count_name),
1296 video_webpage, count_name, default=None))
1298 like_count = _extract_count('like')
1299 dislike_count = _extract_count('dislike')
1302 video_subtitles = self.extract_subtitles(video_id, video_webpage)
1303 automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
1305 if 'length_seconds' not in video_info:
1306 self._downloader.report_warning('unable to extract video duration')
1307 video_duration = None
1309 video_duration = int(compat_urllib_parse_unquote_plus(video_info['length_seconds'][0]))
1312 video_annotations = None
1313 if self._downloader.params.get('writeannotations', False):
1314 video_annotations = self._extract_annotations(video_id)
1316 def _map_to_format_list(urlmap):
1318 for itag, video_real_url in urlmap.items():
1321 'url': video_real_url,
1322 'player_url': player_url,
1324 if itag in self._formats:
1325 dct.update(self._formats[itag])
1329 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1330 self.report_rtmp_download()
1332 'format_id': '_rtmp',
1334 'url': video_info['conn'][0],
1335 'player_url': player_url,
1337 elif len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1:
1338 encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
1339 if 'rtmpe%3Dyes' in encoded_url_map:
1340 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
1342 for url_data_str in encoded_url_map.split(','):
1343 url_data = compat_parse_qs(url_data_str)
1344 if 'itag' not in url_data or 'url' not in url_data:
1346 format_id = url_data['itag'][0]
1347 url = url_data['url'][0]
1349 if 'sig' in url_data:
1350 url += '&signature=' + url_data['sig'][0]
1351 elif 's' in url_data:
1352 encrypted_sig = url_data['s'][0]
1353 ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
1355 jsplayer_url_json = self._search_regex(
1357 embed_webpage if age_gate else video_webpage,
1358 'JS player URL (1)', default=None)
1359 if not jsplayer_url_json and not age_gate:
1360 # We need the embed website after all
1361 if embed_webpage is None:
1362 embed_url = proto + '://www.youtube.com/embed/%s' % video_id
1363 embed_webpage = self._download_webpage(
1364 embed_url, video_id, 'Downloading embed webpage')
1365 jsplayer_url_json = self._search_regex(
1366 ASSETS_RE, embed_webpage, 'JS player URL')
1368 player_url = json.loads(jsplayer_url_json)
1369 if player_url is None:
1370 player_url_json = self._search_regex(
1371 r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
1372 video_webpage, 'age gate player URL')
1373 player_url = json.loads(player_url_json)
1375 if self._downloader.params.get('verbose'):
1376 if player_url is None:
1377 player_version = 'unknown'
1378 player_desc = 'unknown'
1380 if player_url.endswith('swf'):
1381 player_version = self._search_regex(
1382 r'-(.+?)(?:/watch_as3)?\.swf$', player_url,
1383 'flash player', fatal=False)
1384 player_desc = 'flash player %s' % player_version
1386 player_version = self._search_regex(
1387 [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js', r'(?:www|player)-([^/]+)/base\.js'],
1389 'html5 player', fatal=False)
1390 player_desc = 'html5 player %s' % player_version
1392 parts_sizes = self._signature_cache_id(encrypted_sig)
1393 self.to_screen('{%s} signature length %s, %s' %
1394 (format_id, parts_sizes, player_desc))
1396 signature = self._decrypt_signature(
1397 encrypted_sig, video_id, player_url, age_gate)
1398 url += '&signature=' + signature
1399 if 'ratebypass' not in url:
1400 url += '&ratebypass=yes'
1402 # Some itags are not included in DASH manifest thus corresponding formats will
1403 # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).
1404 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
1405 mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
1406 width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
1408 'format_id': format_id,
1410 'player_url': player_url,
1411 'filesize': int_or_none(url_data.get('clen', [None])[0]),
1412 'tbr': float_or_none(url_data.get('bitrate', [None])[0], 1000),
1415 'fps': int_or_none(url_data.get('fps', [None])[0]),
1416 'format_note': url_data.get('quality_label', [None])[0] or url_data.get('quality', [None])[0],
1418 type_ = url_data.get('type', [None])[0]
1420 type_split = type_.split(';')
1421 kind_ext = type_split[0].split('/')
1422 if len(kind_ext) == 2:
1423 kind, ext = kind_ext
1425 if kind in ('audio', 'video'):
1427 for mobj in re.finditer(
1428 r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
1429 if mobj.group('key') == 'codecs':
1430 codecs = mobj.group('val')
1433 codecs = codecs.split(',')
1434 if len(codecs) == 2:
1435 acodec, vcodec = codecs[0], codecs[1]
1437 acodec, vcodec = (codecs[0], 'none') if kind == 'audio' else ('none', codecs[0])
1442 if format_id in self._formats:
1443 dct.update(self._formats[format_id])
1445 elif video_info.get('hlsvp'):
1446 manifest_url = video_info['hlsvp'][0]
1447 url_map = self._extract_from_m3u8(manifest_url, video_id)
1448 formats = _map_to_format_list(url_map)
1450 raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
1452 # Look for the DASH manifest
1453 if self._downloader.params.get('youtube_include_dash_manifest', True):
1454 dash_mpd_fatal = True
1455 for dash_manifest_url in dash_mpds:
1458 for df in self._parse_dash_manifest(
1459 video_id, dash_manifest_url, player_url, age_gate, dash_mpd_fatal):
1460 # Do not overwrite DASH format found in some previous DASH manifest
1461 if df['format_id'] not in dash_formats:
1462 dash_formats[df['format_id']] = df
1463 # Additional DASH manifests may end up in HTTP Error 403 therefore
1464 # allow them to fail without bug report message if we already have
1465 # some DASH manifest succeeded. This is temporary workaround to reduce
1466 # burst of bug reports until we figure out the reason and whether it
1467 # can be fixed at all.
1468 dash_mpd_fatal = False
1469 except (ExtractorError, KeyError) as e:
1470 self.report_warning(
1471 'Skipping DASH manifest: %r' % e, video_id)
1473 # Remove the formats we found through non-DASH, they
1474 # contain less info and it can be wrong, because we use
1475 # fixed values (for example the resolution). See
1476 # https://github.com/rg3/youtube-dl/issues/5774 for an
1478 formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
1479 formats.extend(dash_formats.values())
1481 # Check for malformed aspect ratio
1482 stretched_m = re.search(
1483 r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
1486 ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
1488 if f.get('vcodec') != 'none':
1489 f['stretched_ratio'] = ratio
1491 self._sort_formats(formats)
1495 'uploader': video_uploader,
1496 'uploader_id': video_uploader_id,
1497 'upload_date': upload_date,
1498 'title': video_title,
1499 'thumbnail': video_thumbnail,
1500 'description': video_description,
1501 'categories': video_categories,
1503 'subtitles': video_subtitles,
1504 'automatic_captions': automatic_captions,
1505 'duration': video_duration,
1506 'age_limit': 18 if age_gate else 0,
1507 'annotations': video_annotations,
1508 'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
1509 'view_count': view_count,
1510 'like_count': like_count,
1511 'dislike_count': dislike_count,
1512 'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
1515 'start_time': start_time,
1516 'end_time': end_time,
1520 class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtractor):
1521 IE_DESC = 'YouTube.com playlists'
1522 _VALID_URL = r"""(?x)(?:
1527 (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
1528 \? (?:.*?&)*? (?:p|a|list)=
1532 (?:PL|LL|EC|UU|FL|RD|UL)?[0-9A-Za-z-_]{10,}
1533 # Top tracks, they can also include dots
1538 ((?:PL|LL|EC|UU|FL|RD|UL)[0-9A-Za-z-_]{10,})
1540 _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
1541 _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&[^"]*?index=(?P<index>\d+)(?:[^>]+>(?P<title>[^<]+))?'
1542 IE_NAME = 'youtube:playlist'
1544 'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
1546 'title': 'ytdl test PL',
1547 'id': 'PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
1549 'playlist_count': 3,
1551 'url': 'https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
1553 'id': 'PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
1554 'title': 'YDL_Empty_List',
1556 'playlist_count': 0,
1558 'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
1559 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
1561 'title': '29C3: Not my department',
1562 'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
1564 'playlist_count': 95,
1566 'note': 'issue #673',
1567 'url': 'PLBB231211A4F62143',
1569 'title': '[OLD]Team Fortress 2 (Class-based LP)',
1570 'id': 'PLBB231211A4F62143',
1572 'playlist_mincount': 26,
1574 'note': 'Large playlist',
1575 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
1577 'title': 'Uploads from Cauchemar',
1578 'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
1580 'playlist_mincount': 799,
1582 'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
1584 'title': 'YDL_safe_search',
1585 'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
1587 'playlist_count': 2,
1590 'url': 'http://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
1591 'playlist_count': 4,
1594 'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
1597 'note': 'Embedded SWF player',
1598 'url': 'http://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
1599 'playlist_count': 4,
1602 'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
1605 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
1606 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
1608 'title': 'Uploads from Interstellar Movie',
1609 'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
1611 'playlist_mincout': 21,
1614 def _real_initialize(self):
1617 def _extract_mix(self, playlist_id):
1618 # The mixes are generated from a single video
1619 # the id of the playlist is just 'RD' + video_id
1620 url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id)
1621 webpage = self._download_webpage(
1622 url, playlist_id, 'Downloading Youtube mix')
1623 search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
1625 search_title('playlist-title') or
1626 search_title('title long-title') or
1627 search_title('title'))
1628 title = clean_html(title_span)
1629 ids = orderedSet(re.findall(
1630 r'''(?xs)data-video-username=".*?".*?
1631 href="/watch\?v=([0-9A-Za-z_-]{11})&[^"]*?list=%s''' % re.escape(playlist_id),
1633 url_results = self._ids_to_results(ids)
1635 return self.playlist_result(url_results, playlist_id, title)
1637 def _extract_playlist(self, playlist_id):
1638 url = self._TEMPLATE_URL % playlist_id
1639 page = self._download_webpage(url, playlist_id)
1641 for match in re.findall(r'<div class="yt-alert-message">([^<]+)</div>', page):
1642 match = match.strip()
1643 # Check if the playlist exists or is private
1644 if re.match(r'[^<]*(The|This) playlist (does not exist|is private)[^<]*', match):
1645 raise ExtractorError(
1646 'The playlist doesn\'t exist or is private, use --username or '
1647 '--netrc to access it.',
1649 elif re.match(r'[^<]*Invalid parameters[^<]*', match):
1650 raise ExtractorError(
1651 'Invalid parameters. Maybe URL is incorrect.',
1653 elif re.match(r'[^<]*Choose your language[^<]*', match):
1656 self.report_warning('Youtube gives an alert message: ' + match)
1658 playlist_title = self._html_search_regex(
1659 r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
1662 return self.playlist_result(self._entries(page, playlist_id), playlist_id, playlist_title)
1664 def _real_extract(self, url):
1665 # Extract playlist id
1666 mobj = re.match(self._VALID_URL, url)
1668 raise ExtractorError('Invalid URL: %s' % url)
1669 playlist_id = mobj.group(1) or mobj.group(2)
1671 # Check if it's a video-specific URL
1672 query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
1673 if 'v' in query_dict:
1674 video_id = query_dict['v'][0]
1675 if self._downloader.params.get('noplaylist'):
1676 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1677 return self.url_result(video_id, 'Youtube', video_id=video_id)
1679 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
1681 if playlist_id.startswith('RD') or playlist_id.startswith('UL'):
1682 # Mixes require a custom extraction process
1683 return self._extract_mix(playlist_id)
1685 return self._extract_playlist(playlist_id)
1688 class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
1689 IE_DESC = 'YouTube.com channels'
1690 _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
1691 _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
1692 _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
1693 IE_NAME = 'youtube:channel'
1695 'note': 'paginated channel',
1696 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
1697 'playlist_mincount': 91,
1699 'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
1700 'title': 'Uploads from lex will',
1703 'note': 'Age restricted channel',
1704 # from https://www.youtube.com/user/DeusExOfficial
1705 'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
1706 'playlist_mincount': 64,
1708 'id': 'UUs0ifCMCm1icqRbqhUINa0w',
1709 'title': 'Uploads from Deus Ex',
1713 def _real_extract(self, url):
1714 channel_id = self._match_id(url)
1716 url = self._TEMPLATE_URL % channel_id
1718 # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
1719 # Workaround by extracting as a playlist if managed to obtain channel playlist URL
1720 # otherwise fallback on channel by page extraction
1721 channel_page = self._download_webpage(
1722 url + '?view=57', channel_id,
1723 'Downloading channel page', fatal=False)
1724 if channel_page is False:
1725 channel_playlist_id = False
1727 channel_playlist_id = self._html_search_meta(
1728 'channelId', channel_page, 'channel id', default=None)
1729 if not channel_playlist_id:
1730 channel_playlist_id = self._search_regex(
1731 r'data-(?:channel-external-|yt)id="([^"]+)"',
1732 channel_page, 'channel id', default=None)
1733 if channel_playlist_id and channel_playlist_id.startswith('UC'):
1734 playlist_id = 'UU' + channel_playlist_id[2:]
1735 return self.url_result(
1736 compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')
1738 channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
1739 autogenerated = re.search(r'''(?x)
1741 channel-header-autogenerated-label|
1742 yt-channel-title-autogenerated
1743 )[^"]*"''', channel_page) is not None
1746 # The videos are contained in a single page
1747 # the ajax pages can't be used, they are empty
1750 video_id, 'Youtube', video_id=video_id,
1751 video_title=video_title)
1752 for video_id, video_title in self.extract_videos_from_page(channel_page)]
1753 return self.playlist_result(entries, channel_id)
1755 return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
1758 class YoutubeUserIE(YoutubeChannelIE):
1759 IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
1760 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
1761 _TEMPLATE_URL = 'https://www.youtube.com/user/%s/videos'
1762 IE_NAME = 'youtube:user'
1765 'url': 'https://www.youtube.com/user/TheLinuxFoundation',
1766 'playlist_mincount': 320,
1768 'title': 'TheLinuxFoundation',
1771 'url': 'ytuser:phihag',
1772 'only_matching': True,
1776 def suitable(cls, url):
1777 # Don't return True if the url can be extracted with other youtube
1778 # extractor, the regex would is too permissive and it would match.
1779 other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
1780 if any(ie.suitable(url) for ie in other_ies):
1783 return super(YoutubeUserIE, cls).suitable(url)
1786 class YoutubeUserPlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
1787 IE_DESC = 'YouTube.com user playlists'
1788 _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/user/(?P<id>[^/]+)/playlists'
1789 IE_NAME = 'youtube:user:playlists'
1792 'url': 'http://www.youtube.com/user/ThirstForScience/playlists',
1793 'playlist_mincount': 4,
1795 'id': 'ThirstForScience',
1796 'title': 'Thirst for Science',
1799 # with "Load more" button
1800 'url': 'http://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
1801 'playlist_mincount': 70,
1804 'title': 'Игорь Клейнер',
1809 class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
1810 IE_DESC = 'YouTube.com searches'
1811 # there doesn't appear to be a real limit, for example if you search for
1812 # 'python' you get more than 8.000.000 results
1813 _MAX_RESULTS = float('inf')
1814 IE_NAME = 'youtube:search'
1815 _SEARCH_KEY = 'ytsearch'
1816 _EXTRA_QUERY_ARGS = {}
1819 def _get_n_results(self, query, n):
1820 """Get a specified number of results for a query"""
1825 for pagenum in itertools.count(1):
1827 'search_query': query.encode('utf-8'),
1831 url_query.update(self._EXTRA_QUERY_ARGS)
1832 result_url = 'https://www.youtube.com/results?' + compat_urllib_parse.urlencode(url_query)
1833 data = self._download_json(
1834 result_url, video_id='query "%s"' % query,
1835 note='Downloading page %s' % pagenum,
1836 errnote='Unable to download API page')
1837 html_content = data[1]['body']['content']
1839 if 'class="search-message' in html_content:
1840 raise ExtractorError(
1841 '[youtube] No video results', expected=True)
1843 new_videos = self._ids_to_results(orderedSet(re.findall(
1844 r'href="/watch\?v=(.{11})', html_content)))
1845 videos += new_videos
1846 if not new_videos or len(videos) > limit:
1851 return self.playlist_result(videos, query)
1854 class YoutubeSearchDateIE(YoutubeSearchIE):
1855 IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
1856 _SEARCH_KEY = 'ytsearchdate'
1857 IE_DESC = 'YouTube.com searches, newest videos first'
1858 _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}
1861 class YoutubeSearchURLIE(InfoExtractor):
1862 IE_DESC = 'YouTube.com search URLs'
1863 IE_NAME = 'youtube:search_url'
1864 _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?search_query=(?P<query>[^&]+)(?:[&]|$)'
1866 'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
1867 'playlist_mincount': 5,
1869 'title': 'youtube-dl test video',
1873 def _real_extract(self, url):
1874 mobj = re.match(self._VALID_URL, url)
1875 query = compat_urllib_parse_unquote_plus(mobj.group('query'))
1877 webpage = self._download_webpage(url, query)
1878 result_code = self._search_regex(
1879 r'(?s)<ol[^>]+class="item-section"(.*?)</ol>', webpage, 'result HTML')
1881 part_codes = re.findall(
1882 r'(?s)<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*>(.*?)</h3>', result_code)
1884 for part_code in part_codes:
1885 part_title = self._html_search_regex(
1886 [r'(?s)title="([^"]+)"', r'>([^<]+)</a>'], part_code, 'item title', fatal=False)
1887 part_url_snippet = self._html_search_regex(
1888 r'(?s)href="([^"]+)"', part_code, 'item URL')
1889 part_url = compat_urlparse.urljoin(
1890 'https://www.youtube.com/', part_url_snippet)
1894 'title': part_title,
1898 '_type': 'playlist',
1904 class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
1905 IE_DESC = 'YouTube.com (multi-season) shows'
1906 _VALID_URL = r'https?://www\.youtube\.com/show/(?P<id>[^?#]*)'
1907 IE_NAME = 'youtube:show'
1909 'url': 'https://www.youtube.com/show/airdisasters',
1910 'playlist_mincount': 5,
1912 'id': 'airdisasters',
1913 'title': 'Air Disasters',
1917 def _real_extract(self, url):
1918 playlist_id = self._match_id(url)
1919 return super(YoutubeShowIE, self)._real_extract(
1920 'https://www.youtube.com/show/%s/playlists' % playlist_id)
1923 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
1925 Base class for feed extractors
1926 Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
1928 _LOGIN_REQUIRED = True
1932 return 'youtube:%s' % self._FEED_NAME
1934 def _real_initialize(self):
1937 def _real_extract(self, url):
1938 page = self._download_webpage(
1939 'https://www.youtube.com/feed/%s' % self._FEED_NAME, self._PLAYLIST_TITLE)
1941 # The extraction process is the same as for playlists, but the regex
1942 # for the video ids doesn't contain an index
1944 more_widget_html = content_html = page
1945 for page_num in itertools.count(1):
1946 matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
1948 # 'recommended' feed has infinite 'load more' and each new portion spins
1949 # the same videos in (sometimes) slightly different order, so we'll check
1950 # for unicity and break when portion has no new videos
1951 new_ids = filter(lambda video_id: video_id not in ids, orderedSet(matches))
1957 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
1961 more = self._download_json(
1962 'https://youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
1963 'Downloading page #%s' % page_num,
1964 transform_source=uppercase_escape)
1965 content_html = more['content_html']
1966 more_widget_html = more['load_more_widget_html']
1968 return self.playlist_result(
1969 self._ids_to_results(ids), playlist_title=self._PLAYLIST_TITLE)
1972 class YoutubeWatchLaterIE(YoutubePlaylistIE):
1973 IE_NAME = 'youtube:watchlater'
1974 IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
1975 _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
1977 _TESTS = [] # override PlaylistIE tests
1979 def _real_extract(self, url):
1980 return self._extract_playlist('WL')
1983 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
1984 IE_NAME = 'youtube:favorites'
1985 IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
1986 _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
1987 _LOGIN_REQUIRED = True
1989 def _real_extract(self, url):
1990 webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
1991 playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, 'favourites playlist id')
1992 return self.url_result(playlist_id, 'YoutubePlaylist')
1995 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
1996 IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
1997 _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
1998 _FEED_NAME = 'recommended'
1999 _PLAYLIST_TITLE = 'Youtube Recommended videos'
2002 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
2003 IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
2004 _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
2005 _FEED_NAME = 'subscriptions'
2006 _PLAYLIST_TITLE = 'Youtube Subscriptions'
2009 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
2010 IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
2011 _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
2012 _FEED_NAME = 'history'
2013 _PLAYLIST_TITLE = 'Youtube History'
2016 class YoutubeTruncatedURLIE(InfoExtractor):
2017 IE_NAME = 'youtube:truncated_url'
2018 IE_DESC = False # Do not list
2019 _VALID_URL = r'''(?x)
2021 (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
2024 annotation_id=annotation_[^&]+|
2030 attribution_link\?a=[^&]+
2036 'url': 'http://www.youtube.com/watch?annotation_id=annotation_3951667041',
2037 'only_matching': True,
2039 'url': 'http://www.youtube.com/watch?',
2040 'only_matching': True,
2042 'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
2043 'only_matching': True,
2045 'url': 'https://www.youtube.com/watch?feature=foo',
2046 'only_matching': True,
2048 'url': 'https://www.youtube.com/watch?hl=en-GB',
2049 'only_matching': True,
2051 'url': 'https://www.youtube.com/watch?t=2372',
2052 'only_matching': True,
2055 def _real_extract(self, url):
2056 raise ExtractorError(
2057 'Did you forget to quote the URL? Remember that & is a meta '
2058 'character in most shells, so you want to put the URL in quotes, '
2060 '"http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
2061 ' or simply youtube-dl BaW_jenozKc .',
2065 class YoutubeTruncatedIDIE(InfoExtractor):
2066 IE_NAME = 'youtube:truncated_id'
2067 IE_DESC = False # Do not list
2068 _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
2071 'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
2072 'only_matching': True,
2075 def _real_extract(self, url):
2076 video_id = self._match_id(url)
2077 raise ExtractorError(
2078 'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),