_ Git - youtube-dl/blob - youtube_dl/extractor/youtube.py

   1 # coding: utf-8
   2
   3 from __future__ import unicode_literals
   4
   5
   6 import itertools
   7 import json
   8 import os.path
   9 import random
  10 import re
  11 import time
  12 import traceback
  13
  14 from .common import InfoExtractor, SearchInfoExtractor
  15 from ..jsinterp import JSInterpreter
  16 from ..swfinterp import SWFInterpreter
  17 from ..compat import (
  18     compat_chr,
  19     compat_HTTPError,
  20     compat_kwargs,
  21     compat_parse_qs,
  22     compat_urllib_parse_unquote,
  23     compat_urllib_parse_unquote_plus,
  24     compat_urllib_parse_urlencode,
  25     compat_urllib_parse_urlparse,
  26     compat_urlparse,
  27     compat_str,
  28 )
  29 from ..utils import (
  30     bool_or_none,
  31     clean_html,
  32     dict_get,
  33     error_to_compat_str,
  34     extract_attributes,
  35     ExtractorError,
  36     float_or_none,
  37     get_element_by_attribute,
  38     get_element_by_id,
  39     int_or_none,
  40     mimetype2ext,
  41     orderedSet,
  42     parse_codecs,
  43     parse_duration,
  44     remove_quotes,
  45     remove_start,
  46     smuggle_url,
  47     str_or_none,
  48     str_to_int,
  49     try_get,
  50     unescapeHTML,
  51     unified_strdate,
  52     unsmuggle_url,
  53     uppercase_escape,
  54     url_or_none,
  55     urlencode_postdata,
  56 )
  57
  58
  59 class YoutubeBaseInfoExtractor(InfoExtractor):
  60     """Provide base functions for Youtube extractors"""
  61     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
  62     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
  63
  64     _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
  65     _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
  66     _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
  67
  68     _NETRC_MACHINE = 'youtube'
  69     # If True it will raise an error if no login info is provided
  70     _LOGIN_REQUIRED = False
  71
  72     _PLAYLIST_ID_RE = r'(?:PL|LL|EC|UU|FL|RD|UL|TL|OLAK5uy_)[0-9A-Za-z-_]{10,}'
  73
  74     def _set_language(self):
  75         self._set_cookie(
  76             '.youtube.com', 'PREF', 'f1=50000000&hl=en',
  77             # YouTube sets the expire time to about two months
  78             expire_time=time.time() + 2 * 30 * 24 * 3600)
  79
  80     def _ids_to_results(self, ids):
  81         return [
  82             self.url_result(vid_id, 'Youtube', video_id=vid_id)
  83             for vid_id in ids]
  84
  85     def _login(self):
  86         """
  87         Attempt to log in to YouTube.
  88         True is returned if successful or skipped.
  89         False is returned if login failed.
  90
  91         If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
  92         """
  93         username, password = self._get_login_info()
  94         # No authentication to be performed
  95         if username is None:
  96             if self._LOGIN_REQUIRED and self._downloader.params.get('cookiefile') is None:
  97                 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
  98             return True
  99
 100         login_page = self._download_webpage(
 101             self._LOGIN_URL, None,
 102             note='Downloading login page',
 103             errnote='unable to fetch login page', fatal=False)
 104         if login_page is False:
 105             return
 106
 107         login_form = self._hidden_inputs(login_page)
 108
 109         def req(url, f_req, note, errnote):
 110             data = login_form.copy()
 111             data.update({
 112                 'pstMsg': 1,
 113                 'checkConnection': 'youtube',
 114                 'checkedDomains': 'youtube',
 115                 'hl': 'en',
 116                 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
 117                 'f.req': json.dumps(f_req),
 118                 'flowName': 'GlifWebSignIn',
 119                 'flowEntry': 'ServiceLogin',
 120                 # TODO: reverse actual botguard identifier generation algo
 121                 'bgRequest': '["identifier",""]',
 122             })
 123             return self._download_json(
 124                 url, None, note=note, errnote=errnote,
 125                 transform_source=lambda s: re.sub(r'^[^[]*', '', s),
 126                 fatal=False,
 127                 data=urlencode_postdata(data), headers={
 128                     'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
 129                     'Google-Accounts-XSRF': 1,
 130                 })
 131
 132         def warn(message):
 133             self._downloader.report_warning(message)
 134
 135         lookup_req = [
 136             username,
 137             None, [], None, 'US', None, None, 2, False, True,
 138             [
 139                 None, None,
 140                 [2, 1, None, 1,
 141                  'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
 142                  None, [], 4],
 143                 1, [None, None, []], None, None, None, True
 144             ],
 145             username,
 146         ]
 147
 148         lookup_results = req(
 149             self._LOOKUP_URL, lookup_req,
 150             'Looking up account info', 'Unable to look up account info')
 151
 152         if lookup_results is False:
 153             return False
 154
 155         user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str)
 156         if not user_hash:
 157             warn('Unable to extract user hash')
 158             return False
 159
 160         challenge_req = [
 161             user_hash,
 162             None, 1, None, [1, None, None, None, [password, None, True]],
 163             [
 164                 None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
 165                 1, [None, None, []], None, None, None, True
 166             ]]
 167
 168         challenge_results = req(
 169             self._CHALLENGE_URL, challenge_req,
 170             'Logging in', 'Unable to log in')
 171
 172         if challenge_results is False:
 173             return
 174
 175         login_res = try_get(challenge_results, lambda x: x[0][5], list)
 176         if login_res:
 177             login_msg = try_get(login_res, lambda x: x[5], compat_str)
 178             warn(
 179                 'Unable to login: %s' % 'Invalid password'
 180                 if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg)
 181             return False
 182
 183         res = try_get(challenge_results, lambda x: x[0][-1], list)
 184         if not res:
 185             warn('Unable to extract result entry')
 186             return False
 187
 188         login_challenge = try_get(res, lambda x: x[0][0], list)
 189         if login_challenge:
 190             challenge_str = try_get(login_challenge, lambda x: x[2], compat_str)
 191             if challenge_str == 'TWO_STEP_VERIFICATION':
 192                 # SEND_SUCCESS - TFA code has been successfully sent to phone
 193                 # QUOTA_EXCEEDED - reached the limit of TFA codes
 194                 status = try_get(login_challenge, lambda x: x[5], compat_str)
 195                 if status == 'QUOTA_EXCEEDED':
 196                     warn('Exceeded the limit of TFA codes, try later')
 197                     return False
 198
 199                 tl = try_get(challenge_results, lambda x: x[1][2], compat_str)
 200                 if not tl:
 201                     warn('Unable to extract TL')
 202                     return False
 203
 204                 tfa_code = self._get_tfa_info('2-step verification code')
 205
 206                 if not tfa_code:
 207                     warn(
 208                         'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
 209                         '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
 210                     return False
 211
 212                 tfa_code = remove_start(tfa_code, 'G-')
 213
 214                 tfa_req = [
 215                     user_hash, None, 2, None,
 216                     [
 217                         9, None, None, None, None, None, None, None,
 218                         [None, tfa_code, True, 2]
 219                     ]]
 220
 221                 tfa_results = req(
 222                     self._TFA_URL.format(tl), tfa_req,
 223                     'Submitting TFA code', 'Unable to submit TFA code')
 224
 225                 if tfa_results is False:
 226                     return False
 227
 228                 tfa_res = try_get(tfa_results, lambda x: x[0][5], list)
 229                 if tfa_res:
 230                     tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str)
 231                     warn(
 232                         'Unable to finish TFA: %s' % 'Invalid TFA code'
 233                         if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg)
 234                     return False
 235
 236                 check_cookie_url = try_get(
 237                     tfa_results, lambda x: x[0][-1][2], compat_str)
 238             else:
 239                 CHALLENGES = {
 240                     'LOGIN_CHALLENGE': "This device isn't recognized. For your security, Google wants to make sure it's really you.",
 241                     'USERNAME_RECOVERY': 'Please provide additional information to aid in the recovery process.',
 242                     'REAUTH': "There is something unusual about your activity. For your security, Google wants to make sure it's really you.",
 243                 }
 244                 challenge = CHALLENGES.get(
 245                     challenge_str,
 246                     '%s returned error %s.' % (self.IE_NAME, challenge_str))
 247                 warn('%s\nGo to https://accounts.google.com/, login and solve a challenge.' % challenge)
 248                 return False
 249         else:
 250             check_cookie_url = try_get(res, lambda x: x[2], compat_str)
 251
 252         if not check_cookie_url:
 253             warn('Unable to extract CheckCookie URL')
 254             return False
 255
 256         check_cookie_results = self._download_webpage(
 257             check_cookie_url, None, 'Checking cookie', fatal=False)
 258
 259         if check_cookie_results is False:
 260             return False
 261
 262         if 'https://myaccount.google.com/' not in check_cookie_results:
 263             warn('Unable to log in')
 264             return False
 265
 266         return True
 267
 268     def _download_webpage_handle(self, *args, **kwargs):
 269         query = kwargs.get('query', {}).copy()
 270         query['disable_polymer'] = 'true'
 271         kwargs['query'] = query
 272         return super(YoutubeBaseInfoExtractor, self)._download_webpage_handle(
 273             *args, **compat_kwargs(kwargs))
 274
 275     def _real_initialize(self):
 276         if self._downloader is None:
 277             return
 278         self._set_language()
 279         if not self._login():
 280             return
 281
 282
 283 class YoutubeEntryListBaseInfoExtractor(YoutubeBaseInfoExtractor):
 284     # Extract entries from page with "Load more" button
 285     def _entries(self, page, playlist_id):
 286         more_widget_html = content_html = page
 287         for page_num in itertools.count(1):
 288             for entry in self._process_page(content_html):
 289                 yield entry
 290
 291             mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
 292             if not mobj:
 293                 break
 294
 295             count = 0
 296             retries = 3
 297             while count <= retries:
 298                 try:
 299                     # Downloading page may result in intermittent 5xx HTTP error
 300                     # that is usually worked around with a retry
 301                     more = self._download_json(
 302                         'https://youtube.com/%s' % mobj.group('more'), playlist_id,
 303                         'Downloading page #%s%s'
 304                         % (page_num, ' (retry #%d)' % count if count else ''),
 305                         transform_source=uppercase_escape)
 306                     break
 307                 except ExtractorError as e:
 308                     if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503):
 309                         count += 1
 310                         if count <= retries:
 311                             continue
 312                     raise
 313
 314             content_html = more['content_html']
 315             if not content_html.strip():
 316                 # Some webpages show a "Load more" button but they don't
 317                 # have more videos
 318                 break
 319             more_widget_html = more['load_more_widget_html']
 320
 321
 322 class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
 323     def _process_page(self, content):
 324         for video_id, video_title in self.extract_videos_from_page(content):
 325             yield self.url_result(video_id, 'Youtube', video_id, video_title)
 326
 327     def extract_videos_from_page_impl(self, video_re, page, ids_in_page, titles_in_page):
 328         for mobj in re.finditer(video_re, page):
 329             # The link with index 0 is not the first video of the playlist (not sure if still actual)
 330             if 'index' in mobj.groupdict() and mobj.group('id') == '0':
 331                 continue
 332             video_id = mobj.group('id')
 333             video_title = unescapeHTML(
 334                 mobj.group('title')) if 'title' in mobj.groupdict() else None
 335             if video_title:
 336                 video_title = video_title.strip()
 337             if video_title == '► Play all':
 338                 video_title = None
 339             try:
 340                 idx = ids_in_page.index(video_id)
 341                 if video_title and not titles_in_page[idx]:
 342                     titles_in_page[idx] = video_title
 343             except ValueError:
 344                 ids_in_page.append(video_id)
 345                 titles_in_page.append(video_title)
 346
 347     def extract_videos_from_page(self, page):
 348         ids_in_page = []
 349         titles_in_page = []
 350         self.extract_videos_from_page_impl(
 351             self._VIDEO_RE, page, ids_in_page, titles_in_page)
 352         return zip(ids_in_page, titles_in_page)
 353
 354
 355 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
 356     def _process_page(self, content):
 357         for playlist_id in orderedSet(re.findall(
 358                 r'<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*><a[^>]+href="/?playlist\?list=([0-9A-Za-z-_]{10,})"',
 359                 content)):
 360             yield self.url_result(
 361                 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
 362
 363     def _real_extract(self, url):
 364         playlist_id = self._match_id(url)
 365         webpage = self._download_webpage(url, playlist_id)
 366         title = self._og_search_title(webpage, fatal=False)
 367         return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
 368
 369
 370 class YoutubeIE(YoutubeBaseInfoExtractor):
 371     IE_DESC = 'YouTube.com'
 372     _VALID_URL = r"""(?x)^
 373                      (
 374                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 375                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
 376                             (?:www\.)?deturl\.com/www\.youtube\.com/|
 377                             (?:www\.)?pwnyoutube\.com/|
 378                             (?:www\.)?hooktube\.com/|
 379                             (?:www\.)?yourepeat\.com/|
 380                             tube\.majestyc\.net/|
 381                             # Invidious instances taken from https://github.com/omarroth/invidious/wiki/Invidious-Instances
 382                             (?:(?:www|dev)\.)?invidio\.us/|
 383                             (?:(?:www|no)\.)?invidiou\.sh/|
 384                             (?:(?:www|fi|de)\.)?invidious\.snopyta\.org/|
 385                             (?:www\.)?invidious\.kabi\.tk/|
 386                             (?:www\.)?invidious\.enkirton\.net/|
 387                             (?:www\.)?invidious\.13ad\.de/|
 388                             (?:www\.)?invidious\.mastodon\.host/|
 389                             (?:www\.)?invidious\.nixnet\.xyz/|
 390                             (?:www\.)?tube\.poal\.co/|
 391                             (?:www\.)?vid\.wxzm\.sx/|
 392                             (?:www\.)?yt\.elukerio\.org/|
 393                             (?:www\.)?kgg2m7yk5aybusll\.onion/|
 394                             (?:www\.)?qklhadlycap4cnod\.onion/|
 395                             (?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion/|
 396                             (?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion/|
 397                             (?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion/|
 398                             (?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion/|
 399                             youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
 400                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 401                          (?:                                                  # the various things that can precede the ID:
 402                              (?:(?:v|embed|e)/(?!videoseries))                # v/ or embed/ or e/
 403                              |(?:                                             # or the v= param in all its forms
 404                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 405                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 406                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 407                                  v=
 408                              )
 409                          ))
 410                          |(?:
 411                             youtu\.be|                                        # just youtu.be/xxxx
 412                             vid\.plus|                                        # or vid.plus/xxxx
 413                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 414                          )/
 415                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 416                          )
 417                      )?                                                       # all until now is optional -> you can pass the naked ID
 418                      ([0-9A-Za-z_-]{11})                                      # here is it! the YouTube video ID
 419                      (?!.*?\blist=
 420                         (?:
 421                             %(playlist_id)s|                                  # combined list/video URLs are handled by the playlist IE
 422                             WL                                                # WL are handled by the watch later IE
 423                         )
 424                      )
 425                      (?(1).+)?                                                # if we found the ID, everything can follow
 426                      $""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
 427     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
 428     _formats = {
 429         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 430         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 431         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 432         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 433         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 434         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 435         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 436         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 437         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 438         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 439         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 440         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 441         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 442         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 443         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 444         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 445         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 446         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 447
 448
 449         # 3D videos
 450         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 451         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 452         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 453         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 454         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 455         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 456         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 457
 458         # Apple HTTP Live Streaming
 459         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 460         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 461         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 462         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 463         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 464         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 465         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 466         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 467
 468         # DASH mp4 video
 469         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 470         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 471         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 472         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 473         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 474         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 475         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 476         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 477         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 478         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 479         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 480         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 481
 482         # Dash mp4 audio
 483         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 484         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
 485         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
 486         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 487         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 488         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
 489         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
 490
 491         # Dash webm
 492         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 493         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 494         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 495         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 496         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 497         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 498         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
 499         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 500         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 501         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 502         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 503         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 504         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 505         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 506         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 507         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
 508         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 509         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 510         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 511         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 512         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 513         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 514
 515         # Dash webm audio
 516         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
 517         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
 518
 519         # Dash webm audio with opus inside
 520         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
 521         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
 522         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
 523
 524         # RTMP (unnamed)
 525         '_rtmp': {'protocol': 'rtmp'},
 526
 527         # av01 video only formats sometimes served with "unknown" codecs
 528         '394': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
 529         '395': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
 530         '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
 531         '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
 532     }
 533     _SUBTITLE_FORMATS = ('srv1', 'srv2', 'srv3', 'ttml', 'vtt')
 534
 535     _GEO_BYPASS = False
 536
 537     IE_NAME = 'youtube'
 538     _TESTS = [
 539         {
 540             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
 541             'info_dict': {
 542                 'id': 'BaW_jenozKc',
 543                 'ext': 'mp4',
 544                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
 545                 'uploader': 'Philipp Hagemeister',
 546                 'uploader_id': 'phihag',
 547                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
 548                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
 549                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
 550                 'upload_date': '20121002',
 551                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
 552                 'categories': ['Science & Technology'],
 553                 'tags': ['youtube-dl'],
 554                 'duration': 10,
 555                 'view_count': int,
 556                 'like_count': int,
 557                 'dislike_count': int,
 558                 'start_time': 1,
 559                 'end_time': 9,
 560             }
 561         },
 562         {
 563             'url': 'https://www.youtube.com/watch?v=UxxajLWwzqY',
 564             'note': 'Test generic use_cipher_signature video (#897)',
 565             'info_dict': {
 566                 'id': 'UxxajLWwzqY',
 567                 'ext': 'mp4',
 568                 'upload_date': '20120506',
 569                 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
 570                 'alt_title': 'I Love It (feat. Charli XCX)',
 571                 'description': 'md5:f3ceb5ef83a08d95b9d146f973157cc8',
 572                 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
 573                          'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
 574                          'iconic ep', 'iconic', 'love', 'it'],
 575                 'duration': 180,
 576                 'uploader': 'Icona Pop',
 577                 'uploader_id': 'IconaPop',
 578                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IconaPop',
 579                 'creator': 'Icona Pop',
 580                 'track': 'I Love It (feat. Charli XCX)',
 581                 'artist': 'Icona Pop',
 582             }
 583         },
 584         {
 585             'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
 586             'note': 'Test VEVO video with age protection (#956)',
 587             'info_dict': {
 588                 'id': '07FYdnEawAQ',
 589                 'ext': 'mp4',
 590                 'upload_date': '20130703',
 591                 'title': 'Justin Timberlake - Tunnel Vision (Official Music Video) (Explicit)',
 592                 'alt_title': 'Tunnel Vision',
 593                 'description': 'md5:07dab3356cde4199048e4c7cd93471e1',
 594                 'duration': 419,
 595                 'uploader': 'justintimberlakeVEVO',
 596                 'uploader_id': 'justintimberlakeVEVO',
 597                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
 598                 'creator': 'Justin Timberlake',
 599                 'track': 'Tunnel Vision',
 600                 'artist': 'Justin Timberlake',
 601                 'age_limit': 18,
 602             }
 603         },
 604         {
 605             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
 606             'note': 'Embed-only video (#1746)',
 607             'info_dict': {
 608                 'id': 'yZIXLfi8CZQ',
 609                 'ext': 'mp4',
 610                 'upload_date': '20120608',
 611                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
 612                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
 613                 'uploader': 'SET India',
 614                 'uploader_id': 'setindia',
 615                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
 616                 'age_limit': 18,
 617             }
 618         },
 619         {
 620             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
 621             'note': 'Use the first video ID in the URL',
 622             'info_dict': {
 623                 'id': 'BaW_jenozKc',
 624                 'ext': 'mp4',
 625                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
 626                 'uploader': 'Philipp Hagemeister',
 627                 'uploader_id': 'phihag',
 628                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
 629                 'upload_date': '20121002',
 630                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
 631                 'categories': ['Science & Technology'],
 632                 'tags': ['youtube-dl'],
 633                 'duration': 10,
 634                 'view_count': int,
 635                 'like_count': int,
 636                 'dislike_count': int,
 637             },
 638             'params': {
 639                 'skip_download': True,
 640             },
 641         },
 642         {
 643             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
 644             'note': '256k DASH audio (format 141) via DASH manifest',
 645             'info_dict': {
 646                 'id': 'a9LDPn-MO4I',
 647                 'ext': 'm4a',
 648                 'upload_date': '20121002',
 649                 'uploader_id': '8KVIDEO',
 650                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
 651                 'description': '',
 652                 'uploader': '8KVIDEO',
 653                 'title': 'UHDTV TEST 8K VIDEO.mp4'
 654             },
 655             'params': {
 656                 'youtube_include_dash_manifest': True,
 657                 'format': '141',
 658             },
 659             'skip': 'format 141 not served anymore',
 660         },
 661         # DASH manifest with encrypted signature
 662         {
 663             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
 664             'info_dict': {
 665                 'id': 'IB3lcPjvWLA',
 666                 'ext': 'm4a',
 667                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
 668                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
 669                 'duration': 244,
 670                 'uploader': 'AfrojackVEVO',
 671                 'uploader_id': 'AfrojackVEVO',
 672                 'upload_date': '20131011',
 673             },
 674             'params': {
 675                 'youtube_include_dash_manifest': True,
 676                 'format': '141/bestaudio[ext=m4a]',
 677             },
 678         },
 679         # JS player signature function name containing $
 680         {
 681             'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
 682             'info_dict': {
 683                 'id': 'nfWlot6h_JM',
 684                 'ext': 'm4a',
 685                 'title': 'Taylor Swift - Shake It Off',
 686                 'description': 'md5:bec2185232c05479482cb5a9b82719bf',
 687                 'duration': 242,
 688                 'uploader': 'TaylorSwiftVEVO',
 689                 'uploader_id': 'TaylorSwiftVEVO',
 690                 'upload_date': '20140818',
 691                 'creator': 'Taylor Swift',
 692             },
 693             'params': {
 694                 'youtube_include_dash_manifest': True,
 695                 'format': '141/bestaudio[ext=m4a]',
 696             },
 697         },
 698         # Controversy video
 699         {
 700             'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
 701             'info_dict': {
 702                 'id': 'T4XJQO3qol8',
 703                 'ext': 'mp4',
 704                 'duration': 219,
 705                 'upload_date': '20100909',
 706                 'uploader': 'Amazing Atheist',
 707                 'uploader_id': 'TheAmazingAtheist',
 708                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
 709                 'title': 'Burning Everyone\'s Koran',
 710                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
 711             }
 712         },
 713         # Normal age-gate video (No vevo, embed allowed)
 714         {
 715             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
 716             'info_dict': {
 717                 'id': 'HtVdAasjOgU',
 718                 'ext': 'mp4',
 719                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
 720                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
 721                 'duration': 142,
 722                 'uploader': 'The Witcher',
 723                 'uploader_id': 'WitcherGame',
 724                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
 725                 'upload_date': '20140605',
 726                 'age_limit': 18,
 727             },
 728         },
 729         # Age-gate video with encrypted signature
 730         {
 731             'url': 'https://www.youtube.com/watch?v=6kLq3WMV1nU',
 732             'info_dict': {
 733                 'id': '6kLq3WMV1nU',
 734                 'ext': 'mp4',
 735                 'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
 736                 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
 737                 'duration': 246,
 738                 'uploader': 'LloydVEVO',
 739                 'uploader_id': 'LloydVEVO',
 740                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
 741                 'upload_date': '20110629',
 742                 'age_limit': 18,
 743             },
 744         },
 745         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
 746         # YouTube Red ad is not captured for creator
 747         {
 748             'url': '__2ABJjxzNo',
 749             'info_dict': {
 750                 'id': '__2ABJjxzNo',
 751                 'ext': 'mp4',
 752                 'duration': 266,
 753                 'upload_date': '20100430',
 754                 'uploader_id': 'deadmau5',
 755                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
 756                 'creator': 'deadmau5',
 757                 'description': 'md5:12c56784b8032162bb936a5f76d55360',
 758                 'uploader': 'deadmau5',
 759                 'title': 'Deadmau5 - Some Chords (HD)',
 760                 'alt_title': 'Some Chords',
 761             },
 762             'expected_warnings': [
 763                 'DASH manifest missing',
 764             ]
 765         },
 766         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
 767         {
 768             'url': 'lqQg6PlCWgI',
 769             'info_dict': {
 770                 'id': 'lqQg6PlCWgI',
 771                 'ext': 'mp4',
 772                 'duration': 6085,
 773                 'upload_date': '20150827',
 774                 'uploader_id': 'olympic',
 775                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
 776                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
 777                 'uploader': 'Olympic',
 778                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
 779             },
 780             'params': {
 781                 'skip_download': 'requires avconv',
 782             }
 783         },
 784         # Non-square pixels
 785         {
 786             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
 787             'info_dict': {
 788                 'id': '_b-2C3KPAM0',
 789                 'ext': 'mp4',
 790                 'stretched_ratio': 16 / 9.,
 791                 'duration': 85,
 792                 'upload_date': '20110310',
 793                 'uploader_id': 'AllenMeow',
 794                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
 795                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
 796                 'uploader': '孫ᄋᄅ',
 797                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
 798             },
 799         },
 800         # url_encoded_fmt_stream_map is empty string
 801         {
 802             'url': 'qEJwOuvDf7I',
 803             'info_dict': {
 804                 'id': 'qEJwOuvDf7I',
 805                 'ext': 'webm',
 806                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
 807                 'description': '',
 808                 'upload_date': '20150404',
 809                 'uploader_id': 'spbelect',
 810                 'uploader': 'Наблюдатели Петербурга',
 811             },
 812             'params': {
 813                 'skip_download': 'requires avconv',
 814             },
 815             'skip': 'This live event has ended.',
 816         },
 817         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
 818         {
 819             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
 820             'info_dict': {
 821                 'id': 'FIl7x6_3R5Y',
 822                 'ext': 'webm',
 823                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
 824                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
 825                 'duration': 220,
 826                 'upload_date': '20150625',
 827                 'uploader_id': 'dorappi2000',
 828                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
 829                 'uploader': 'dorappi2000',
 830                 'formats': 'mincount:31',
 831             },
 832             'skip': 'not actual anymore',
 833         },
 834         # DASH manifest with segment_list
 835         {
 836             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
 837             'md5': '8ce563a1d667b599d21064e982ab9e31',
 838             'info_dict': {
 839                 'id': 'CsmdDsKjzN8',
 840                 'ext': 'mp4',
 841                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
 842                 'uploader': 'Airtek',
 843                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
 844                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
 845                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
 846             },
 847             'params': {
 848                 'youtube_include_dash_manifest': True,
 849                 'format': '135',  # bestvideo
 850             },
 851             'skip': 'This live event has ended.',
 852         },
 853         {
 854             # Multifeed videos (multiple cameras), URL is for Main Camera
 855             'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
 856             'info_dict': {
 857                 'id': 'jqWvoWXjCVs',
 858                 'title': 'teamPGP: Rocket League Noob Stream',
 859                 'description': 'md5:dc7872fb300e143831327f1bae3af010',
 860             },
 861             'playlist': [{
 862                 'info_dict': {
 863                     'id': 'jqWvoWXjCVs',
 864                     'ext': 'mp4',
 865                     'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
 866                     'description': 'md5:dc7872fb300e143831327f1bae3af010',
 867                     'duration': 7335,
 868                     'upload_date': '20150721',
 869                     'uploader': 'Beer Games Beer',
 870                     'uploader_id': 'beergamesbeer',
 871                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
 872                     'license': 'Standard YouTube License',
 873                 },
 874             }, {
 875                 'info_dict': {
 876                     'id': '6h8e8xoXJzg',
 877                     'ext': 'mp4',
 878                     'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
 879                     'description': 'md5:dc7872fb300e143831327f1bae3af010',
 880                     'duration': 7337,
 881                     'upload_date': '20150721',
 882                     'uploader': 'Beer Games Beer',
 883                     'uploader_id': 'beergamesbeer',
 884                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
 885                     'license': 'Standard YouTube License',
 886                 },
 887             }, {
 888                 'info_dict': {
 889                     'id': 'PUOgX5z9xZw',
 890                     'ext': 'mp4',
 891                     'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
 892                     'description': 'md5:dc7872fb300e143831327f1bae3af010',
 893                     'duration': 7337,
 894                     'upload_date': '20150721',
 895                     'uploader': 'Beer Games Beer',
 896                     'uploader_id': 'beergamesbeer',
 897                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
 898                     'license': 'Standard YouTube License',
 899                 },
 900             }, {
 901                 'info_dict': {
 902                     'id': 'teuwxikvS5k',
 903                     'ext': 'mp4',
 904                     'title': 'teamPGP: Rocket League Noob Stream (zim)',
 905                     'description': 'md5:dc7872fb300e143831327f1bae3af010',
 906                     'duration': 7334,
 907                     'upload_date': '20150721',
 908                     'uploader': 'Beer Games Beer',
 909                     'uploader_id': 'beergamesbeer',
 910                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
 911                     'license': 'Standard YouTube License',
 912                 },
 913             }],
 914             'params': {
 915                 'skip_download': True,
 916             },
 917             'skip': 'This video is not available.',
 918         },
 919         {
 920             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
 921             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
 922             'info_dict': {
 923                 'id': 'gVfLd0zydlo',
 924                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
 925             },
 926             'playlist_count': 2,
 927             'skip': 'Not multifeed anymore',
 928         },
 929         {
 930             'url': 'https://vid.plus/FlRa-iH7PGw',
 931             'only_matching': True,
 932         },
 933         {
 934             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
 935             'only_matching': True,
 936         },
 937         {
 938             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
 939             # Also tests cut-off URL expansion in video description (see
 940             # https://github.com/ytdl-org/youtube-dl/issues/1892,
 941             # https://github.com/ytdl-org/youtube-dl/issues/8164)
 942             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
 943             'info_dict': {
 944                 'id': 'lsguqyKfVQg',
 945                 'ext': 'mp4',
 946                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
 947                 'alt_title': 'Dark Walk - Position Music',
 948                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
 949                 'duration': 133,
 950                 'upload_date': '20151119',
 951                 'uploader_id': 'IronSoulElf',
 952                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
 953                 'uploader': 'IronSoulElf',
 954                 'creator': 'Todd Haberman,  Daniel Law Heath and Aaron Kaplan',
 955                 'track': 'Dark Walk - Position Music',
 956                 'artist': 'Todd Haberman,  Daniel Law Heath and Aaron Kaplan',
 957                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
 958             },
 959             'params': {
 960                 'skip_download': True,
 961             },
 962         },
 963         {
 964             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
 965             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
 966             'only_matching': True,
 967         },
 968         {
 969             # Video with yt:stretch=17:0
 970             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
 971             'info_dict': {
 972                 'id': 'Q39EVAstoRM',
 973                 'ext': 'mp4',
 974                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
 975                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
 976                 'upload_date': '20151107',
 977                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
 978                 'uploader': 'CH GAMER DROID',
 979             },
 980             'params': {
 981                 'skip_download': True,
 982             },
 983             'skip': 'This video does not exist.',
 984         },
 985         {
 986             # Video licensed under Creative Commons
 987             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
 988             'info_dict': {
 989                 'id': 'M4gD1WSo5mA',
 990                 'ext': 'mp4',
 991                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
 992                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
 993                 'duration': 721,
 994                 'upload_date': '20150127',
 995                 'uploader_id': 'BerkmanCenter',
 996                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
 997                 'uploader': 'The Berkman Klein Center for Internet & Society',
 998                 'license': 'Creative Commons Attribution license (reuse allowed)',
 999             },
1000             'params': {
1001                 'skip_download': True,
1002             },
1003         },
1004         {
1005             # Channel-like uploader_url
1006             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1007             'info_dict': {
1008                 'id': 'eQcmzGIKrzg',
1009                 'ext': 'mp4',
1010                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1011                 'description': 'md5:dda0d780d5a6e120758d1711d062a867',
1012                 'duration': 4060,
1013                 'upload_date': '20151119',
1014                 'uploader': 'Bernie Sanders',
1015                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1016                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1017                 'license': 'Creative Commons Attribution license (reuse allowed)',
1018             },
1019             'params': {
1020                 'skip_download': True,
1021             },
1022         },
1023         {
1024             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1025             'only_matching': True,
1026         },
1027         {
1028             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1029             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1030             'only_matching': True,
1031         },
1032         {
1033             # Rental video preview
1034             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1035             'info_dict': {
1036                 'id': 'uGpuVWrhIzE',
1037                 'ext': 'mp4',
1038                 'title': 'Piku - Trailer',
1039                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1040                 'upload_date': '20150811',
1041                 'uploader': 'FlixMatrix',
1042                 'uploader_id': 'FlixMatrixKaravan',
1043                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1044                 'license': 'Standard YouTube License',
1045             },
1046             'params': {
1047                 'skip_download': True,
1048             },
1049             'skip': 'This video is not available.',
1050         },
1051         {
1052             # YouTube Red video with episode data
1053             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1054             'info_dict': {
1055                 'id': 'iqKdEhx-dD4',
1056                 'ext': 'mp4',
1057                 'title': 'Isolation - Mind Field (Ep 1)',
1058                 'description': 'md5:46a29be4ceffa65b92d277b93f463c0f',
1059                 'duration': 2085,
1060                 'upload_date': '20170118',
1061                 'uploader': 'Vsauce',
1062                 'uploader_id': 'Vsauce',
1063                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1064                 'series': 'Mind Field',
1065                 'season_number': 1,
1066                 'episode_number': 1,
1067             },
1068             'params': {
1069                 'skip_download': True,
1070             },
1071             'expected_warnings': [
1072                 'Skipping DASH manifest',
1073             ],
1074         },
1075         {
1076             # The following content has been identified by the YouTube community
1077             # as inappropriate or offensive to some audiences.
1078             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1079             'info_dict': {
1080                 'id': '6SJNVb0GnPI',
1081                 'ext': 'mp4',
1082                 'title': 'Race Differences in Intelligence',
1083                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1084                 'duration': 965,
1085                 'upload_date': '20140124',
1086                 'uploader': 'New Century Foundation',
1087                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1088                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1089             },
1090             'params': {
1091                 'skip_download': True,
1092             },
1093         },
1094         {
1095             # itag 212
1096             'url': '1t24XAntNCY',
1097             'only_matching': True,
1098         },
1099         {
1100             # geo restricted to JP
1101             'url': 'sJL6WA-aGkQ',
1102             'only_matching': True,
1103         },
1104         {
1105             'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
1106             'only_matching': True,
1107         },
1108         {
1109             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1110             'only_matching': True,
1111         },
1112         {
1113             # DRM protected
1114             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1115             'only_matching': True,
1116         },
1117         {
1118             # Video with unsupported adaptive stream type formats
1119             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1120             'info_dict': {
1121                 'id': 'Z4Vy8R84T1U',
1122                 'ext': 'mp4',
1123                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1124                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1125                 'duration': 433,
1126                 'upload_date': '20130923',
1127                 'uploader': 'Amelia Putri Harwita',
1128                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1129                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1130                 'formats': 'maxcount:10',
1131             },
1132             'params': {
1133                 'skip_download': True,
1134                 'youtube_include_dash_manifest': False,
1135             },
1136         },
1137         {
1138             # Youtube Music Auto-generated description
1139             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1140             'info_dict': {
1141                 'id': 'MgNrAu2pzNs',
1142                 'ext': 'mp4',
1143                 'title': 'Voyeur Girl',
1144                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1145                 'upload_date': '20190312',
1146                 'uploader': 'Various Artists - Topic',
1147                 'uploader_id': 'UCVWKBi1ELZn0QX2CBLSkiyw',
1148                 'artist': 'Stephen',
1149                 'track': 'Voyeur Girl',
1150                 'album': 'it\'s too much love to know my dear',
1151                 'release_date': '20190313',
1152                 'release_year': 2019,
1153             },
1154             'params': {
1155                 'skip_download': True,
1156             },
1157         },
1158         {
1159             # Youtube Music Auto-generated description
1160             # Retrieve 'artist' field from 'Artist:' in video description
1161             # when it is present on youtube music video
1162             'url': 'https://www.youtube.com/watch?v=k0jLE7tTwjY',
1163             'info_dict': {
1164                 'id': 'k0jLE7tTwjY',
1165                 'ext': 'mp4',
1166                 'title': 'Latch Feat. Sam Smith',
1167                 'description': 'md5:3cb1e8101a7c85fcba9b4fb41b951335',
1168                 'upload_date': '20150110',
1169                 'uploader': 'Various Artists - Topic',
1170                 'uploader_id': 'UCNkEcmYdjrH4RqtNgh7BZ9w',
1171                 'artist': 'Disclosure',
1172                 'track': 'Latch Feat. Sam Smith',
1173                 'album': 'Latch Featuring Sam Smith',
1174                 'release_date': '20121008',
1175                 'release_year': 2012,
1176             },
1177             'params': {
1178                 'skip_download': True,
1179             },
1180         },
1181         {
1182             # Youtube Music Auto-generated description
1183             # handle multiple artists on youtube music video
1184             'url': 'https://www.youtube.com/watch?v=74qn0eJSjpA',
1185             'info_dict': {
1186                 'id': '74qn0eJSjpA',
1187                 'ext': 'mp4',
1188                 'title': 'Eastside',
1189                 'description': 'md5:290516bb73dcbfab0dcc4efe6c3de5f2',
1190                 'upload_date': '20180710',
1191                 'uploader': 'Benny Blanco - Topic',
1192                 'uploader_id': 'UCzqz_ksRu_WkIzmivMdIS7A',
1193                 'artist': 'benny blanco, Halsey, Khalid',
1194                 'track': 'Eastside',
1195                 'album': 'Eastside',
1196                 'release_date': '20180713',
1197                 'release_year': 2018,
1198             },
1199             'params': {
1200                 'skip_download': True,
1201             },
1202         },
1203         {
1204             # Youtube Music Auto-generated description
1205             # handle youtube music video with release_year and no release_date
1206             'url': 'https://www.youtube.com/watch?v=-hcAI0g-f5M',
1207             'info_dict': {
1208                 'id': '-hcAI0g-f5M',
1209                 'ext': 'mp4',
1210                 'title': 'Put It On Me',
1211                 'description': 'md5:93c55acc682ae7b0c668f2e34e1c069e',
1212                 'upload_date': '20180426',
1213                 'uploader': 'Matt Maeson - Topic',
1214                 'uploader_id': 'UCnEkIGqtGcQMLk73Kp-Q5LQ',
1215                 'artist': 'Matt Maeson',
1216                 'track': 'Put It On Me',
1217                 'album': 'The Hearse',
1218                 'release_date': None,
1219                 'release_year': 2018,
1220             },
1221             'params': {
1222                 'skip_download': True,
1223             },
1224         },
1225     ]
1226
1227     def __init__(self, *args, **kwargs):
1228         super(YoutubeIE, self).__init__(*args, **kwargs)
1229         self._player_cache = {}
1230
1231     def report_video_info_webpage_download(self, video_id):
1232         """Report attempt to download video info webpage."""
1233         self.to_screen('%s: Downloading video info webpage' % video_id)
1234
1235     def report_information_extraction(self, video_id):
1236         """Report attempt to extract video information."""
1237         self.to_screen('%s: Extracting video information' % video_id)
1238
1239     def report_unavailable_format(self, video_id, format):
1240         """Report extracted video URL."""
1241         self.to_screen('%s: Format %s not available' % (video_id, format))
1242
1243     def report_rtmp_download(self):
1244         """Indicate the download will use the RTMP protocol."""
1245         self.to_screen('RTMP download detected')
1246
1247     def _signature_cache_id(self, example_sig):
1248         """ Return a string representation of a signature """
1249         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
1250
1251     def _extract_signature_function(self, video_id, player_url, example_sig):
1252         id_m = re.match(
1253             r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|(?:/[a-z]{2,3}_[A-Z]{2})?/base)?\.(?P<ext>[a-z]+)$',
1254             player_url)
1255         if not id_m:
1256             raise ExtractorError('Cannot identify player %r' % player_url)
1257         player_type = id_m.group('ext')
1258         player_id = id_m.group('id')
1259
1260         # Read from filesystem cache
1261         func_id = '%s_%s_%s' % (
1262             player_type, player_id, self._signature_cache_id(example_sig))
1263         assert os.path.basename(func_id) == func_id
1264
1265         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
1266         if cache_spec is not None:
1267             return lambda s: ''.join(s[i] for i in cache_spec)
1268
1269         download_note = (
1270             'Downloading player %s' % player_url
1271             if self._downloader.params.get('verbose') else
1272             'Downloading %s player %s' % (player_type, player_id)
1273         )
1274         if player_type == 'js':
1275             code = self._download_webpage(
1276                 player_url, video_id,
1277                 note=download_note,
1278                 errnote='Download of %s failed' % player_url)
1279             res = self._parse_sig_js(code)
1280         elif player_type == 'swf':
1281             urlh = self._request_webpage(
1282                 player_url, video_id,
1283                 note=download_note,
1284                 errnote='Download of %s failed' % player_url)
1285             code = urlh.read()
1286             res = self._parse_sig_swf(code)
1287         else:
1288             assert False, 'Invalid player type %r' % player_type
1289
1290         test_string = ''.join(map(compat_chr, range(len(example_sig))))
1291         cache_res = res(test_string)
1292         cache_spec = [ord(c) for c in cache_res]
1293
1294         self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
1295         return res
1296
1297     def _print_sig_code(self, func, example_sig):
1298         def gen_sig_code(idxs):
1299             def _genslice(start, end, step):
1300                 starts = '' if start == 0 else str(start)
1301                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
1302                 steps = '' if step == 1 else (':%d' % step)
1303                 return 's[%s%s%s]' % (starts, ends, steps)
1304
1305             step = None
1306             # Quelch pyflakes warnings - start will be set when step is set
1307             start = '(Never used)'
1308             for i, prev in zip(idxs[1:], idxs[:-1]):
1309                 if step is not None:
1310                     if i - prev == step:
1311                         continue
1312                     yield _genslice(start, prev, step)
1313                     step = None
1314                     continue
1315                 if i - prev in [-1, 1]:
1316                     step = i - prev
1317                     start = prev
1318                     continue
1319                 else:
1320                     yield 's[%d]' % prev
1321             if step is None:
1322                 yield 's[%d]' % i
1323             else:
1324                 yield _genslice(start, i, step)
1325
1326         test_string = ''.join(map(compat_chr, range(len(example_sig))))
1327         cache_res = func(test_string)
1328         cache_spec = [ord(c) for c in cache_res]
1329         expr_code = ' + '.join(gen_sig_code(cache_spec))
1330         signature_id_tuple = '(%s)' % (
1331             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
1332         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
1333                 '    return %s\n') % (signature_id_tuple, expr_code)
1334         self.to_screen('Extracted signature function:\n' + code)
1335
1336     def _parse_sig_js(self, jscode):
1337         funcname = self._search_regex(
1338             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1339              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1340              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1341              # Obsolete patterns
1342              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1343              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
1344              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1345              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1346              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1347              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1348              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1349              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
1350             jscode, 'Initial JS player signature function name', group='sig')
1351
1352         jsi = JSInterpreter(jscode)
1353         initial_function = jsi.extract_function(funcname)
1354         return lambda s: initial_function([s])
1355
1356     def _parse_sig_swf(self, file_contents):
1357         swfi = SWFInterpreter(file_contents)
1358         TARGET_CLASSNAME = 'SignatureDecipher'
1359         searched_class = swfi.extract_class(TARGET_CLASSNAME)
1360         initial_function = swfi.extract_function(searched_class, 'decipher')
1361         return lambda s: initial_function([s])
1362
1363     def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
1364         """Turn the encrypted s field into a working signature"""
1365
1366         if player_url is None:
1367             raise ExtractorError('Cannot decrypt signature without player_url')
1368
1369         if player_url.startswith('//'):
1370             player_url = 'https:' + player_url
1371         elif not re.match(r'https?://', player_url):
1372             player_url = compat_urlparse.urljoin(
1373                 'https://www.youtube.com', player_url)
1374         try:
1375             player_id = (player_url, self._signature_cache_id(s))
1376             if player_id not in self._player_cache:
1377                 func = self._extract_signature_function(
1378                     video_id, player_url, s
1379                 )
1380                 self._player_cache[player_id] = func
1381             func = self._player_cache[player_id]
1382             if self._downloader.params.get('youtube_print_sig_code'):
1383                 self._print_sig_code(func, s)
1384             return func(s)
1385         except Exception as e:
1386             tb = traceback.format_exc()
1387             raise ExtractorError(
1388                 'Signature extraction failed: ' + tb, cause=e)
1389
1390     def _get_subtitles(self, video_id, webpage):
1391         try:
1392             subs_doc = self._download_xml(
1393                 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
1394                 video_id, note=False)
1395         except ExtractorError as err:
1396             self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
1397             return {}
1398
1399         sub_lang_list = {}
1400         for track in subs_doc.findall('track'):
1401             lang = track.attrib['lang_code']
1402             if lang in sub_lang_list:
1403                 continue
1404             sub_formats = []
1405             for ext in self._SUBTITLE_FORMATS:
1406                 params = compat_urllib_parse_urlencode({
1407                     'lang': lang,
1408                     'v': video_id,
1409                     'fmt': ext,
1410                     'name': track.attrib['name'].encode('utf-8'),
1411                 })
1412                 sub_formats.append({
1413                     'url': 'https://www.youtube.com/api/timedtext?' + params,
1414                     'ext': ext,
1415                 })
1416             sub_lang_list[lang] = sub_formats
1417         if not sub_lang_list:
1418             self._downloader.report_warning('video doesn\'t have subtitles')
1419             return {}
1420         return sub_lang_list
1421
1422     def _get_ytplayer_config(self, video_id, webpage):
1423         patterns = (
1424             # User data may contain arbitrary character sequences that may affect
1425             # JSON extraction with regex, e.g. when '};' is contained the second
1426             # regex won't capture the whole JSON. Yet working around by trying more
1427             # concrete regex first keeping in mind proper quoted string handling
1428             # to be implemented in future that will replace this workaround (see
1429             # https://github.com/ytdl-org/youtube-dl/issues/7468,
1430             # https://github.com/ytdl-org/youtube-dl/pull/7599)
1431             r';ytplayer\.config\s*=\s*({.+?});ytplayer',
1432             r';ytplayer\.config\s*=\s*({.+?});',
1433         )
1434         config = self._search_regex(
1435             patterns, webpage, 'ytplayer.config', default=None)
1436         if config:
1437             return self._parse_json(
1438                 uppercase_escape(config), video_id, fatal=False)
1439
1440     def _get_automatic_captions(self, video_id, webpage):
1441         """We need the webpage for getting the captions url, pass it as an
1442            argument to speed up the process."""
1443         self.to_screen('%s: Looking for automatic captions' % video_id)
1444         player_config = self._get_ytplayer_config(video_id, webpage)
1445         err_msg = 'Couldn\'t find automatic captions for %s' % video_id
1446         if not player_config:
1447             self._downloader.report_warning(err_msg)
1448             return {}
1449         try:
1450             args = player_config['args']
1451             caption_url = args.get('ttsurl')
1452             if caption_url:
1453                 timestamp = args['timestamp']
1454                 # We get the available subtitles
1455                 list_params = compat_urllib_parse_urlencode({
1456                     'type': 'list',
1457                     'tlangs': 1,
1458                     'asrs': 1,
1459                 })
1460                 list_url = caption_url + '&' + list_params
1461                 caption_list = self._download_xml(list_url, video_id)
1462                 original_lang_node = caption_list.find('track')
1463                 if original_lang_node is None:
1464                     self._downloader.report_warning('Video doesn\'t have automatic captions')
1465                     return {}
1466                 original_lang = original_lang_node.attrib['lang_code']
1467                 caption_kind = original_lang_node.attrib.get('kind', '')
1468
1469                 sub_lang_list = {}
1470                 for lang_node in caption_list.findall('target'):
1471                     sub_lang = lang_node.attrib['lang_code']
1472                     sub_formats = []
1473                     for ext in self._SUBTITLE_FORMATS:
1474                         params = compat_urllib_parse_urlencode({
1475                             'lang': original_lang,
1476                             'tlang': sub_lang,
1477                             'fmt': ext,
1478                             'ts': timestamp,
1479                             'kind': caption_kind,
1480                         })
1481                         sub_formats.append({
1482                             'url': caption_url + '&' + params,
1483                             'ext': ext,
1484                         })
1485                     sub_lang_list[sub_lang] = sub_formats
1486                 return sub_lang_list
1487
1488             def make_captions(sub_url, sub_langs):
1489                 parsed_sub_url = compat_urllib_parse_urlparse(sub_url)
1490                 caption_qs = compat_parse_qs(parsed_sub_url.query)
1491                 captions = {}
1492                 for sub_lang in sub_langs:
1493                     sub_formats = []
1494                     for ext in self._SUBTITLE_FORMATS:
1495                         caption_qs.update({
1496                             'tlang': [sub_lang],
1497                             'fmt': [ext],
1498                         })
1499                         sub_url = compat_urlparse.urlunparse(parsed_sub_url._replace(
1500                             query=compat_urllib_parse_urlencode(caption_qs, True)))
1501                         sub_formats.append({
1502                             'url': sub_url,
1503                             'ext': ext,
1504                         })
1505                     captions[sub_lang] = sub_formats
1506                 return captions
1507
1508             # New captions format as of 22.06.2017
1509             player_response = args.get('player_response')
1510             if player_response and isinstance(player_response, compat_str):
1511                 player_response = self._parse_json(
1512                     player_response, video_id, fatal=False)
1513                 if player_response:
1514                     renderer = player_response['captions']['playerCaptionsTracklistRenderer']
1515                     base_url = renderer['captionTracks'][0]['baseUrl']
1516                     sub_lang_list = []
1517                     for lang in renderer['translationLanguages']:
1518                         lang_code = lang.get('languageCode')
1519                         if lang_code:
1520                             sub_lang_list.append(lang_code)
1521                     return make_captions(base_url, sub_lang_list)
1522
1523             # Some videos don't provide ttsurl but rather caption_tracks and
1524             # caption_translation_languages (e.g. 20LmZk1hakA)
1525             # Does not used anymore as of 22.06.2017
1526             caption_tracks = args['caption_tracks']
1527             caption_translation_languages = args['caption_translation_languages']
1528             caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
1529             sub_lang_list = []
1530             for lang in caption_translation_languages.split(','):
1531                 lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
1532                 sub_lang = lang_qs.get('lc', [None])[0]
1533                 if sub_lang:
1534                     sub_lang_list.append(sub_lang)
1535             return make_captions(caption_url, sub_lang_list)
1536         # An extractor error can be raise by the download process if there are
1537         # no automatic captions but there are subtitles
1538         except (KeyError, IndexError, ExtractorError):
1539             self._downloader.report_warning(err_msg)
1540             return {}
1541
1542     def _mark_watched(self, video_id, video_info, player_response):
1543         playback_url = url_or_none(try_get(
1544             player_response,
1545             lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl']) or try_get(
1546             video_info, lambda x: x['videostats_playback_base_url'][0]))
1547         if not playback_url:
1548             return
1549         parsed_playback_url = compat_urlparse.urlparse(playback_url)
1550         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
1551
1552         # cpn generation algorithm is reverse engineered from base.js.
1553         # In fact it works even with dummy cpn.
1554         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
1555         cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
1556
1557         qs.update({
1558             'ver': ['2'],
1559             'cpn': [cpn],
1560         })
1561         playback_url = compat_urlparse.urlunparse(
1562             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
1563
1564         self._download_webpage(
1565             playback_url, video_id, 'Marking watched',
1566             'Unable to mark watched', fatal=False)
1567
1568     @staticmethod
1569     def _extract_urls(webpage):
1570         # Embedded YouTube player
1571         entries = [
1572             unescapeHTML(mobj.group('url'))
1573             for mobj in re.finditer(r'''(?x)
1574             (?:
1575                 <iframe[^>]+?src=|
1576                 data-video-url=|
1577                 <embed[^>]+?src=|
1578                 embedSWF\(?:\s*|
1579                 <object[^>]+data=|
1580                 new\s+SWFObject\(
1581             )
1582             (["\'])
1583                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1584                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1585             \1''', webpage)]
1586
1587         # lazyYT YouTube embed
1588         entries.extend(list(map(
1589             unescapeHTML,
1590             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
1591
1592         # Wordpress "YouTube Video Importer" plugin
1593         matches = re.findall(r'''(?x)<div[^>]+
1594             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
1595             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
1596         entries.extend(m[-1] for m in matches)
1597
1598         return entries
1599
1600     @staticmethod
1601     def _extract_url(webpage):
1602         urls = YoutubeIE._extract_urls(webpage)
1603         return urls[0] if urls else None
1604
1605     @classmethod
1606     def extract_id(cls, url):
1607         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
1608         if mobj is None:
1609             raise ExtractorError('Invalid URL: %s' % url)
1610         video_id = mobj.group(2)
1611         return video_id
1612
1613     @staticmethod
1614     def _extract_chapters(description, duration):
1615         if not description:
1616             return None
1617         chapter_lines = re.findall(
1618             r'(?:^|<br\s*/>)([^<]*<a[^>]+onclick=["\']yt\.www\.watch\.player\.seekTo[^>]+>(\d{1,2}:\d{1,2}(?::\d{1,2})?)</a>[^>]*)(?=$|<br\s*/>)',
1619             description)
1620         if not chapter_lines:
1621             return None
1622         chapters = []
1623         for next_num, (chapter_line, time_point) in enumerate(
1624                 chapter_lines, start=1):
1625             start_time = parse_duration(time_point)
1626             if start_time is None:
1627                 continue
1628             if start_time > duration:
1629                 break
1630             end_time = (duration if next_num == len(chapter_lines)
1631                         else parse_duration(chapter_lines[next_num][1]))
1632             if end_time is None:
1633                 continue
1634             if end_time > duration:
1635                 end_time = duration
1636             if start_time > end_time:
1637                 break
1638             chapter_title = re.sub(
1639                 r'<a[^>]+>[^<]+</a>', '', chapter_line).strip(' \t-')
1640             chapter_title = re.sub(r'\s+', ' ', chapter_title)
1641             chapters.append({
1642                 'start_time': start_time,
1643                 'end_time': end_time,
1644                 'title': chapter_title,
1645             })
1646         return chapters
1647
1648     def _real_extract(self, url):
1649         url, smuggled_data = unsmuggle_url(url, {})
1650
1651         proto = (
1652             'http' if self._downloader.params.get('prefer_insecure', False)
1653             else 'https')
1654
1655         start_time = None
1656         end_time = None
1657         parsed_url = compat_urllib_parse_urlparse(url)
1658         for component in [parsed_url.fragment, parsed_url.query]:
1659             query = compat_parse_qs(component)
1660             if start_time is None and 't' in query:
1661                 start_time = parse_duration(query['t'][0])
1662             if start_time is None and 'start' in query:
1663                 start_time = parse_duration(query['start'][0])
1664             if end_time is None and 'end' in query:
1665                 end_time = parse_duration(query['end'][0])
1666
1667         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1668         mobj = re.search(self._NEXT_URL_RE, url)
1669         if mobj:
1670             url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
1671         video_id = self.extract_id(url)
1672
1673         # Get video webpage
1674         url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
1675         video_webpage = self._download_webpage(url, video_id)
1676
1677         # Attempt to extract SWF player URL
1678         mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1679         if mobj is not None:
1680             player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1681         else:
1682             player_url = None
1683
1684         dash_mpds = []
1685
1686         def add_dash_mpd(video_info):
1687             dash_mpd = video_info.get('dashmpd')
1688             if dash_mpd and dash_mpd[0] not in dash_mpds:
1689                 dash_mpds.append(dash_mpd[0])
1690
1691         def add_dash_mpd_pr(pl_response):
1692             dash_mpd = url_or_none(try_get(
1693                 pl_response, lambda x: x['streamingData']['dashManifestUrl'],
1694                 compat_str))
1695             if dash_mpd and dash_mpd not in dash_mpds:
1696                 dash_mpds.append(dash_mpd)
1697
1698         is_live = None
1699         view_count = None
1700
1701         def extract_view_count(v_info):
1702             return int_or_none(try_get(v_info, lambda x: x['view_count'][0]))
1703
1704         def extract_token(v_info):
1705             return dict_get(v_info, ('account_playback_token', 'accountPlaybackToken', 'token'))
1706
1707         def extract_player_response(player_response, video_id):
1708             pl_response = str_or_none(player_response)
1709             if not pl_response:
1710                 return
1711             pl_response = self._parse_json(pl_response, video_id, fatal=False)
1712             if isinstance(pl_response, dict):
1713                 add_dash_mpd_pr(pl_response)
1714                 return pl_response
1715
1716         player_response = {}
1717
1718         # Get video info
1719         embed_webpage = None
1720         if re.search(r'player-age-gate-content">', video_webpage) is not None:
1721             age_gate = True
1722             # We simulate the access to the video from www.youtube.com/v/{video_id}
1723             # this can be viewed without login into Youtube
1724             url = proto + '://www.youtube.com/embed/%s' % video_id
1725             embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
1726             data = compat_urllib_parse_urlencode({
1727                 'video_id': video_id,
1728                 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1729                 'sts': self._search_regex(
1730                     r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
1731             })
1732             video_info_url = proto + '://www.youtube.com/get_video_info?' + data
1733             video_info_webpage = self._download_webpage(
1734                 video_info_url, video_id,
1735                 note='Refetching age-gated info webpage',
1736                 errnote='unable to download video info webpage')
1737             video_info = compat_parse_qs(video_info_webpage)
1738             pl_response = video_info.get('player_response', [None])[0]
1739             player_response = extract_player_response(pl_response, video_id)
1740             add_dash_mpd(video_info)
1741             view_count = extract_view_count(video_info)
1742         else:
1743             age_gate = False
1744             video_info = None
1745             sts = None
1746             # Try looking directly into the video webpage
1747             ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
1748             if ytplayer_config:
1749                 args = ytplayer_config['args']
1750                 if args.get('url_encoded_fmt_stream_map') or args.get('hlsvp'):
1751                     # Convert to the same format returned by compat_parse_qs
1752                     video_info = dict((k, [v]) for k, v in args.items())
1753                     add_dash_mpd(video_info)
1754                 # Rental video is not rented but preview is available (e.g.
1755                 # https://www.youtube.com/watch?v=yYr8q0y5Jfg,
1756                 # https://github.com/ytdl-org/youtube-dl/issues/10532)
1757                 if not video_info and args.get('ypc_vid'):
1758                     return self.url_result(
1759                         args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
1760                 if args.get('livestream') == '1' or args.get('live_playback') == 1:
1761                     is_live = True
1762                 sts = ytplayer_config.get('sts')
1763                 if not player_response:
1764                     player_response = extract_player_response(args.get('player_response'), video_id)
1765             if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
1766                 add_dash_mpd_pr(player_response)
1767                 # We also try looking in get_video_info since it may contain different dashmpd
1768                 # URL that points to a DASH manifest with possibly different itag set (some itags
1769                 # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
1770                 # manifest pointed by get_video_info's dashmpd).
1771                 # The general idea is to take a union of itags of both DASH manifests (for example
1772                 # video with such 'manifest behavior' see https://github.com/ytdl-org/youtube-dl/issues/6093)
1773                 self.report_video_info_webpage_download(video_id)
1774                 for el in ('embedded', 'detailpage', 'vevo', ''):
1775                     query = {
1776                         'video_id': video_id,
1777                         'ps': 'default',
1778                         'eurl': '',
1779                         'gl': 'US',
1780                         'hl': 'en',
1781                     }
1782                     if el:
1783                         query['el'] = el
1784                     if sts:
1785                         query['sts'] = sts
1786                     video_info_webpage = self._download_webpage(
1787                         '%s://www.youtube.com/get_video_info' % proto,
1788                         video_id, note=False,
1789                         errnote='unable to download video info webpage',
1790                         fatal=False, query=query)
1791                     if not video_info_webpage:
1792                         continue
1793                     get_video_info = compat_parse_qs(video_info_webpage)
1794                     if not player_response:
1795                         pl_response = get_video_info.get('player_response', [None])[0]
1796                         player_response = extract_player_response(pl_response, video_id)
1797                     add_dash_mpd(get_video_info)
1798                     if view_count is None:
1799                         view_count = extract_view_count(get_video_info)
1800                     if not video_info:
1801                         video_info = get_video_info
1802                     get_token = extract_token(get_video_info)
1803                     if get_token:
1804                         # Different get_video_info requests may report different results, e.g.
1805                         # some may report video unavailability, but some may serve it without
1806                         # any complaint (see https://github.com/ytdl-org/youtube-dl/issues/7362,
1807                         # the original webpage as well as el=info and el=embedded get_video_info
1808                         # requests report video unavailability due to geo restriction while
1809                         # el=detailpage succeeds and returns valid data). This is probably
1810                         # due to YouTube measures against IP ranges of hosting providers.
1811                         # Working around by preferring the first succeeded video_info containing
1812                         # the token if no such video_info yet was found.
1813                         token = extract_token(video_info)
1814                         if not token:
1815                             video_info = get_video_info
1816                         break
1817
1818         def extract_unavailable_message():
1819             messages = []
1820             for tag, kind in (('h1', 'message'), ('div', 'submessage')):
1821                 msg = self._html_search_regex(
1822                     r'(?s)<{tag}[^>]+id=["\']unavailable-{kind}["\'][^>]*>(.+?)</{tag}>'.format(tag=tag, kind=kind),
1823                     video_webpage, 'unavailable %s' % kind, default=None)
1824                 if msg:
1825                     messages.append(msg)
1826             if messages:
1827                 return '\n'.join(messages)
1828
1829         if not video_info:
1830             unavailable_message = extract_unavailable_message()
1831             if not unavailable_message:
1832                 unavailable_message = 'Unable to extract video data'
1833             raise ExtractorError(
1834                 'YouTube said: %s' % unavailable_message, expected=True, video_id=video_id)
1835
1836         video_details = try_get(
1837             player_response, lambda x: x['videoDetails'], dict) or {}
1838
1839         video_title = video_info.get('title', [None])[0] or video_details.get('title')
1840         if not video_title:
1841             self._downloader.report_warning('Unable to extract video title')
1842             video_title = '_'
1843
1844         description_original = video_description = get_element_by_id("eow-description", video_webpage)
1845         if video_description:
1846
1847             def replace_url(m):
1848                 redir_url = compat_urlparse.urljoin(url, m.group(1))
1849                 parsed_redir_url = compat_urllib_parse_urlparse(redir_url)
1850                 if re.search(r'^(?:www\.)?(?:youtube(?:-nocookie)?\.com|youtu\.be)$', parsed_redir_url.netloc) and parsed_redir_url.path == '/redirect':
1851                     qs = compat_parse_qs(parsed_redir_url.query)
1852                     q = qs.get('q')
1853                     if q and q[0]:
1854                         return q[0]
1855                 return redir_url
1856
1857             description_original = video_description = re.sub(r'''(?x)
1858                 <a\s+
1859                     (?:[a-zA-Z-]+="[^"]*"\s+)*?
1860                     (?:title|href)="([^"]+)"\s+
1861                     (?:[a-zA-Z-]+="[^"]*"\s+)*?
1862                     class="[^"]*"[^>]*>
1863                 [^<]+\.{3}\s*
1864                 </a>
1865             ''', replace_url, video_description)
1866             video_description = clean_html(video_description)
1867         else:
1868             video_description = self._html_search_meta('description', video_webpage) or video_details.get('shortDescription')
1869
1870         if not smuggled_data.get('force_singlefeed', False):
1871             if not self._downloader.params.get('noplaylist'):
1872                 multifeed_metadata_list = try_get(
1873                     player_response,
1874                     lambda x: x['multicamera']['playerLegacyMulticameraRenderer']['metadataList'],
1875                     compat_str) or try_get(
1876                     video_info, lambda x: x['multifeed_metadata_list'][0], compat_str)
1877                 if multifeed_metadata_list:
1878                     entries = []
1879                     feed_ids = []
1880                     for feed in multifeed_metadata_list.split(','):
1881                         # Unquote should take place before split on comma (,) since textual
1882                         # fields may contain comma as well (see
1883                         # https://github.com/ytdl-org/youtube-dl/issues/8536)
1884                         feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
1885                         entries.append({
1886                             '_type': 'url_transparent',
1887                             'ie_key': 'Youtube',
1888                             'url': smuggle_url(
1889                                 '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
1890                                 {'force_singlefeed': True}),
1891                             'title': '%s (%s)' % (video_title, feed_data['title'][0]),
1892                         })
1893                         feed_ids.append(feed_data['id'][0])
1894                     self.to_screen(
1895                         'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
1896                         % (', '.join(feed_ids), video_id))
1897                     return self.playlist_result(entries, video_id, video_title, video_description)
1898             else:
1899                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1900
1901         if view_count is None:
1902             view_count = extract_view_count(video_info)
1903         if view_count is None and video_details:
1904             view_count = int_or_none(video_details.get('viewCount'))
1905
1906         if is_live is None:
1907             is_live = bool_or_none(video_details.get('isLive'))
1908
1909         # Check for "rental" videos
1910         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1911             raise ExtractorError('"rental" videos not supported. See https://github.com/ytdl-org/youtube-dl/issues/359 for more information.', expected=True)
1912
1913         def _extract_filesize(media_url):
1914             return int_or_none(self._search_regex(
1915                 r'\bclen[=/](\d+)', media_url, 'filesize', default=None))
1916
1917         streaming_formats = try_get(player_response, lambda x: x['streamingData']['formats'], list) or []
1918         streaming_formats.extend(try_get(player_response, lambda x: x['streamingData']['adaptiveFormats'], list) or [])
1919
1920         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1921             self.report_rtmp_download()
1922             formats = [{
1923                 'format_id': '_rtmp',
1924                 'protocol': 'rtmp',
1925                 'url': video_info['conn'][0],
1926                 'player_url': player_url,
1927             }]
1928         elif not is_live and (streaming_formats or len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1):
1929             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
1930             if 'rtmpe%3Dyes' in encoded_url_map:
1931                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/ytdl-org/youtube-dl/issues/343 for more information.', expected=True)
1932             formats = []
1933             formats_spec = {}
1934             fmt_list = video_info.get('fmt_list', [''])[0]
1935             if fmt_list:
1936                 for fmt in fmt_list.split(','):
1937                     spec = fmt.split('/')
1938                     if len(spec) > 1:
1939                         width_height = spec[1].split('x')
1940                         if len(width_height) == 2:
1941                             formats_spec[spec[0]] = {
1942                                 'resolution': spec[1],
1943                                 'width': int_or_none(width_height[0]),
1944                                 'height': int_or_none(width_height[1]),
1945                             }
1946             for fmt in streaming_formats:
1947                 itag = str_or_none(fmt.get('itag'))
1948                 if not itag:
1949                     continue
1950                 quality = fmt.get('quality')
1951                 quality_label = fmt.get('qualityLabel') or quality
1952                 formats_spec[itag] = {
1953                     'asr': int_or_none(fmt.get('audioSampleRate')),
1954                     'filesize': int_or_none(fmt.get('contentLength')),
1955                     'format_note': quality_label,
1956                     'fps': int_or_none(fmt.get('fps')),
1957                     'height': int_or_none(fmt.get('height')),
1958                     # bitrate for itag 43 is always 2147483647
1959                     'tbr': float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000) if itag != '43' else None,
1960                     'width': int_or_none(fmt.get('width')),
1961                 }
1962
1963             for fmt in streaming_formats:
1964                 if fmt.get('drm_families'):
1965                     continue
1966                 url = url_or_none(fmt.get('url'))
1967
1968                 if not url:
1969                     cipher = fmt.get('cipher')
1970                     if not cipher:
1971                         continue
1972                     url_data = compat_parse_qs(cipher)
1973                     url = url_or_none(try_get(url_data, lambda x: x['url'][0], compat_str))
1974                     if not url:
1975                         continue
1976                 else:
1977                     cipher = None
1978                     url_data = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
1979
1980                 stream_type = int_or_none(try_get(url_data, lambda x: x['stream_type'][0]))
1981                 # Unsupported FORMAT_STREAM_TYPE_OTF
1982                 if stream_type == 3:
1983                     continue
1984
1985                 format_id = fmt.get('itag') or url_data['itag'][0]
1986                 if not format_id:
1987                     continue
1988                 format_id = compat_str(format_id)
1989
1990                 if cipher:
1991                     if 's' in url_data or self._downloader.params.get('youtube_include_dash_manifest', True):
1992                         ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
1993                         jsplayer_url_json = self._search_regex(
1994                             ASSETS_RE,
1995                             embed_webpage if age_gate else video_webpage,
1996                             'JS player URL (1)', default=None)
1997                         if not jsplayer_url_json and not age_gate:
1998                             # We need the embed website after all
1999                             if embed_webpage is None:
2000                                 embed_url = proto + '://www.youtube.com/embed/%s' % video_id
2001                                 embed_webpage = self._download_webpage(
2002                                     embed_url, video_id, 'Downloading embed webpage')
2003                             jsplayer_url_json = self._search_regex(
2004                                 ASSETS_RE, embed_webpage, 'JS player URL')
2005
2006                         player_url = json.loads(jsplayer_url_json)
2007                         if player_url is None:
2008                             player_url_json = self._search_regex(
2009                                 r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
2010                                 video_webpage, 'age gate player URL')
2011                             player_url = json.loads(player_url_json)
2012
2013                     if 'sig' in url_data:
2014                         url += '&signature=' + url_data['sig'][0]
2015                     elif 's' in url_data:
2016                         encrypted_sig = url_data['s'][0]
2017
2018                         if self._downloader.params.get('verbose'):
2019                             if player_url is None:
2020                                 player_version = 'unknown'
2021                                 player_desc = 'unknown'
2022                             else:
2023                                 if player_url.endswith('swf'):
2024                                     player_version = self._search_regex(
2025                                         r'-(.+?)(?:/watch_as3)?\.swf$', player_url,
2026                                         'flash player', fatal=False)
2027                                     player_desc = 'flash player %s' % player_version
2028                                 else:
2029                                     player_version = self._search_regex(
2030                                         [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js',
2031                                          r'(?:www|player(?:_ias)?)-([^/]+)(?:/[a-z]{2,3}_[A-Z]{2})?/base\.js'],
2032                                         player_url,
2033                                         'html5 player', fatal=False)
2034                                     player_desc = 'html5 player %s' % player_version
2035
2036                             parts_sizes = self._signature_cache_id(encrypted_sig)
2037                             self.to_screen('{%s} signature length %s, %s' %
2038                                            (format_id, parts_sizes, player_desc))
2039
2040                         signature = self._decrypt_signature(
2041                             encrypted_sig, video_id, player_url, age_gate)
2042                         sp = try_get(url_data, lambda x: x['sp'][0], compat_str) or 'signature'
2043                         url += '&%s=%s' % (sp, signature)
2044                 if 'ratebypass' not in url:
2045                     url += '&ratebypass=yes'
2046
2047                 dct = {
2048                     'format_id': format_id,
2049                     'url': url,
2050                     'player_url': player_url,
2051                 }
2052                 if format_id in self._formats:
2053                     dct.update(self._formats[format_id])
2054                 if format_id in formats_spec:
2055                     dct.update(formats_spec[format_id])
2056
2057                 # Some itags are not included in DASH manifest thus corresponding formats will
2058                 # lack metadata (see https://github.com/ytdl-org/youtube-dl/pull/5993).
2059                 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
2060                 mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
2061                 width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
2062
2063                 if width is None:
2064                     width = int_or_none(fmt.get('width'))
2065                 if height is None:
2066                     height = int_or_none(fmt.get('height'))
2067
2068                 filesize = int_or_none(url_data.get(
2069                     'clen', [None])[0]) or _extract_filesize(url)
2070
2071                 quality = url_data.get('quality', [None])[0] or fmt.get('quality')
2072                 quality_label = url_data.get('quality_label', [None])[0] or fmt.get('qualityLabel')
2073
2074                 tbr = (float_or_none(url_data.get('bitrate', [None])[0], 1000)
2075                        or float_or_none(fmt.get('bitrate'), 1000)) if format_id != '43' else None
2076                 fps = int_or_none(url_data.get('fps', [None])[0]) or int_or_none(fmt.get('fps'))
2077
2078                 more_fields = {
2079                     'filesize': filesize,
2080                     'tbr': tbr,
2081                     'width': width,
2082                     'height': height,
2083                     'fps': fps,
2084                     'format_note': quality_label or quality,
2085                 }
2086                 for key, value in more_fields.items():
2087                     if value:
2088                         dct[key] = value
2089                 type_ = url_data.get('type', [None])[0] or fmt.get('mimeType')
2090                 if type_:
2091                     type_split = type_.split(';')
2092                     kind_ext = type_split[0].split('/')
2093                     if len(kind_ext) == 2:
2094                         kind, _ = kind_ext
2095                         dct['ext'] = mimetype2ext(type_split[0])
2096                         if kind in ('audio', 'video'):
2097                             codecs = None
2098                             for mobj in re.finditer(
2099                                     r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
2100                                 if mobj.group('key') == 'codecs':
2101                                     codecs = mobj.group('val')
2102                                     break
2103                             if codecs:
2104                                 dct.update(parse_codecs(codecs))
2105                 if dct.get('acodec') == 'none' or dct.get('vcodec') == 'none':
2106                     dct['downloader_options'] = {
2107                         # Youtube throttles chunks >~10M
2108                         'http_chunk_size': 10485760,
2109                     }
2110                 formats.append(dct)
2111         else:
2112             manifest_url = (
2113                 url_or_none(try_get(
2114                     player_response,
2115                     lambda x: x['streamingData']['hlsManifestUrl'],
2116                     compat_str))
2117                 or url_or_none(try_get(
2118                     video_info, lambda x: x['hlsvp'][0], compat_str)))
2119             if manifest_url:
2120                 formats = []
2121                 m3u8_formats = self._extract_m3u8_formats(
2122                     manifest_url, video_id, 'mp4', fatal=False)
2123                 for a_format in m3u8_formats:
2124                     itag = self._search_regex(
2125                         r'/itag/(\d+)/', a_format['url'], 'itag', default=None)
2126                     if itag:
2127                         a_format['format_id'] = itag
2128                         if itag in self._formats:
2129                             dct = self._formats[itag].copy()
2130                             dct.update(a_format)
2131                             a_format = dct
2132                     a_format['player_url'] = player_url
2133                     # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
2134                     a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
2135                     formats.append(a_format)
2136             else:
2137                 error_message = extract_unavailable_message()
2138                 if not error_message:
2139                     error_message = clean_html(try_get(
2140                         player_response, lambda x: x['playabilityStatus']['reason'],
2141                         compat_str))
2142                 if not error_message:
2143                     error_message = clean_html(
2144                         try_get(video_info, lambda x: x['reason'][0], compat_str))
2145                 if error_message:
2146                     raise ExtractorError(error_message, expected=True)
2147                 raise ExtractorError('no conn, hlsvp, hlsManifestUrl or url_encoded_fmt_stream_map information found in video info')
2148
2149         # uploader
2150         video_uploader = try_get(
2151             video_info, lambda x: x['author'][0],
2152             compat_str) or str_or_none(video_details.get('author'))
2153         if video_uploader:
2154             video_uploader = compat_urllib_parse_unquote_plus(video_uploader)
2155         else:
2156             self._downloader.report_warning('unable to extract uploader name')
2157
2158         # uploader_id
2159         video_uploader_id = None
2160         video_uploader_url = None
2161         mobj = re.search(
2162             r'<link itemprop="url" href="(?P<uploader_url>https?://www\.youtube\.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
2163             video_webpage)
2164         if mobj is not None:
2165             video_uploader_id = mobj.group('uploader_id')
2166             video_uploader_url = mobj.group('uploader_url')
2167         else:
2168             self._downloader.report_warning('unable to extract uploader nickname')
2169
2170         channel_id = (
2171             str_or_none(video_details.get('channelId'))
2172             or self._html_search_meta(
2173                 'channelId', video_webpage, 'channel id', default=None)
2174             or self._search_regex(
2175                 r'data-channel-external-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
2176                 video_webpage, 'channel id', default=None, group='id'))
2177         channel_url = 'http://www.youtube.com/channel/%s' % channel_id if channel_id else None
2178
2179         # thumbnail image
2180         # We try first to get a high quality image:
2181         m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
2182                             video_webpage, re.DOTALL)
2183         if m_thumb is not None:
2184             video_thumbnail = m_thumb.group(1)
2185         elif 'thumbnail_url' not in video_info:
2186             self._downloader.report_warning('unable to extract video thumbnail')
2187             video_thumbnail = None
2188         else:   # don't panic if we can't find it
2189             video_thumbnail = compat_urllib_parse_unquote_plus(video_info['thumbnail_url'][0])
2190
2191         # upload date
2192         upload_date = self._html_search_meta(
2193             'datePublished', video_webpage, 'upload date', default=None)
2194         if not upload_date:
2195             upload_date = self._search_regex(
2196                 [r'(?s)id="eow-date.*?>(.*?)</span>',
2197                  r'(?:id="watch-uploader-info".*?>.*?|["\']simpleText["\']\s*:\s*["\'])(?:Published|Uploaded|Streamed live|Started) on (.+?)[<"\']'],
2198                 video_webpage, 'upload date', default=None)
2199         upload_date = unified_strdate(upload_date)
2200
2201         video_license = self._html_search_regex(
2202             r'<h4[^>]+class="title"[^>]*>\s*License\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li',
2203             video_webpage, 'license', default=None)
2204
2205         m_music = re.search(
2206             r'''(?x)
2207                 <h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*
2208                 <ul[^>]*>\s*
2209                 <li>(?P<title>.+?)
2210                 by (?P<creator>.+?)
2211                 (?:
2212                     \(.+?\)|
2213                     <a[^>]*
2214                         (?:
2215                             \bhref=["\']/red[^>]*>|             # drop possible
2216                             >\s*Listen ad-free with YouTube Red # YouTube Red ad
2217                         )
2218                     .*?
2219                 )?</li
2220             ''',
2221             video_webpage)
2222         if m_music:
2223             video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))
2224             video_creator = clean_html(m_music.group('creator'))
2225         else:
2226             video_alt_title = video_creator = None
2227
2228         def extract_meta(field):
2229             return self._html_search_regex(
2230                 r'<h4[^>]+class="title"[^>]*>\s*%s\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li>\s*' % field,
2231                 video_webpage, field, default=None)
2232
2233         track = extract_meta('Song')
2234         artist = extract_meta('Artist')
2235         album = extract_meta('Album')
2236
2237         # Youtube Music Auto-generated description
2238         release_date = release_year = None
2239         if video_description:
2240             mobj = re.search(r'(?s)Provided to YouTube by [^\n]+\n+(?P<track>[^·]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?', video_description)
2241             if mobj:
2242                 if not track:
2243                     track = mobj.group('track').strip()
2244                 if not artist:
2245                     artist = mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·'))
2246                 if not album:
2247                     album = mobj.group('album'.strip())
2248                 release_year = mobj.group('release_year')
2249                 release_date = mobj.group('release_date')
2250                 if release_date:
2251                     release_date = release_date.replace('-', '')
2252                     if not release_year:
2253                         release_year = int(release_date[:4])
2254                 if release_year:
2255                     release_year = int(release_year)
2256
2257         m_episode = re.search(
2258             r'<div[^>]+id="watch7-headline"[^>]*>\s*<span[^>]*>.*?>(?P<series>[^<]+)</a></b>\s*S(?P<season>\d+)\s*•\s*E(?P<episode>\d+)</span>',
2259             video_webpage)
2260         if m_episode:
2261             series = unescapeHTML(m_episode.group('series'))
2262             season_number = int(m_episode.group('season'))
2263             episode_number = int(m_episode.group('episode'))
2264         else:
2265             series = season_number = episode_number = None
2266
2267         m_cat_container = self._search_regex(
2268             r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
2269             video_webpage, 'categories', default=None)
2270         if m_cat_container:
2271             category = self._html_search_regex(
2272                 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
2273                 default=None)
2274             video_categories = None if category is None else [category]
2275         else:
2276             video_categories = None
2277
2278         video_tags = [
2279             unescapeHTML(m.group('content'))
2280             for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
2281
2282         def _extract_count(count_name):
2283             return str_to_int(self._search_regex(
2284                 r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
2285                 % re.escape(count_name),
2286                 video_webpage, count_name, default=None))
2287
2288         like_count = _extract_count('like')
2289         dislike_count = _extract_count('dislike')
2290
2291         if view_count is None:
2292             view_count = str_to_int(self._search_regex(
2293                 r'<[^>]+class=["\']watch-view-count[^>]+>\s*([\d,\s]+)', video_webpage,
2294                 'view count', default=None))
2295
2296         average_rating = (
2297             float_or_none(video_details.get('averageRating'))
2298             or try_get(video_info, lambda x: float_or_none(x['avg_rating'][0])))
2299
2300         # subtitles
2301         video_subtitles = self.extract_subtitles(video_id, video_webpage)
2302         automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
2303
2304         video_duration = try_get(
2305             video_info, lambda x: int_or_none(x['length_seconds'][0]))
2306         if not video_duration:
2307             video_duration = int_or_none(video_details.get('lengthSeconds'))
2308         if not video_duration:
2309             video_duration = parse_duration(self._html_search_meta(
2310                 'duration', video_webpage, 'video duration'))
2311
2312         # annotations
2313         video_annotations = None
2314         if self._downloader.params.get('writeannotations', False):
2315             xsrf_token = self._search_regex(
2316                 r'([\'"])XSRF_TOKEN\1\s*:\s*([\'"])(?P<xsrf_token>[A-Za-z0-9+/=]+)\2',
2317                 video_webpage, 'xsrf token', group='xsrf_token', fatal=False)
2318             invideo_url = try_get(
2319                 player_response, lambda x: x['annotations'][0]['playerAnnotationsUrlsRenderer']['invideoUrl'], compat_str)
2320             if xsrf_token and invideo_url:
2321                 xsrf_field_name = self._search_regex(
2322                     r'([\'"])XSRF_FIELD_NAME\1\s*:\s*([\'"])(?P<xsrf_field_name>\w+)\2',
2323                     video_webpage, 'xsrf field name',
2324                     group='xsrf_field_name', default='session_token')
2325                 video_annotations = self._download_webpage(
2326                     self._proto_relative_url(invideo_url),
2327                     video_id, note='Downloading annotations',
2328                     errnote='Unable to download video annotations', fatal=False,
2329                     data=urlencode_postdata({xsrf_field_name: xsrf_token}))
2330
2331         chapters = self._extract_chapters(description_original, video_duration)
2332
2333         # Look for the DASH manifest
2334         if self._downloader.params.get('youtube_include_dash_manifest', True):
2335             dash_mpd_fatal = True
2336             for mpd_url in dash_mpds:
2337                 dash_formats = {}
2338                 try:
2339                     def decrypt_sig(mobj):
2340                         s = mobj.group(1)
2341                         dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
2342                         return '/signature/%s' % dec_s
2343
2344                     mpd_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, mpd_url)
2345
2346                     for df in self._extract_mpd_formats(
2347                             mpd_url, video_id, fatal=dash_mpd_fatal,
2348                             formats_dict=self._formats):
2349                         if not df.get('filesize'):
2350                             df['filesize'] = _extract_filesize(df['url'])
2351                         # Do not overwrite DASH format found in some previous DASH manifest
2352                         if df['format_id'] not in dash_formats:
2353                             dash_formats[df['format_id']] = df
2354                         # Additional DASH manifests may end up in HTTP Error 403 therefore
2355                         # allow them to fail without bug report message if we already have
2356                         # some DASH manifest succeeded. This is temporary workaround to reduce
2357                         # burst of bug reports until we figure out the reason and whether it
2358                         # can be fixed at all.
2359                         dash_mpd_fatal = False
2360                 except (ExtractorError, KeyError) as e:
2361                     self.report_warning(
2362                         'Skipping DASH manifest: %r' % e, video_id)
2363                 if dash_formats:
2364                     # Remove the formats we found through non-DASH, they
2365                     # contain less info and it can be wrong, because we use
2366                     # fixed values (for example the resolution). See
2367                     # https://github.com/ytdl-org/youtube-dl/issues/5774 for an
2368                     # example.
2369                     formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
2370                     formats.extend(dash_formats.values())
2371
2372         # Check for malformed aspect ratio
2373         stretched_m = re.search(
2374             r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
2375             video_webpage)
2376         if stretched_m:
2377             w = float(stretched_m.group('w'))
2378             h = float(stretched_m.group('h'))
2379             # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
2380             # We will only process correct ratios.
2381             if w > 0 and h > 0:
2382                 ratio = w / h
2383                 for f in formats:
2384                     if f.get('vcodec') != 'none':
2385                         f['stretched_ratio'] = ratio
2386
2387         if not formats:
2388             token = extract_token(video_info)
2389             if not token:
2390                 if 'reason' in video_info:
2391                     if 'The uploader has not made this video available in your country.' in video_info['reason']:
2392                         regions_allowed = self._html_search_meta(
2393                             'regionsAllowed', video_webpage, default=None)
2394                         countries = regions_allowed.split(',') if regions_allowed else None
2395                         self.raise_geo_restricted(
2396                             msg=video_info['reason'][0], countries=countries)
2397                     reason = video_info['reason'][0]
2398                     if 'Invalid parameters' in reason:
2399                         unavailable_message = extract_unavailable_message()
2400                         if unavailable_message:
2401                             reason = unavailable_message
2402                     raise ExtractorError(
2403                         'YouTube said: %s' % reason,
2404                         expected=True, video_id=video_id)
2405                 else:
2406                     raise ExtractorError(
2407                         '"token" parameter not in video info for unknown reason',
2408                         video_id=video_id)
2409
2410         if not formats and (video_info.get('license_info') or try_get(player_response, lambda x: x['streamingData']['licenseInfos'])):
2411             raise ExtractorError('This video is DRM protected.', expected=True)
2412
2413         self._sort_formats(formats)
2414
2415         self.mark_watched(video_id, video_info, player_response)
2416
2417         return {
2418             'id': video_id,
2419             'uploader': video_uploader,
2420             'uploader_id': video_uploader_id,
2421             'uploader_url': video_uploader_url,
2422             'channel_id': channel_id,
2423             'channel_url': channel_url,
2424             'upload_date': upload_date,
2425             'license': video_license,
2426             'creator': video_creator or artist,
2427             'title': video_title,
2428             'alt_title': video_alt_title or track,
2429             'thumbnail': video_thumbnail,
2430             'description': video_description,
2431             'categories': video_categories,
2432             'tags': video_tags,
2433             'subtitles': video_subtitles,
2434             'automatic_captions': automatic_captions,
2435             'duration': video_duration,
2436             'age_limit': 18 if age_gate else 0,
2437             'annotations': video_annotations,
2438             'chapters': chapters,
2439             'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
2440             'view_count': view_count,
2441             'like_count': like_count,
2442             'dislike_count': dislike_count,
2443             'average_rating': average_rating,
2444             'formats': formats,
2445             'is_live': is_live,
2446             'start_time': start_time,
2447             'end_time': end_time,
2448             'series': series,
2449             'season_number': season_number,
2450             'episode_number': episode_number,
2451             'track': track,
2452             'artist': artist,
2453             'album': album,
2454             'release_date': release_date,
2455             'release_year': release_year,
2456         }
2457
2458
2459 class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
2460     IE_DESC = 'YouTube.com playlists'
2461     _VALID_URL = r"""(?x)(?:
2462                         (?:https?://)?
2463                         (?:\w+\.)?
2464                         (?:
2465                             (?:
2466                                 youtube\.com|
2467                                 invidio\.us
2468                             )
2469                             /
2470                             (?:
2471                                (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/(?:videoseries|[0-9A-Za-z_-]{11}))
2472                                \? (?:.*?[&;])*? (?:p|a|list)=
2473                             |  p/
2474                             )|
2475                             youtu\.be/[0-9A-Za-z_-]{11}\?.*?\blist=
2476                         )
2477                         (
2478                             (?:PL|LL|EC|UU|FL|RD|UL|TL|OLAK5uy_)?[0-9A-Za-z-_]{10,}
2479                             # Top tracks, they can also include dots
2480                             |(?:MC)[\w\.]*
2481                         )
2482                         .*
2483                      |
2484                         (%(playlist_id)s)
2485                      )""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
2486     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
2487     _VIDEO_RE_TPL = r'href="\s*/watch\?v=%s(?:&amp;(?:[^"]*?index=(?P<index>\d+))?(?:[^>]+>(?P<title>[^<]+))?)?'
2488     _VIDEO_RE = _VIDEO_RE_TPL % r'(?P<id>[0-9A-Za-z_-]{11})'
2489     IE_NAME = 'youtube:playlist'
2490     _TESTS = [{
2491         'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
2492         'info_dict': {
2493             'title': 'ytdl test PL',
2494             'id': 'PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
2495         },
2496         'playlist_count': 3,
2497     }, {
2498         'url': 'https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
2499         'info_dict': {
2500             'id': 'PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
2501             'title': 'YDL_Empty_List',
2502         },
2503         'playlist_count': 0,
2504         'skip': 'This playlist is private',
2505     }, {
2506         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
2507         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2508         'info_dict': {
2509             'title': '29C3: Not my department',
2510             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2511             'uploader': 'Christiaan008',
2512             'uploader_id': 'ChRiStIaAn008',
2513         },
2514         'playlist_count': 95,
2515     }, {
2516         'note': 'issue #673',
2517         'url': 'PLBB231211A4F62143',
2518         'info_dict': {
2519             'title': '[OLD]Team Fortress 2 (Class-based LP)',
2520             'id': 'PLBB231211A4F62143',
2521             'uploader': 'Wickydoo',
2522             'uploader_id': 'Wickydoo',
2523         },
2524         'playlist_mincount': 26,
2525     }, {
2526         'note': 'Large playlist',
2527         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
2528         'info_dict': {
2529             'title': 'Uploads from Cauchemar',
2530             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
2531             'uploader': 'Cauchemar',
2532             'uploader_id': 'Cauchemar89',
2533         },
2534         'playlist_mincount': 799,
2535     }, {
2536         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2537         'info_dict': {
2538             'title': 'YDL_safe_search',
2539             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2540         },
2541         'playlist_count': 2,
2542         'skip': 'This playlist is private',
2543     }, {
2544         'note': 'embedded',
2545         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2546         'playlist_count': 4,
2547         'info_dict': {
2548             'title': 'JODA15',
2549             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2550             'uploader': 'milan',
2551             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
2552         }
2553     }, {
2554         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2555         'playlist_mincount': 485,
2556         'info_dict': {
2557             'title': '2018 Chinese New Singles (11/6 updated)',
2558             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2559             'uploader': 'LBK',
2560             'uploader_id': 'sdragonfang',
2561         }
2562     }, {
2563         'note': 'Embedded SWF player',
2564         'url': 'https://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
2565         'playlist_count': 4,
2566         'info_dict': {
2567             'title': 'JODA7',
2568             'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
2569         },
2570         'skip': 'This playlist does not exist',
2571     }, {
2572         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
2573         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
2574         'info_dict': {
2575             'title': 'Uploads from Interstellar Movie',
2576             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
2577             'uploader': 'Interstellar Movie',
2578             'uploader_id': 'InterstellarMovie1',
2579         },
2580         'playlist_mincount': 21,
2581     }, {
2582         # Playlist URL that does not actually serve a playlist
2583         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
2584         'info_dict': {
2585             'id': 'FqZTN594JQw',
2586             'ext': 'webm',
2587             'title': "Smiley's People 01 detective, Adventure Series, Action",
2588             'uploader': 'STREEM',
2589             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
2590             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
2591             'upload_date': '20150526',
2592             'license': 'Standard YouTube License',
2593             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
2594             'categories': ['People & Blogs'],
2595             'tags': list,
2596             'view_count': int,
2597             'like_count': int,
2598             'dislike_count': int,
2599         },
2600         'params': {
2601             'skip_download': True,
2602         },
2603         'skip': 'This video is not available.',
2604         'add_ie': [YoutubeIE.ie_key()],
2605     }, {
2606         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
2607         'info_dict': {
2608             'id': 'yeWKywCrFtk',
2609             'ext': 'mp4',
2610             'title': 'Small Scale Baler and Braiding Rugs',
2611             'uploader': 'Backus-Page House Museum',
2612             'uploader_id': 'backuspagemuseum',
2613             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
2614             'upload_date': '20161008',
2615             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
2616             'categories': ['Nonprofits & Activism'],
2617             'tags': list,
2618             'like_count': int,
2619             'dislike_count': int,
2620         },
2621         'params': {
2622             'noplaylist': True,
2623             'skip_download': True,
2624         },
2625     }, {
2626         # https://github.com/ytdl-org/youtube-dl/issues/21844
2627         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
2628         'info_dict': {
2629             'title': 'Data Analysis with Dr Mike Pound',
2630             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
2631             'uploader_id': 'Computerphile',
2632             'uploader': 'Computerphile',
2633         },
2634         'playlist_mincount': 11,
2635     }, {
2636         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
2637         'only_matching': True,
2638     }, {
2639         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
2640         'only_matching': True,
2641     }, {
2642         # music album playlist
2643         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
2644         'only_matching': True,
2645     }, {
2646         'url': 'https://invidio.us/playlist?list=PLDIoUOhQQPlXr63I_vwF9GD8sAKh77dWU',
2647         'only_matching': True,
2648     }]
2649
2650     def _real_initialize(self):
2651         self._login()
2652
2653     def extract_videos_from_page(self, page):
2654         ids_in_page = []
2655         titles_in_page = []
2656
2657         for item in re.findall(
2658                 r'(<[^>]*\bdata-video-id\s*=\s*["\'][0-9A-Za-z_-]{11}[^>]+>)', page):
2659             attrs = extract_attributes(item)
2660             video_id = attrs['data-video-id']
2661             video_title = unescapeHTML(attrs.get('data-title'))
2662             if video_title:
2663                 video_title = video_title.strip()
2664             ids_in_page.append(video_id)
2665             titles_in_page.append(video_title)
2666
2667         # Fallback with old _VIDEO_RE
2668         self.extract_videos_from_page_impl(
2669             self._VIDEO_RE, page, ids_in_page, titles_in_page)
2670
2671         # Relaxed fallbacks
2672         self.extract_videos_from_page_impl(
2673             r'href="\s*/watch\?v\s*=\s*(?P<id>[0-9A-Za-z_-]{11})', page,
2674             ids_in_page, titles_in_page)
2675         self.extract_videos_from_page_impl(
2676             r'data-video-ids\s*=\s*["\'](?P<id>[0-9A-Za-z_-]{11})', page,
2677             ids_in_page, titles_in_page)
2678
2679         return zip(ids_in_page, titles_in_page)
2680
2681     def _extract_mix(self, playlist_id):
2682         # The mixes are generated from a single video
2683         # the id of the playlist is just 'RD' + video_id
2684         ids = []
2685         last_id = playlist_id[-11:]
2686         for n in itertools.count(1):
2687             url = 'https://youtube.com/watch?v=%s&list=%s' % (last_id, playlist_id)
2688             webpage = self._download_webpage(
2689                 url, playlist_id, 'Downloading page {0} of Youtube mix'.format(n))
2690             new_ids = orderedSet(re.findall(
2691                 r'''(?xs)data-video-username=".*?".*?
2692                            href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s''' % re.escape(playlist_id),
2693                 webpage))
2694             # Fetch new pages until all the videos are repeated, it seems that
2695             # there are always 51 unique videos.
2696             new_ids = [_id for _id in new_ids if _id not in ids]
2697             if not new_ids:
2698                 break
2699             ids.extend(new_ids)
2700             last_id = ids[-1]
2701
2702         url_results = self._ids_to_results(ids)
2703
2704         search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
2705         title_span = (
2706             search_title('playlist-title')
2707             or search_title('title long-title')
2708             or search_title('title'))
2709         title = clean_html(title_span)
2710
2711         return self.playlist_result(url_results, playlist_id, title)
2712
2713     def _extract_playlist(self, playlist_id):
2714         url = self._TEMPLATE_URL % playlist_id
2715         page = self._download_webpage(url, playlist_id)
2716
2717         # the yt-alert-message now has tabindex attribute (see https://github.com/ytdl-org/youtube-dl/issues/11604)
2718         for match in re.findall(r'<div class="yt-alert-message"[^>]*>([^<]+)</div>', page):
2719             match = match.strip()
2720             # Check if the playlist exists or is private
2721             mobj = re.match(r'[^<]*(?:The|This) playlist (?P<reason>does not exist|is private)[^<]*', match)
2722             if mobj:
2723                 reason = mobj.group('reason')
2724                 message = 'This playlist %s' % reason
2725                 if 'private' in reason:
2726                     message += ', use --username or --netrc to access it'
2727                 message += '.'
2728                 raise ExtractorError(message, expected=True)
2729             elif re.match(r'[^<]*Invalid parameters[^<]*', match):
2730                 raise ExtractorError(
2731                     'Invalid parameters. Maybe URL is incorrect.',
2732                     expected=True)
2733             elif re.match(r'[^<]*Choose your language[^<]*', match):
2734                 continue
2735             else:
2736                 self.report_warning('Youtube gives an alert message: ' + match)
2737
2738         playlist_title = self._html_search_regex(
2739             r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
2740             page, 'title', default=None)
2741
2742         _UPLOADER_BASE = r'class=["\']pl-header-details[^>]+>\s*<li>\s*<a[^>]+\bhref='
2743         uploader = self._search_regex(
2744             r'%s["\']/(?:user|channel)/[^>]+>([^<]+)' % _UPLOADER_BASE,
2745             page, 'uploader', default=None)
2746         mobj = re.search(
2747             r'%s(["\'])(?P<path>/(?:user|channel)/(?P<uploader_id>.+?))\1' % _UPLOADER_BASE,
2748             page)
2749         if mobj:
2750             uploader_id = mobj.group('uploader_id')
2751             uploader_url = compat_urlparse.urljoin(url, mobj.group('path'))
2752         else:
2753             uploader_id = uploader_url = None
2754
2755         has_videos = True
2756
2757         if not playlist_title:
2758             try:
2759                 # Some playlist URLs don't actually serve a playlist (e.g.
2760                 # https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4)
2761                 next(self._entries(page, playlist_id))
2762             except StopIteration:
2763                 has_videos = False
2764
2765         playlist = self.playlist_result(
2766             self._entries(page, playlist_id), playlist_id, playlist_title)
2767         playlist.update({
2768             'uploader': uploader,
2769             'uploader_id': uploader_id,
2770             'uploader_url': uploader_url,
2771         })
2772
2773         return has_videos, playlist
2774
2775     def _check_download_just_video(self, url, playlist_id):
2776         # Check if it's a video-specific URL
2777         query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
2778         video_id = query_dict.get('v', [None])[0] or self._search_regex(
2779             r'(?:(?:^|//)youtu\.be/|youtube\.com/embed/(?!videoseries))([0-9A-Za-z_-]{11})', url,
2780             'video id', default=None)
2781         if video_id:
2782             if self._downloader.params.get('noplaylist'):
2783                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
2784                 return video_id, self.url_result(video_id, 'Youtube', video_id=video_id)
2785             else:
2786                 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
2787                 return video_id, None
2788         return None, None
2789
2790     def _real_extract(self, url):
2791         # Extract playlist id
2792         mobj = re.match(self._VALID_URL, url)
2793         if mobj is None:
2794             raise ExtractorError('Invalid URL: %s' % url)
2795         playlist_id = mobj.group(1) or mobj.group(2)
2796
2797         video_id, video = self._check_download_just_video(url, playlist_id)
2798         if video:
2799             return video
2800
2801         if playlist_id.startswith(('RD', 'UL', 'PU')):
2802             # Mixes require a custom extraction process
2803             return self._extract_mix(playlist_id)
2804
2805         has_videos, playlist = self._extract_playlist(playlist_id)
2806         if has_videos or not video_id:
2807             return playlist
2808
2809         # Some playlist URLs don't actually serve a playlist (see
2810         # https://github.com/ytdl-org/youtube-dl/issues/10537).
2811         # Fallback to plain video extraction if there is a video id
2812         # along with playlist id.
2813         return self.url_result(video_id, 'Youtube', video_id=video_id)
2814
2815
2816 class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
2817     IE_DESC = 'YouTube.com channels'
2818     _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com|(?:www\.)?invidio\.us)/channel/(?P<id>[0-9A-Za-z_-]+)'
2819     _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
2820     _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
2821     IE_NAME = 'youtube:channel'
2822     _TESTS = [{
2823         'note': 'paginated channel',
2824         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
2825         'playlist_mincount': 91,
2826         'info_dict': {
2827             'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
2828             'title': 'Uploads from lex will',
2829             'uploader': 'lex will',
2830             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2831         }
2832     }, {
2833         'note': 'Age restricted channel',
2834         # from https://www.youtube.com/user/DeusExOfficial
2835         'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
2836         'playlist_mincount': 64,
2837         'info_dict': {
2838             'id': 'UUs0ifCMCm1icqRbqhUINa0w',
2839             'title': 'Uploads from Deus Ex',
2840             'uploader': 'Deus Ex',
2841             'uploader_id': 'DeusExOfficial',
2842         },
2843     }, {
2844         'url': 'https://invidio.us/channel/UC23qupoDRn9YOAVzeoxjOQA',
2845         'only_matching': True,
2846     }]
2847
2848     @classmethod
2849     def suitable(cls, url):
2850         return (False if YoutubePlaylistsIE.suitable(url) or YoutubeLiveIE.suitable(url)
2851                 else super(YoutubeChannelIE, cls).suitable(url))
2852
2853     def _build_template_url(self, url, channel_id):
2854         return self._TEMPLATE_URL % channel_id
2855
2856     def _real_extract(self, url):
2857         channel_id = self._match_id(url)
2858
2859         url = self._build_template_url(url, channel_id)
2860
2861         # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
2862         # Workaround by extracting as a playlist if managed to obtain channel playlist URL
2863         # otherwise fallback on channel by page extraction
2864         channel_page = self._download_webpage(
2865             url + '?view=57', channel_id,
2866             'Downloading channel page', fatal=False)
2867         if channel_page is False:
2868             channel_playlist_id = False
2869         else:
2870             channel_playlist_id = self._html_search_meta(
2871                 'channelId', channel_page, 'channel id', default=None)
2872             if not channel_playlist_id:
2873                 channel_url = self._html_search_meta(
2874                     ('al:ios:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad'),
2875                     channel_page, 'channel url', default=None)
2876                 if channel_url:
2877                     channel_playlist_id = self._search_regex(
2878                         r'vnd\.youtube://user/([0-9A-Za-z_-]+)',
2879                         channel_url, 'channel id', default=None)
2880         if channel_playlist_id and channel_playlist_id.startswith('UC'):
2881             playlist_id = 'UU' + channel_playlist_id[2:]
2882             return self.url_result(
2883                 compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')
2884
2885         channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
2886         autogenerated = re.search(r'''(?x)
2887                 class="[^"]*?(?:
2888                     channel-header-autogenerated-label|
2889                     yt-channel-title-autogenerated
2890                 )[^"]*"''', channel_page) is not None
2891
2892         if autogenerated:
2893             # The videos are contained in a single page
2894             # the ajax pages can't be used, they are empty
2895             entries = [
2896                 self.url_result(
2897                     video_id, 'Youtube', video_id=video_id,
2898                     video_title=video_title)
2899                 for video_id, video_title in self.extract_videos_from_page(channel_page)]
2900             return self.playlist_result(entries, channel_id)
2901
2902         try:
2903             next(self._entries(channel_page, channel_id))
2904         except StopIteration:
2905             alert_message = self._html_search_regex(
2906                 r'(?s)<div[^>]+class=(["\']).*?\byt-alert-message\b.*?\1[^>]*>(?P<alert>[^<]+)</div>',
2907                 channel_page, 'alert', default=None, group='alert')
2908             if alert_message:
2909                 raise ExtractorError('Youtube said: %s' % alert_message, expected=True)
2910
2911         return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
2912
2913
2914 class YoutubeUserIE(YoutubeChannelIE):
2915     IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
2916     _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:(?P<user>user|c)/)?(?!(?:attribution_link|watch|results|shared)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
2917     _TEMPLATE_URL = 'https://www.youtube.com/%s/%s/videos'
2918     IE_NAME = 'youtube:user'
2919
2920     _TESTS = [{
2921         'url': 'https://www.youtube.com/user/TheLinuxFoundation',
2922         'playlist_mincount': 320,
2923         'info_dict': {
2924             'id': 'UUfX55Sx5hEFjoC3cNs6mCUQ',
2925             'title': 'Uploads from The Linux Foundation',
2926             'uploader': 'The Linux Foundation',
2927             'uploader_id': 'TheLinuxFoundation',
2928         }
2929     }, {
2930         # Only available via https://www.youtube.com/c/12minuteathlete/videos
2931         # but not https://www.youtube.com/user/12minuteathlete/videos
2932         'url': 'https://www.youtube.com/c/12minuteathlete/videos',
2933         'playlist_mincount': 249,
2934         'info_dict': {
2935             'id': 'UUVjM-zV6_opMDx7WYxnjZiQ',
2936             'title': 'Uploads from 12 Minute Athlete',
2937             'uploader': '12 Minute Athlete',
2938             'uploader_id': 'the12minuteathlete',
2939         }
2940     }, {
2941         'url': 'ytuser:phihag',
2942         'only_matching': True,
2943     }, {
2944         'url': 'https://www.youtube.com/c/gametrailers',
2945         'only_matching': True,
2946     }, {
2947         'url': 'https://www.youtube.com/gametrailers',
2948         'only_matching': True,
2949     }, {
2950         # This channel is not available, geo restricted to JP
2951         'url': 'https://www.youtube.com/user/kananishinoSMEJ/videos',
2952         'only_matching': True,
2953     }]
2954
2955     @classmethod
2956     def suitable(cls, url):
2957         # Don't return True if the url can be extracted with other youtube
2958         # extractor, the regex would is too permissive and it would match.
2959         other_yt_ies = iter(klass for (name, klass) in globals().items() if name.startswith('Youtube') and name.endswith('IE') and klass is not cls)
2960         if any(ie.suitable(url) for ie in other_yt_ies):
2961             return False
2962         else:
2963             return super(YoutubeUserIE, cls).suitable(url)
2964
2965     def _build_template_url(self, url, channel_id):
2966         mobj = re.match(self._VALID_URL, url)
2967         return self._TEMPLATE_URL % (mobj.group('user') or 'user', mobj.group('id'))
2968
2969
2970 class YoutubeLiveIE(YoutubeBaseInfoExtractor):
2971     IE_DESC = 'YouTube.com live streams'
2972     _VALID_URL = r'(?P<base_url>https?://(?:\w+\.)?youtube\.com/(?:(?:user|channel|c)/)?(?P<id>[^/]+))/live'
2973     IE_NAME = 'youtube:live'
2974
2975     _TESTS = [{
2976         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
2977         'info_dict': {
2978             'id': 'a48o2S1cPoo',
2979             'ext': 'mp4',
2980             'title': 'The Young Turks - Live Main Show',
2981             'uploader': 'The Young Turks',
2982             'uploader_id': 'TheYoungTurks',
2983             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
2984             'upload_date': '20150715',
2985             'license': 'Standard YouTube License',
2986             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
2987             'categories': ['News & Politics'],
2988             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
2989             'like_count': int,
2990             'dislike_count': int,
2991         },
2992         'params': {
2993             'skip_download': True,
2994         },
2995     }, {
2996         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
2997         'only_matching': True,
2998     }, {
2999         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
3000         'only_matching': True,
3001     }, {
3002         'url': 'https://www.youtube.com/TheYoungTurks/live',
3003         'only_matching': True,
3004     }]
3005
3006     def _real_extract(self, url):
3007         mobj = re.match(self._VALID_URL, url)
3008         channel_id = mobj.group('id')
3009         base_url = mobj.group('base_url')
3010         webpage = self._download_webpage(url, channel_id, fatal=False)
3011         if webpage:
3012             page_type = self._og_search_property(
3013                 'type', webpage, 'page type', default='')
3014             video_id = self._html_search_meta(
3015                 'videoId', webpage, 'video id', default=None)
3016             if page_type.startswith('video') and video_id and re.match(
3017                     r'^[0-9A-Za-z_-]{11}$', video_id):
3018                 return self.url_result(video_id, YoutubeIE.ie_key())
3019         return self.url_result(base_url)
3020
3021
3022 class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
3023     IE_DESC = 'YouTube.com user/channel playlists'
3024     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+)/playlists'
3025     IE_NAME = 'youtube:playlists'
3026
3027     _TESTS = [{
3028         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
3029         'playlist_mincount': 4,
3030         'info_dict': {
3031             'id': 'ThirstForScience',
3032             'title': 'ThirstForScience',
3033         },
3034     }, {
3035         # with "Load more" button
3036         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
3037         'playlist_mincount': 70,
3038         'info_dict': {
3039             'id': 'igorkle1',
3040             'title': 'Игорь Клейнер',
3041         },
3042     }, {
3043         'url': 'https://www.youtube.com/channel/UCiU1dHvZObB2iP6xkJ__Icw/playlists',
3044         'playlist_mincount': 17,
3045         'info_dict': {
3046             'id': 'UCiU1dHvZObB2iP6xkJ__Icw',
3047             'title': 'Chem Player',
3048         },
3049         'skip': 'Blocked',
3050     }]
3051
3052
3053 class YoutubeSearchBaseInfoExtractor(YoutubePlaylistBaseInfoExtractor):
3054     _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})(?:[^"]*"[^>]+\btitle="(?P<title>[^"]+))?'
3055
3056
3057 class YoutubeSearchIE(SearchInfoExtractor, YoutubeSearchBaseInfoExtractor):
3058     IE_DESC = 'YouTube.com searches'
3059     # there doesn't appear to be a real limit, for example if you search for
3060     # 'python' you get more than 8.000.000 results
3061     _MAX_RESULTS = float('inf')
3062     IE_NAME = 'youtube:search'
3063     _SEARCH_KEY = 'ytsearch'
3064     _EXTRA_QUERY_ARGS = {}
3065     _TESTS = []
3066
3067     def _get_n_results(self, query, n):
3068         """Get a specified number of results for a query"""
3069
3070         videos = []
3071         limit = n
3072
3073         url_query = {
3074             'search_query': query.encode('utf-8'),
3075         }
3076         url_query.update(self._EXTRA_QUERY_ARGS)
3077         result_url = 'https://www.youtube.com/results?' + compat_urllib_parse_urlencode(url_query)
3078
3079         for pagenum in itertools.count(1):
3080             data = self._download_json(
3081                 result_url, video_id='query "%s"' % query,
3082                 note='Downloading page %s' % pagenum,
3083                 errnote='Unable to download API page',
3084                 query={'spf': 'navigate'})
3085             html_content = data[1]['body']['content']
3086
3087             if 'class="search-message' in html_content:
3088                 raise ExtractorError(
3089                     '[youtube] No video results', expected=True)
3090
3091             new_videos = list(self._process_page(html_content))
3092             videos += new_videos
3093             if not new_videos or len(videos) > limit:
3094                 break
3095             next_link = self._html_search_regex(
3096                 r'href="(/results\?[^"]*\bsp=[^"]+)"[^>]*>\s*<span[^>]+class="[^"]*\byt-uix-button-content\b[^"]*"[^>]*>Next',
3097                 html_content, 'next link', default=None)
3098             if next_link is None:
3099                 break
3100             result_url = compat_urlparse.urljoin('https://www.youtube.com/', next_link)
3101
3102         if len(videos) > n:
3103             videos = videos[:n]
3104         return self.playlist_result(videos, query)
3105
3106
3107 class YoutubeSearchDateIE(YoutubeSearchIE):
3108     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
3109     _SEARCH_KEY = 'ytsearchdate'
3110     IE_DESC = 'YouTube.com searches, newest videos first'
3111     _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}
3112
3113
3114 class YoutubeSearchURLIE(YoutubeSearchBaseInfoExtractor):
3115     IE_DESC = 'YouTube.com search URLs'
3116     IE_NAME = 'youtube:search_url'
3117     _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
3118     _TESTS = [{
3119         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
3120         'playlist_mincount': 5,
3121         'info_dict': {
3122             'title': 'youtube-dl test video',
3123         }
3124     }, {
3125         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
3126         'only_matching': True,
3127     }]
3128
3129     def _real_extract(self, url):
3130         mobj = re.match(self._VALID_URL, url)
3131         query = compat_urllib_parse_unquote_plus(mobj.group('query'))
3132         webpage = self._download_webpage(url, query)
3133         return self.playlist_result(self._process_page(webpage), playlist_title=query)
3134
3135
3136 class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
3137     IE_DESC = 'YouTube.com (multi-season) shows'
3138     _VALID_URL = r'https?://(?:www\.)?youtube\.com/show/(?P<id>[^?#]*)'
3139     IE_NAME = 'youtube:show'
3140     _TESTS = [{
3141         'url': 'https://www.youtube.com/show/airdisasters',
3142         'playlist_mincount': 5,
3143         'info_dict': {
3144             'id': 'airdisasters',
3145             'title': 'Air Disasters',
3146         }
3147     }]
3148
3149     def _real_extract(self, url):
3150         playlist_id = self._match_id(url)
3151         return super(YoutubeShowIE, self)._real_extract(
3152             'https://www.youtube.com/show/%s/playlists' % playlist_id)
3153
3154
3155 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
3156     """
3157     Base class for feed extractors
3158     Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
3159     """
3160     _LOGIN_REQUIRED = True
3161
3162     @property
3163     def IE_NAME(self):
3164         return 'youtube:%s' % self._FEED_NAME
3165
3166     def _real_initialize(self):
3167         self._login()
3168
3169     def _entries(self, page):
3170         # The extraction process is the same as for playlists, but the regex
3171         # for the video ids doesn't contain an index
3172         ids = []
3173         more_widget_html = content_html = page
3174         for page_num in itertools.count(1):
3175             matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
3176
3177             # 'recommended' feed has infinite 'load more' and each new portion spins
3178             # the same videos in (sometimes) slightly different order, so we'll check
3179             # for unicity and break when portion has no new videos
3180             new_ids = list(filter(lambda video_id: video_id not in ids, orderedSet(matches)))
3181             if not new_ids:
3182                 break
3183
3184             ids.extend(new_ids)
3185
3186             for entry in self._ids_to_results(new_ids):
3187                 yield entry
3188
3189             mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
3190             if not mobj:
3191                 break
3192
3193             more = self._download_json(
3194                 'https://youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
3195                 'Downloading page #%s' % page_num,
3196                 transform_source=uppercase_escape)
3197             content_html = more['content_html']
3198             more_widget_html = more['load_more_widget_html']
3199
3200     def _real_extract(self, url):
3201         page = self._download_webpage(
3202             'https://www.youtube.com/feed/%s' % self._FEED_NAME,
3203             self._PLAYLIST_TITLE)
3204         return self.playlist_result(
3205             self._entries(page), playlist_title=self._PLAYLIST_TITLE)
3206
3207
3208 class YoutubeWatchLaterIE(YoutubePlaylistIE):
3209     IE_NAME = 'youtube:watchlater'
3210     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
3211     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'
3212
3213     _TESTS = [{
3214         'url': 'https://www.youtube.com/playlist?list=WL',
3215         'only_matching': True,
3216     }, {
3217         'url': 'https://www.youtube.com/watch?v=bCNU9TrbiRk&index=1&list=WL',
3218         'only_matching': True,
3219     }]
3220
3221     def _real_extract(self, url):
3222         _, video = self._check_download_just_video(url, 'WL')
3223         if video:
3224             return video
3225         _, playlist = self._extract_playlist('WL')
3226         return playlist
3227
3228
3229 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
3230     IE_NAME = 'youtube:favorites'
3231     IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
3232     _VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
3233     _LOGIN_REQUIRED = True
3234
3235     def _real_extract(self, url):
3236         webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
3237         playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, 'favourites playlist id')
3238         return self.url_result(playlist_id, 'YoutubePlaylist')
3239
3240
3241 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
3242     IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
3243     _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/recommended|:ytrec(?:ommended)?'
3244     _FEED_NAME = 'recommended'
3245     _PLAYLIST_TITLE = 'Youtube Recommended videos'
3246
3247
3248 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
3249     IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
3250     _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
3251     _FEED_NAME = 'subscriptions'
3252     _PLAYLIST_TITLE = 'Youtube Subscriptions'
3253
3254
3255 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
3256     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
3257     _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/history|:ythistory'
3258     _FEED_NAME = 'history'
3259     _PLAYLIST_TITLE = 'Youtube History'
3260
3261
3262 class YoutubeTruncatedURLIE(InfoExtractor):
3263     IE_NAME = 'youtube:truncated_url'
3264     IE_DESC = False  # Do not list
3265     _VALID_URL = r'''(?x)
3266         (?:https?://)?
3267         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
3268         (?:watch\?(?:
3269             feature=[a-z_]+|
3270             annotation_id=annotation_[^&]+|
3271             x-yt-cl=[0-9]+|
3272             hl=[^&]*|
3273             t=[0-9]+
3274         )?
3275         |
3276             attribution_link\?a=[^&]+
3277         )
3278         $
3279     '''
3280
3281     _TESTS = [{
3282         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
3283         'only_matching': True,
3284     }, {
3285         'url': 'https://www.youtube.com/watch?',
3286         'only_matching': True,
3287     }, {
3288         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
3289         'only_matching': True,
3290     }, {
3291         'url': 'https://www.youtube.com/watch?feature=foo',
3292         'only_matching': True,
3293     }, {
3294         'url': 'https://www.youtube.com/watch?hl=en-GB',
3295         'only_matching': True,
3296     }, {
3297         'url': 'https://www.youtube.com/watch?t=2372',
3298         'only_matching': True,
3299     }]
3300
3301     def _real_extract(self, url):
3302         raise ExtractorError(
3303             'Did you forget to quote the URL? Remember that & is a meta '
3304             'character in most shells, so you want to put the URL in quotes, '
3305             'like  youtube-dl '
3306             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
3307             ' or simply  youtube-dl BaW_jenozKc  .',
3308             expected=True)
3309
3310
3311 class YoutubeTruncatedIDIE(InfoExtractor):
3312     IE_NAME = 'youtube:truncated_id'
3313     IE_DESC = False  # Do not list
3314     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
3315
3316     _TESTS = [{
3317         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
3318         'only_matching': True,
3319     }]
3320
3321     def _real_extract(self, url):
3322         video_id = self._match_id(url)
3323         raise ExtractorError(
3324             'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
3325             expected=True)