_ Git - youtube-dl/blob - youtube_dl/extractor/youtube.py

   1 # coding: utf-8
   2
   3 from __future__ import unicode_literals
   4
   5
   6 import itertools
   7 import json
   8 import os.path
   9 import random
  10 import re
  11 import time
  12 import traceback
  13
  14 from .common import InfoExtractor, SearchInfoExtractor
  15 from ..jsinterp import JSInterpreter
  16 from ..swfinterp import SWFInterpreter
  17 from ..compat import (
  18     compat_chr,
  19     compat_HTTPError,
  20     compat_kwargs,
  21     compat_parse_qs,
  22     compat_urllib_parse_unquote,
  23     compat_urllib_parse_unquote_plus,
  24     compat_urllib_parse_urlencode,
  25     compat_urllib_parse_urlparse,
  26     compat_urlparse,
  27     compat_str,
  28 )
  29 from ..utils import (
  30     bool_or_none,
  31     clean_html,
  32     error_to_compat_str,
  33     extract_attributes,
  34     ExtractorError,
  35     float_or_none,
  36     get_element_by_attribute,
  37     get_element_by_id,
  38     int_or_none,
  39     mimetype2ext,
  40     orderedSet,
  41     parse_codecs,
  42     parse_duration,
  43     remove_quotes,
  44     remove_start,
  45     smuggle_url,
  46     str_or_none,
  47     str_to_int,
  48     try_get,
  49     unescapeHTML,
  50     unified_strdate,
  51     unsmuggle_url,
  52     uppercase_escape,
  53     url_or_none,
  54     urlencode_postdata,
  55 )
  56
  57
  58 class YoutubeBaseInfoExtractor(InfoExtractor):
  59     """Provide base functions for Youtube extractors"""
  60     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
  61     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
  62
  63     _LOOKUP_URL = 'https://accounts.google.com/_/signin/sl/lookup'
  64     _CHALLENGE_URL = 'https://accounts.google.com/_/signin/sl/challenge'
  65     _TFA_URL = 'https://accounts.google.com/_/signin/challenge?hl=en&TL={0}'
  66
  67     _NETRC_MACHINE = 'youtube'
  68     # If True it will raise an error if no login info is provided
  69     _LOGIN_REQUIRED = False
  70
  71     _PLAYLIST_ID_RE = r'(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}'
  72
  73     _YOUTUBE_CLIENT_HEADERS = {
  74         'x-youtube-client-name': '1',
  75         'x-youtube-client-version': '1.20200609.04.02',
  76     }
  77
  78     def _set_language(self):
  79         self._set_cookie(
  80             '.youtube.com', 'PREF', 'f1=50000000&f6=8&hl=en',
  81             # YouTube sets the expire time to about two months
  82             expire_time=time.time() + 2 * 30 * 24 * 3600)
  83
  84     def _ids_to_results(self, ids):
  85         return [
  86             self.url_result(vid_id, 'Youtube', video_id=vid_id)
  87             for vid_id in ids]
  88
  89     def _login(self):
  90         """
  91         Attempt to log in to YouTube.
  92         True is returned if successful or skipped.
  93         False is returned if login failed.
  94
  95         If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
  96         """
  97         username, password = self._get_login_info()
  98         # No authentication to be performed
  99         if username is None:
 100             if self._LOGIN_REQUIRED and self._downloader.params.get('cookiefile') is None:
 101                 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
 102             return True
 103
 104         login_page = self._download_webpage(
 105             self._LOGIN_URL, None,
 106             note='Downloading login page',
 107             errnote='unable to fetch login page', fatal=False)
 108         if login_page is False:
 109             return
 110
 111         login_form = self._hidden_inputs(login_page)
 112
 113         def req(url, f_req, note, errnote):
 114             data = login_form.copy()
 115             data.update({
 116                 'pstMsg': 1,
 117                 'checkConnection': 'youtube',
 118                 'checkedDomains': 'youtube',
 119                 'hl': 'en',
 120                 'deviceinfo': '[null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]',
 121                 'f.req': json.dumps(f_req),
 122                 'flowName': 'GlifWebSignIn',
 123                 'flowEntry': 'ServiceLogin',
 124                 # TODO: reverse actual botguard identifier generation algo
 125                 'bgRequest': '["identifier",""]',
 126             })
 127             return self._download_json(
 128                 url, None, note=note, errnote=errnote,
 129                 transform_source=lambda s: re.sub(r'^[^[]*', '', s),
 130                 fatal=False,
 131                 data=urlencode_postdata(data), headers={
 132                     'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8',
 133                     'Google-Accounts-XSRF': 1,
 134                 })
 135
 136         def warn(message):
 137             self._downloader.report_warning(message)
 138
 139         lookup_req = [
 140             username,
 141             None, [], None, 'US', None, None, 2, False, True,
 142             [
 143                 None, None,
 144                 [2, 1, None, 1,
 145                  'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
 146                  None, [], 4],
 147                 1, [None, None, []], None, None, None, True
 148             ],
 149             username,
 150         ]
 151
 152         lookup_results = req(
 153             self._LOOKUP_URL, lookup_req,
 154             'Looking up account info', 'Unable to look up account info')
 155
 156         if lookup_results is False:
 157             return False
 158
 159         user_hash = try_get(lookup_results, lambda x: x[0][2], compat_str)
 160         if not user_hash:
 161             warn('Unable to extract user hash')
 162             return False
 163
 164         challenge_req = [
 165             user_hash,
 166             None, 1, None, [1, None, None, None, [password, None, True]],
 167             [
 168                 None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
 169                 1, [None, None, []], None, None, None, True
 170             ]]
 171
 172         challenge_results = req(
 173             self._CHALLENGE_URL, challenge_req,
 174             'Logging in', 'Unable to log in')
 175
 176         if challenge_results is False:
 177             return
 178
 179         login_res = try_get(challenge_results, lambda x: x[0][5], list)
 180         if login_res:
 181             login_msg = try_get(login_res, lambda x: x[5], compat_str)
 182             warn(
 183                 'Unable to login: %s' % 'Invalid password'
 184                 if login_msg == 'INCORRECT_ANSWER_ENTERED' else login_msg)
 185             return False
 186
 187         res = try_get(challenge_results, lambda x: x[0][-1], list)
 188         if not res:
 189             warn('Unable to extract result entry')
 190             return False
 191
 192         login_challenge = try_get(res, lambda x: x[0][0], list)
 193         if login_challenge:
 194             challenge_str = try_get(login_challenge, lambda x: x[2], compat_str)
 195             if challenge_str == 'TWO_STEP_VERIFICATION':
 196                 # SEND_SUCCESS - TFA code has been successfully sent to phone
 197                 # QUOTA_EXCEEDED - reached the limit of TFA codes
 198                 status = try_get(login_challenge, lambda x: x[5], compat_str)
 199                 if status == 'QUOTA_EXCEEDED':
 200                     warn('Exceeded the limit of TFA codes, try later')
 201                     return False
 202
 203                 tl = try_get(challenge_results, lambda x: x[1][2], compat_str)
 204                 if not tl:
 205                     warn('Unable to extract TL')
 206                     return False
 207
 208                 tfa_code = self._get_tfa_info('2-step verification code')
 209
 210                 if not tfa_code:
 211                     warn(
 212                         'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
 213                         '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
 214                     return False
 215
 216                 tfa_code = remove_start(tfa_code, 'G-')
 217
 218                 tfa_req = [
 219                     user_hash, None, 2, None,
 220                     [
 221                         9, None, None, None, None, None, None, None,
 222                         [None, tfa_code, True, 2]
 223                     ]]
 224
 225                 tfa_results = req(
 226                     self._TFA_URL.format(tl), tfa_req,
 227                     'Submitting TFA code', 'Unable to submit TFA code')
 228
 229                 if tfa_results is False:
 230                     return False
 231
 232                 tfa_res = try_get(tfa_results, lambda x: x[0][5], list)
 233                 if tfa_res:
 234                     tfa_msg = try_get(tfa_res, lambda x: x[5], compat_str)
 235                     warn(
 236                         'Unable to finish TFA: %s' % 'Invalid TFA code'
 237                         if tfa_msg == 'INCORRECT_ANSWER_ENTERED' else tfa_msg)
 238                     return False
 239
 240                 check_cookie_url = try_get(
 241                     tfa_results, lambda x: x[0][-1][2], compat_str)
 242             else:
 243                 CHALLENGES = {
 244                     'LOGIN_CHALLENGE': "This device isn't recognized. For your security, Google wants to make sure it's really you.",
 245                     'USERNAME_RECOVERY': 'Please provide additional information to aid in the recovery process.',
 246                     'REAUTH': "There is something unusual about your activity. For your security, Google wants to make sure it's really you.",
 247                 }
 248                 challenge = CHALLENGES.get(
 249                     challenge_str,
 250                     '%s returned error %s.' % (self.IE_NAME, challenge_str))
 251                 warn('%s\nGo to https://accounts.google.com/, login and solve a challenge.' % challenge)
 252                 return False
 253         else:
 254             check_cookie_url = try_get(res, lambda x: x[2], compat_str)
 255
 256         if not check_cookie_url:
 257             warn('Unable to extract CheckCookie URL')
 258             return False
 259
 260         check_cookie_results = self._download_webpage(
 261             check_cookie_url, None, 'Checking cookie', fatal=False)
 262
 263         if check_cookie_results is False:
 264             return False
 265
 266         if 'https://myaccount.google.com/' not in check_cookie_results:
 267             warn('Unable to log in')
 268             return False
 269
 270         return True
 271
 272     def _download_webpage_handle(self, *args, **kwargs):
 273         query = kwargs.get('query', {}).copy()
 274         query['disable_polymer'] = 'true'
 275         kwargs['query'] = query
 276         return super(YoutubeBaseInfoExtractor, self)._download_webpage_handle(
 277             *args, **compat_kwargs(kwargs))
 278
 279     def _real_initialize(self):
 280         if self._downloader is None:
 281             return
 282         self._set_language()
 283         if not self._login():
 284             return
 285
 286
 287 class YoutubeEntryListBaseInfoExtractor(YoutubeBaseInfoExtractor):
 288     # Extract entries from page with "Load more" button
 289     def _entries(self, page, playlist_id):
 290         more_widget_html = content_html = page
 291         for page_num in itertools.count(1):
 292             for entry in self._process_page(content_html):
 293                 yield entry
 294
 295             mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
 296             if not mobj:
 297                 break
 298
 299             count = 0
 300             retries = 3
 301             while count <= retries:
 302                 try:
 303                     # Downloading page may result in intermittent 5xx HTTP error
 304                     # that is usually worked around with a retry
 305                     more = self._download_json(
 306                         'https://www.youtube.com/%s' % mobj.group('more'), playlist_id,
 307                         'Downloading page #%s%s'
 308                         % (page_num, ' (retry #%d)' % count if count else ''),
 309                         transform_source=uppercase_escape,
 310                         headers=self._YOUTUBE_CLIENT_HEADERS)
 311                     break
 312                 except ExtractorError as e:
 313                     if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503):
 314                         count += 1
 315                         if count <= retries:
 316                             continue
 317                     raise
 318
 319             content_html = more['content_html']
 320             if not content_html.strip():
 321                 # Some webpages show a "Load more" button but they don't
 322                 # have more videos
 323                 break
 324             more_widget_html = more['load_more_widget_html']
 325
 326
 327 class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
 328     def _process_page(self, content):
 329         for video_id, video_title in self.extract_videos_from_page(content):
 330             yield self.url_result(video_id, 'Youtube', video_id, video_title)
 331
 332     def extract_videos_from_page_impl(self, video_re, page, ids_in_page, titles_in_page):
 333         for mobj in re.finditer(video_re, page):
 334             # The link with index 0 is not the first video of the playlist (not sure if still actual)
 335             if 'index' in mobj.groupdict() and mobj.group('id') == '0':
 336                 continue
 337             video_id = mobj.group('id')
 338             video_title = unescapeHTML(
 339                 mobj.group('title')) if 'title' in mobj.groupdict() else None
 340             if video_title:
 341                 video_title = video_title.strip()
 342             if video_title == '► Play all':
 343                 video_title = None
 344             try:
 345                 idx = ids_in_page.index(video_id)
 346                 if video_title and not titles_in_page[idx]:
 347                     titles_in_page[idx] = video_title
 348             except ValueError:
 349                 ids_in_page.append(video_id)
 350                 titles_in_page.append(video_title)
 351
 352     def extract_videos_from_page(self, page):
 353         ids_in_page = []
 354         titles_in_page = []
 355         self.extract_videos_from_page_impl(
 356             self._VIDEO_RE, page, ids_in_page, titles_in_page)
 357         return zip(ids_in_page, titles_in_page)
 358
 359
 360 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
 361     def _process_page(self, content):
 362         for playlist_id in orderedSet(re.findall(
 363                 r'<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*><a[^>]+href="/?playlist\?list=([0-9A-Za-z-_]{10,})"',
 364                 content)):
 365             yield self.url_result(
 366                 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
 367
 368     def _real_extract(self, url):
 369         playlist_id = self._match_id(url)
 370         webpage = self._download_webpage(url, playlist_id)
 371         title = self._og_search_title(webpage, fatal=False)
 372         return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
 373
 374
 375 class YoutubeIE(YoutubeBaseInfoExtractor):
 376     IE_DESC = 'YouTube.com'
 377     _VALID_URL = r"""(?x)^
 378                      (
 379                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
 380                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com/|
 381                             (?:www\.)?deturl\.com/www\.youtube\.com/|
 382                             (?:www\.)?pwnyoutube\.com/|
 383                             (?:www\.)?hooktube\.com/|
 384                             (?:www\.)?yourepeat\.com/|
 385                             tube\.majestyc\.net/|
 386                             # Invidious instances taken from https://github.com/omarroth/invidious/wiki/Invidious-Instances
 387                             (?:(?:www|dev)\.)?invidio\.us/|
 388                             (?:(?:www|no)\.)?invidiou\.sh/|
 389                             (?:(?:www|fi|de)\.)?invidious\.snopyta\.org/|
 390                             (?:www\.)?invidious\.kabi\.tk/|
 391                             (?:www\.)?invidious\.13ad\.de/|
 392                             (?:www\.)?invidious\.mastodon\.host/|
 393                             (?:www\.)?invidious\.nixnet\.xyz/|
 394                             (?:www\.)?invidious\.drycat\.fr/|
 395                             (?:www\.)?tube\.poal\.co/|
 396                             (?:www\.)?vid\.wxzm\.sx/|
 397                             (?:www\.)?yewtu\.be/|
 398                             (?:www\.)?yt\.elukerio\.org/|
 399                             (?:www\.)?yt\.lelux\.fi/|
 400                             (?:www\.)?invidious\.ggc-project\.de/|
 401                             (?:www\.)?yt\.maisputain\.ovh/|
 402                             (?:www\.)?invidious\.13ad\.de/|
 403                             (?:www\.)?invidious\.toot\.koeln/|
 404                             (?:www\.)?invidious\.fdn\.fr/|
 405                             (?:www\.)?watch\.nettohikari\.com/|
 406                             (?:www\.)?kgg2m7yk5aybusll\.onion/|
 407                             (?:www\.)?qklhadlycap4cnod\.onion/|
 408                             (?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion/|
 409                             (?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion/|
 410                             (?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion/|
 411                             (?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion/|
 412                             (?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p/|
 413                             (?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion/|
 414                             youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
 415                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 416                          (?:                                                  # the various things that can precede the ID:
 417                              (?:(?:v|embed|e)/(?!videoseries))                # v/ or embed/ or e/
 418                              |(?:                                             # or the v= param in all its forms
 419                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 420                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 421                                  (?:.*?[&;])??                                # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&amp;v=V36LpHqtcDY)
 422                                  v=
 423                              )
 424                          ))
 425                          |(?:
 426                             youtu\.be|                                        # just youtu.be/xxxx
 427                             vid\.plus|                                        # or vid.plus/xxxx
 428                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
 429                          )/
 430                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
 431                          )
 432                      )?                                                       # all until now is optional -> you can pass the naked ID
 433                      ([0-9A-Za-z_-]{11})                                      # here is it! the YouTube video ID
 434                      (?!.*?\blist=
 435                         (?:
 436                             %(playlist_id)s|                                  # combined list/video URLs are handled by the playlist IE
 437                             WL                                                # WL are handled by the watch later IE
 438                         )
 439                      )
 440                      (?(1).+)?                                                # if we found the ID, everything can follow
 441                      $""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
 442     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
 443     _PLAYER_INFO_RE = (
 444         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?/base\.(?P<ext>[a-z]+)$',
 445         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.(?P<ext>[a-z]+)$',
 446     )
 447     _formats = {
 448         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 449         '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
 450         '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
 451         '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
 452         '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
 453         '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 454         '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 455         '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 456         # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
 457         '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
 458         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 459         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
 460         '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 461         '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
 462         '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 463         '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
 464         '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 465         '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
 466
 467
 468         # 3D videos
 469         '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 470         '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
 471         '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 472         '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
 473         '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
 474         '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 475         '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
 476
 477         # Apple HTTP Live Streaming
 478         '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 479         '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 480         '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 481         '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
 482         '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 483         '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
 484         '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
 485         '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
 486
 487         # DASH mp4 video
 488         '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
 489         '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
 490         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 491         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
 492         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
 493         '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
 494         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
 495         '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
 496         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
 497         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 498         '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
 499         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
 500
 501         # Dash mp4 audio
 502         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
 503         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
 504         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
 505         '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 506         '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
 507         '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
 508         '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
 509
 510         # Dash webm
 511         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 512         '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 513         '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 514         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 515         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 516         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
 517         '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
 518         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 519         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 520         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 521         '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 522         '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 523         '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 524         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 525         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 526         # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
 527         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 528         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 529         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 530         '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 531         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
 532         '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
 533
 534         # Dash webm audio
 535         '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
 536         '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
 537
 538         # Dash webm audio with opus inside
 539         '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
 540         '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
 541         '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
 542
 543         # RTMP (unnamed)
 544         '_rtmp': {'protocol': 'rtmp'},
 545
 546         # av01 video only formats sometimes served with "unknown" codecs
 547         '394': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
 548         '395': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
 549         '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
 550         '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
 551     }
 552     _SUBTITLE_FORMATS = ('srv1', 'srv2', 'srv3', 'ttml', 'vtt')
 553
 554     _GEO_BYPASS = False
 555
 556     IE_NAME = 'youtube'
 557     _TESTS = [
 558         {
 559             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&t=1s&end=9',
 560             'info_dict': {
 561                 'id': 'BaW_jenozKc',
 562                 'ext': 'mp4',
 563                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
 564                 'uploader': 'Philipp Hagemeister',
 565                 'uploader_id': 'phihag',
 566                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
 567                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
 568                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
 569                 'upload_date': '20121002',
 570                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
 571                 'categories': ['Science & Technology'],
 572                 'tags': ['youtube-dl'],
 573                 'duration': 10,
 574                 'view_count': int,
 575                 'like_count': int,
 576                 'dislike_count': int,
 577                 'start_time': 1,
 578                 'end_time': 9,
 579             }
 580         },
 581         {
 582             'url': 'https://www.youtube.com/watch?v=UxxajLWwzqY',
 583             'note': 'Test generic use_cipher_signature video (#897)',
 584             'info_dict': {
 585                 'id': 'UxxajLWwzqY',
 586                 'ext': 'mp4',
 587                 'upload_date': '20120506',
 588                 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
 589                 'alt_title': 'I Love It (feat. Charli XCX)',
 590                 'description': 'md5:19a2f98d9032b9311e686ed039564f63',
 591                 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
 592                          'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
 593                          'iconic ep', 'iconic', 'love', 'it'],
 594                 'duration': 180,
 595                 'uploader': 'Icona Pop',
 596                 'uploader_id': 'IconaPop',
 597                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IconaPop',
 598                 'creator': 'Icona Pop',
 599                 'track': 'I Love It (feat. Charli XCX)',
 600                 'artist': 'Icona Pop',
 601             }
 602         },
 603         {
 604             'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
 605             'note': 'Test VEVO video with age protection (#956)',
 606             'info_dict': {
 607                 'id': '07FYdnEawAQ',
 608                 'ext': 'mp4',
 609                 'upload_date': '20130703',
 610                 'title': 'Justin Timberlake - Tunnel Vision (Official Music Video) (Explicit)',
 611                 'alt_title': 'Tunnel Vision',
 612                 'description': 'md5:07dab3356cde4199048e4c7cd93471e1',
 613                 'duration': 419,
 614                 'uploader': 'justintimberlakeVEVO',
 615                 'uploader_id': 'justintimberlakeVEVO',
 616                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/justintimberlakeVEVO',
 617                 'creator': 'Justin Timberlake',
 618                 'track': 'Tunnel Vision',
 619                 'artist': 'Justin Timberlake',
 620                 'age_limit': 18,
 621             }
 622         },
 623         {
 624             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
 625             'note': 'Embed-only video (#1746)',
 626             'info_dict': {
 627                 'id': 'yZIXLfi8CZQ',
 628                 'ext': 'mp4',
 629                 'upload_date': '20120608',
 630                 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
 631                 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
 632                 'uploader': 'SET India',
 633                 'uploader_id': 'setindia',
 634                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
 635                 'age_limit': 18,
 636             }
 637         },
 638         {
 639             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=UxxajLWwzqY',
 640             'note': 'Use the first video ID in the URL',
 641             'info_dict': {
 642                 'id': 'BaW_jenozKc',
 643                 'ext': 'mp4',
 644                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
 645                 'uploader': 'Philipp Hagemeister',
 646                 'uploader_id': 'phihag',
 647                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
 648                 'upload_date': '20121002',
 649                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
 650                 'categories': ['Science & Technology'],
 651                 'tags': ['youtube-dl'],
 652                 'duration': 10,
 653                 'view_count': int,
 654                 'like_count': int,
 655                 'dislike_count': int,
 656             },
 657             'params': {
 658                 'skip_download': True,
 659             },
 660         },
 661         {
 662             'url': 'https://www.youtube.com/watch?v=a9LDPn-MO4I',
 663             'note': '256k DASH audio (format 141) via DASH manifest',
 664             'info_dict': {
 665                 'id': 'a9LDPn-MO4I',
 666                 'ext': 'm4a',
 667                 'upload_date': '20121002',
 668                 'uploader_id': '8KVIDEO',
 669                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
 670                 'description': '',
 671                 'uploader': '8KVIDEO',
 672                 'title': 'UHDTV TEST 8K VIDEO.mp4'
 673             },
 674             'params': {
 675                 'youtube_include_dash_manifest': True,
 676                 'format': '141',
 677             },
 678             'skip': 'format 141 not served anymore',
 679         },
 680         # DASH manifest with encrypted signature
 681         {
 682             'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
 683             'info_dict': {
 684                 'id': 'IB3lcPjvWLA',
 685                 'ext': 'm4a',
 686                 'title': 'Afrojack, Spree Wilson - The Spark (Official Music Video) ft. Spree Wilson',
 687                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
 688                 'duration': 244,
 689                 'uploader': 'AfrojackVEVO',
 690                 'uploader_id': 'AfrojackVEVO',
 691                 'upload_date': '20131011',
 692             },
 693             'params': {
 694                 'youtube_include_dash_manifest': True,
 695                 'format': '141/bestaudio[ext=m4a]',
 696             },
 697         },
 698         # JS player signature function name containing $
 699         {
 700             'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
 701             'info_dict': {
 702                 'id': 'nfWlot6h_JM',
 703                 'ext': 'm4a',
 704                 'title': 'Taylor Swift - Shake It Off',
 705                 'description': 'md5:307195cd21ff7fa352270fe884570ef0',
 706                 'duration': 242,
 707                 'uploader': 'TaylorSwiftVEVO',
 708                 'uploader_id': 'TaylorSwiftVEVO',
 709                 'upload_date': '20140818',
 710             },
 711             'params': {
 712                 'youtube_include_dash_manifest': True,
 713                 'format': '141/bestaudio[ext=m4a]',
 714             },
 715         },
 716         # Controversy video
 717         {
 718             'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
 719             'info_dict': {
 720                 'id': 'T4XJQO3qol8',
 721                 'ext': 'mp4',
 722                 'duration': 219,
 723                 'upload_date': '20100909',
 724                 'uploader': 'Amazing Atheist',
 725                 'uploader_id': 'TheAmazingAtheist',
 726                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
 727                 'title': 'Burning Everyone\'s Koran',
 728                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
 729             }
 730         },
 731         # Normal age-gate video (No vevo, embed allowed)
 732         {
 733             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
 734             'info_dict': {
 735                 'id': 'HtVdAasjOgU',
 736                 'ext': 'mp4',
 737                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
 738                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
 739                 'duration': 142,
 740                 'uploader': 'The Witcher',
 741                 'uploader_id': 'WitcherGame',
 742                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
 743                 'upload_date': '20140605',
 744                 'age_limit': 18,
 745             },
 746         },
 747         # Age-gate video with encrypted signature
 748         {
 749             'url': 'https://www.youtube.com/watch?v=6kLq3WMV1nU',
 750             'info_dict': {
 751                 'id': '6kLq3WMV1nU',
 752                 'ext': 'mp4',
 753                 'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
 754                 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
 755                 'duration': 246,
 756                 'uploader': 'LloydVEVO',
 757                 'uploader_id': 'LloydVEVO',
 758                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/LloydVEVO',
 759                 'upload_date': '20110629',
 760                 'age_limit': 18,
 761             },
 762         },
 763         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
 764         # YouTube Red ad is not captured for creator
 765         {
 766             'url': '__2ABJjxzNo',
 767             'info_dict': {
 768                 'id': '__2ABJjxzNo',
 769                 'ext': 'mp4',
 770                 'duration': 266,
 771                 'upload_date': '20100430',
 772                 'uploader_id': 'deadmau5',
 773                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
 774                 'creator': 'Dada Life, deadmau5',
 775                 'description': 'md5:12c56784b8032162bb936a5f76d55360',
 776                 'uploader': 'deadmau5',
 777                 'title': 'Deadmau5 - Some Chords (HD)',
 778                 'alt_title': 'This Machine Kills Some Chords',
 779             },
 780             'expected_warnings': [
 781                 'DASH manifest missing',
 782             ]
 783         },
 784         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
 785         {
 786             'url': 'lqQg6PlCWgI',
 787             'info_dict': {
 788                 'id': 'lqQg6PlCWgI',
 789                 'ext': 'mp4',
 790                 'duration': 6085,
 791                 'upload_date': '20150827',
 792                 'uploader_id': 'olympic',
 793                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
 794                 'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
 795                 'uploader': 'Olympic',
 796                 'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
 797             },
 798             'params': {
 799                 'skip_download': 'requires avconv',
 800             }
 801         },
 802         # Non-square pixels
 803         {
 804             'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
 805             'info_dict': {
 806                 'id': '_b-2C3KPAM0',
 807                 'ext': 'mp4',
 808                 'stretched_ratio': 16 / 9.,
 809                 'duration': 85,
 810                 'upload_date': '20110310',
 811                 'uploader_id': 'AllenMeow',
 812                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
 813                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
 814                 'uploader': '孫ᄋᄅ',
 815                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
 816             },
 817         },
 818         # url_encoded_fmt_stream_map is empty string
 819         {
 820             'url': 'qEJwOuvDf7I',
 821             'info_dict': {
 822                 'id': 'qEJwOuvDf7I',
 823                 'ext': 'webm',
 824                 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
 825                 'description': '',
 826                 'upload_date': '20150404',
 827                 'uploader_id': 'spbelect',
 828                 'uploader': 'Наблюдатели Петербурга',
 829             },
 830             'params': {
 831                 'skip_download': 'requires avconv',
 832             },
 833             'skip': 'This live event has ended.',
 834         },
 835         # Extraction from multiple DASH manifests (https://github.com/ytdl-org/youtube-dl/pull/6097)
 836         {
 837             'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
 838             'info_dict': {
 839                 'id': 'FIl7x6_3R5Y',
 840                 'ext': 'webm',
 841                 'title': 'md5:7b81415841e02ecd4313668cde88737a',
 842                 'description': 'md5:116377fd2963b81ec4ce64b542173306',
 843                 'duration': 220,
 844                 'upload_date': '20150625',
 845                 'uploader_id': 'dorappi2000',
 846                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/dorappi2000',
 847                 'uploader': 'dorappi2000',
 848                 'formats': 'mincount:31',
 849             },
 850             'skip': 'not actual anymore',
 851         },
 852         # DASH manifest with segment_list
 853         {
 854             'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
 855             'md5': '8ce563a1d667b599d21064e982ab9e31',
 856             'info_dict': {
 857                 'id': 'CsmdDsKjzN8',
 858                 'ext': 'mp4',
 859                 'upload_date': '20150501',  # According to '<meta itemprop="datePublished"', but in other places it's 20150510
 860                 'uploader': 'Airtek',
 861                 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
 862                 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
 863                 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
 864             },
 865             'params': {
 866                 'youtube_include_dash_manifest': True,
 867                 'format': '135',  # bestvideo
 868             },
 869             'skip': 'This live event has ended.',
 870         },
 871         {
 872             # Multifeed videos (multiple cameras), URL is for Main Camera
 873             'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
 874             'info_dict': {
 875                 'id': 'jqWvoWXjCVs',
 876                 'title': 'teamPGP: Rocket League Noob Stream',
 877                 'description': 'md5:dc7872fb300e143831327f1bae3af010',
 878             },
 879             'playlist': [{
 880                 'info_dict': {
 881                     'id': 'jqWvoWXjCVs',
 882                     'ext': 'mp4',
 883                     'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
 884                     'description': 'md5:dc7872fb300e143831327f1bae3af010',
 885                     'duration': 7335,
 886                     'upload_date': '20150721',
 887                     'uploader': 'Beer Games Beer',
 888                     'uploader_id': 'beergamesbeer',
 889                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
 890                     'license': 'Standard YouTube License',
 891                 },
 892             }, {
 893                 'info_dict': {
 894                     'id': '6h8e8xoXJzg',
 895                     'ext': 'mp4',
 896                     'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
 897                     'description': 'md5:dc7872fb300e143831327f1bae3af010',
 898                     'duration': 7337,
 899                     'upload_date': '20150721',
 900                     'uploader': 'Beer Games Beer',
 901                     'uploader_id': 'beergamesbeer',
 902                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
 903                     'license': 'Standard YouTube License',
 904                 },
 905             }, {
 906                 'info_dict': {
 907                     'id': 'PUOgX5z9xZw',
 908                     'ext': 'mp4',
 909                     'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
 910                     'description': 'md5:dc7872fb300e143831327f1bae3af010',
 911                     'duration': 7337,
 912                     'upload_date': '20150721',
 913                     'uploader': 'Beer Games Beer',
 914                     'uploader_id': 'beergamesbeer',
 915                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
 916                     'license': 'Standard YouTube License',
 917                 },
 918             }, {
 919                 'info_dict': {
 920                     'id': 'teuwxikvS5k',
 921                     'ext': 'mp4',
 922                     'title': 'teamPGP: Rocket League Noob Stream (zim)',
 923                     'description': 'md5:dc7872fb300e143831327f1bae3af010',
 924                     'duration': 7334,
 925                     'upload_date': '20150721',
 926                     'uploader': 'Beer Games Beer',
 927                     'uploader_id': 'beergamesbeer',
 928                     'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
 929                     'license': 'Standard YouTube License',
 930                 },
 931             }],
 932             'params': {
 933                 'skip_download': True,
 934             },
 935             'skip': 'This video is not available.',
 936         },
 937         {
 938             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
 939             'url': 'https://www.youtube.com/watch?v=gVfLd0zydlo',
 940             'info_dict': {
 941                 'id': 'gVfLd0zydlo',
 942                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
 943             },
 944             'playlist_count': 2,
 945             'skip': 'Not multifeed anymore',
 946         },
 947         {
 948             'url': 'https://vid.plus/FlRa-iH7PGw',
 949             'only_matching': True,
 950         },
 951         {
 952             'url': 'https://zwearz.com/watch/9lWxNJF-ufM/electra-woman-dyna-girl-official-trailer-grace-helbig.html',
 953             'only_matching': True,
 954         },
 955         {
 956             # Title with JS-like syntax "};" (see https://github.com/ytdl-org/youtube-dl/issues/7468)
 957             # Also tests cut-off URL expansion in video description (see
 958             # https://github.com/ytdl-org/youtube-dl/issues/1892,
 959             # https://github.com/ytdl-org/youtube-dl/issues/8164)
 960             'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
 961             'info_dict': {
 962                 'id': 'lsguqyKfVQg',
 963                 'ext': 'mp4',
 964                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
 965                 'alt_title': 'Dark Walk - Position Music',
 966                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
 967                 'duration': 133,
 968                 'upload_date': '20151119',
 969                 'uploader_id': 'IronSoulElf',
 970                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
 971                 'uploader': 'IronSoulElf',
 972                 'creator': 'Todd Haberman,  Daniel Law Heath and Aaron Kaplan',
 973                 'track': 'Dark Walk - Position Music',
 974                 'artist': 'Todd Haberman,  Daniel Law Heath and Aaron Kaplan',
 975                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
 976             },
 977             'params': {
 978                 'skip_download': True,
 979             },
 980         },
 981         {
 982             # Tags with '};' (see https://github.com/ytdl-org/youtube-dl/issues/7468)
 983             'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
 984             'only_matching': True,
 985         },
 986         {
 987             # Video with yt:stretch=17:0
 988             'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
 989             'info_dict': {
 990                 'id': 'Q39EVAstoRM',
 991                 'ext': 'mp4',
 992                 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
 993                 'description': 'md5:ee18a25c350637c8faff806845bddee9',
 994                 'upload_date': '20151107',
 995                 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
 996                 'uploader': 'CH GAMER DROID',
 997             },
 998             'params': {
 999                 'skip_download': True,
1000             },
1001             'skip': 'This video does not exist.',
1002         },
1003         {
1004             # Video licensed under Creative Commons
1005             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
1006             'info_dict': {
1007                 'id': 'M4gD1WSo5mA',
1008                 'ext': 'mp4',
1009                 'title': 'md5:e41008789470fc2533a3252216f1c1d1',
1010                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
1011                 'duration': 721,
1012                 'upload_date': '20150127',
1013                 'uploader_id': 'BerkmanCenter',
1014                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
1015                 'uploader': 'The Berkman Klein Center for Internet & Society',
1016                 'license': 'Creative Commons Attribution license (reuse allowed)',
1017             },
1018             'params': {
1019                 'skip_download': True,
1020             },
1021         },
1022         {
1023             # Channel-like uploader_url
1024             'url': 'https://www.youtube.com/watch?v=eQcmzGIKrzg',
1025             'info_dict': {
1026                 'id': 'eQcmzGIKrzg',
1027                 'ext': 'mp4',
1028                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
1029                 'description': 'md5:dda0d780d5a6e120758d1711d062a867',
1030                 'duration': 4060,
1031                 'upload_date': '20151119',
1032                 'uploader': 'Bernie Sanders',
1033                 'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
1034                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
1035                 'license': 'Creative Commons Attribution license (reuse allowed)',
1036             },
1037             'params': {
1038                 'skip_download': True,
1039             },
1040         },
1041         {
1042             'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;amp;v=V36LpHqtcDY',
1043             'only_matching': True,
1044         },
1045         {
1046             # YouTube Red paid video (https://github.com/ytdl-org/youtube-dl/issues/10059)
1047             'url': 'https://www.youtube.com/watch?v=i1Ko8UG-Tdo',
1048             'only_matching': True,
1049         },
1050         {
1051             # Rental video preview
1052             'url': 'https://www.youtube.com/watch?v=yYr8q0y5Jfg',
1053             'info_dict': {
1054                 'id': 'uGpuVWrhIzE',
1055                 'ext': 'mp4',
1056                 'title': 'Piku - Trailer',
1057                 'description': 'md5:c36bd60c3fd6f1954086c083c72092eb',
1058                 'upload_date': '20150811',
1059                 'uploader': 'FlixMatrix',
1060                 'uploader_id': 'FlixMatrixKaravan',
1061                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/FlixMatrixKaravan',
1062                 'license': 'Standard YouTube License',
1063             },
1064             'params': {
1065                 'skip_download': True,
1066             },
1067             'skip': 'This video is not available.',
1068         },
1069         {
1070             # YouTube Red video with episode data
1071             'url': 'https://www.youtube.com/watch?v=iqKdEhx-dD4',
1072             'info_dict': {
1073                 'id': 'iqKdEhx-dD4',
1074                 'ext': 'mp4',
1075                 'title': 'Isolation - Mind Field (Ep 1)',
1076                 'description': 'md5:46a29be4ceffa65b92d277b93f463c0f',
1077                 'duration': 2085,
1078                 'upload_date': '20170118',
1079                 'uploader': 'Vsauce',
1080                 'uploader_id': 'Vsauce',
1081                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
1082                 'series': 'Mind Field',
1083                 'season_number': 1,
1084                 'episode_number': 1,
1085             },
1086             'params': {
1087                 'skip_download': True,
1088             },
1089             'expected_warnings': [
1090                 'Skipping DASH manifest',
1091             ],
1092         },
1093         {
1094             # The following content has been identified by the YouTube community
1095             # as inappropriate or offensive to some audiences.
1096             'url': 'https://www.youtube.com/watch?v=6SJNVb0GnPI',
1097             'info_dict': {
1098                 'id': '6SJNVb0GnPI',
1099                 'ext': 'mp4',
1100                 'title': 'Race Differences in Intelligence',
1101                 'description': 'md5:5d161533167390427a1f8ee89a1fc6f1',
1102                 'duration': 965,
1103                 'upload_date': '20140124',
1104                 'uploader': 'New Century Foundation',
1105                 'uploader_id': 'UCEJYpZGqgUob0zVVEaLhvVg',
1106                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCEJYpZGqgUob0zVVEaLhvVg',
1107             },
1108             'params': {
1109                 'skip_download': True,
1110             },
1111         },
1112         {
1113             # itag 212
1114             'url': '1t24XAntNCY',
1115             'only_matching': True,
1116         },
1117         {
1118             # geo restricted to JP
1119             'url': 'sJL6WA-aGkQ',
1120             'only_matching': True,
1121         },
1122         {
1123             'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
1124             'only_matching': True,
1125         },
1126         {
1127             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
1128             'only_matching': True,
1129         },
1130         {
1131             # DRM protected
1132             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',
1133             'only_matching': True,
1134         },
1135         {
1136             # Video with unsupported adaptive stream type formats
1137             'url': 'https://www.youtube.com/watch?v=Z4Vy8R84T1U',
1138             'info_dict': {
1139                 'id': 'Z4Vy8R84T1U',
1140                 'ext': 'mp4',
1141                 'title': 'saman SMAN 53 Jakarta(Sancety) opening COFFEE4th at SMAN 53 Jakarta',
1142                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
1143                 'duration': 433,
1144                 'upload_date': '20130923',
1145                 'uploader': 'Amelia Putri Harwita',
1146                 'uploader_id': 'UCpOxM49HJxmC1qCalXyB3_Q',
1147                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCpOxM49HJxmC1qCalXyB3_Q',
1148                 'formats': 'maxcount:10',
1149             },
1150             'params': {
1151                 'skip_download': True,
1152                 'youtube_include_dash_manifest': False,
1153             },
1154             'skip': 'not actual anymore',
1155         },
1156         {
1157             # Youtube Music Auto-generated description
1158             'url': 'https://music.youtube.com/watch?v=MgNrAu2pzNs',
1159             'info_dict': {
1160                 'id': 'MgNrAu2pzNs',
1161                 'ext': 'mp4',
1162                 'title': 'Voyeur Girl',
1163                 'description': 'md5:7ae382a65843d6df2685993e90a8628f',
1164                 'upload_date': '20190312',
1165                 'uploader': 'Stephen - Topic',
1166                 'uploader_id': 'UC-pWHpBjdGG69N9mM2auIAA',
1167                 'artist': 'Stephen',
1168                 'track': 'Voyeur Girl',
1169                 'album': 'it\'s too much love to know my dear',
1170                 'release_date': '20190313',
1171                 'release_year': 2019,
1172             },
1173             'params': {
1174                 'skip_download': True,
1175             },
1176         },
1177         {
1178             # Youtube Music Auto-generated description
1179             # Retrieve 'artist' field from 'Artist:' in video description
1180             # when it is present on youtube music video
1181             'url': 'https://www.youtube.com/watch?v=k0jLE7tTwjY',
1182             'info_dict': {
1183                 'id': 'k0jLE7tTwjY',
1184                 'ext': 'mp4',
1185                 'title': 'Latch Feat. Sam Smith',
1186                 'description': 'md5:3cb1e8101a7c85fcba9b4fb41b951335',
1187                 'upload_date': '20150110',
1188                 'uploader': 'Various Artists - Topic',
1189                 'uploader_id': 'UCNkEcmYdjrH4RqtNgh7BZ9w',
1190                 'artist': 'Disclosure',
1191                 'track': 'Latch Feat. Sam Smith',
1192                 'album': 'Latch Featuring Sam Smith',
1193                 'release_date': '20121008',
1194                 'release_year': 2012,
1195             },
1196             'params': {
1197                 'skip_download': True,
1198             },
1199         },
1200         {
1201             # Youtube Music Auto-generated description
1202             # handle multiple artists on youtube music video
1203             'url': 'https://www.youtube.com/watch?v=74qn0eJSjpA',
1204             'info_dict': {
1205                 'id': '74qn0eJSjpA',
1206                 'ext': 'mp4',
1207                 'title': 'Eastside',
1208                 'description': 'md5:290516bb73dcbfab0dcc4efe6c3de5f2',
1209                 'upload_date': '20180710',
1210                 'uploader': 'Benny Blanco - Topic',
1211                 'uploader_id': 'UCzqz_ksRu_WkIzmivMdIS7A',
1212                 'artist': 'benny blanco, Halsey, Khalid',
1213                 'track': 'Eastside',
1214                 'album': 'Eastside',
1215                 'release_date': '20180713',
1216                 'release_year': 2018,
1217             },
1218             'params': {
1219                 'skip_download': True,
1220             },
1221         },
1222         {
1223             # Youtube Music Auto-generated description
1224             # handle youtube music video with release_year and no release_date
1225             'url': 'https://www.youtube.com/watch?v=-hcAI0g-f5M',
1226             'info_dict': {
1227                 'id': '-hcAI0g-f5M',
1228                 'ext': 'mp4',
1229                 'title': 'Put It On Me',
1230                 'description': 'md5:f6422397c07c4c907c6638e1fee380a5',
1231                 'upload_date': '20180426',
1232                 'uploader': 'Matt Maeson - Topic',
1233                 'uploader_id': 'UCnEkIGqtGcQMLk73Kp-Q5LQ',
1234                 'artist': 'Matt Maeson',
1235                 'track': 'Put It On Me',
1236                 'album': 'The Hearse',
1237                 'release_date': None,
1238                 'release_year': 2018,
1239             },
1240             'params': {
1241                 'skip_download': True,
1242             },
1243         },
1244         {
1245             'url': 'https://www.youtubekids.com/watch?v=3b8nCWDgZ6Q',
1246             'only_matching': True,
1247         },
1248         {
1249             # invalid -> valid video id redirection
1250             'url': 'DJztXj2GPfl',
1251             'info_dict': {
1252                 'id': 'DJztXj2GPfk',
1253                 'ext': 'mp4',
1254                 'title': 'Panjabi MC - Mundian To Bach Ke (The Dictator Soundtrack)',
1255                 'description': 'md5:bf577a41da97918e94fa9798d9228825',
1256                 'upload_date': '20090125',
1257                 'uploader': 'Prochorowka',
1258                 'uploader_id': 'Prochorowka',
1259                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Prochorowka',
1260                 'artist': 'Panjabi MC',
1261                 'track': 'Beware of the Boys (Mundian to Bach Ke) - Motivo Hi-Lectro Remix',
1262                 'album': 'Beware of the Boys (Mundian To Bach Ke)',
1263             },
1264             'params': {
1265                 'skip_download': True,
1266             },
1267         },
1268         {
1269             # empty description results in an empty string
1270             'url': 'https://www.youtube.com/watch?v=x41yOUIvK2k',
1271             'info_dict': {
1272                 'id': 'x41yOUIvK2k',
1273                 'ext': 'mp4',
1274                 'title': 'IMG 3456',
1275                 'description': '',
1276                 'upload_date': '20170613',
1277                 'uploader_id': 'ElevageOrVert',
1278                 'uploader': 'ElevageOrVert',
1279             },
1280             'params': {
1281                 'skip_download': True,
1282             },
1283         },
1284     ]
1285
1286     def __init__(self, *args, **kwargs):
1287         super(YoutubeIE, self).__init__(*args, **kwargs)
1288         self._player_cache = {}
1289
1290     def report_video_info_webpage_download(self, video_id):
1291         """Report attempt to download video info webpage."""
1292         self.to_screen('%s: Downloading video info webpage' % video_id)
1293
1294     def report_information_extraction(self, video_id):
1295         """Report attempt to extract video information."""
1296         self.to_screen('%s: Extracting video information' % video_id)
1297
1298     def report_unavailable_format(self, video_id, format):
1299         """Report extracted video URL."""
1300         self.to_screen('%s: Format %s not available' % (video_id, format))
1301
1302     def report_rtmp_download(self):
1303         """Indicate the download will use the RTMP protocol."""
1304         self.to_screen('RTMP download detected')
1305
1306     def _signature_cache_id(self, example_sig):
1307         """ Return a string representation of a signature """
1308         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
1309
1310     @classmethod
1311     def _extract_player_info(cls, player_url):
1312         for player_re in cls._PLAYER_INFO_RE:
1313             id_m = re.search(player_re, player_url)
1314             if id_m:
1315                 break
1316         else:
1317             raise ExtractorError('Cannot identify player %r' % player_url)
1318         return id_m.group('ext'), id_m.group('id')
1319
1320     def _extract_signature_function(self, video_id, player_url, example_sig):
1321         player_type, player_id = self._extract_player_info(player_url)
1322
1323         # Read from filesystem cache
1324         func_id = '%s_%s_%s' % (
1325             player_type, player_id, self._signature_cache_id(example_sig))
1326         assert os.path.basename(func_id) == func_id
1327
1328         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
1329         if cache_spec is not None:
1330             return lambda s: ''.join(s[i] for i in cache_spec)
1331
1332         download_note = (
1333             'Downloading player %s' % player_url
1334             if self._downloader.params.get('verbose') else
1335             'Downloading %s player %s' % (player_type, player_id)
1336         )
1337         if player_type == 'js':
1338             code = self._download_webpage(
1339                 player_url, video_id,
1340                 note=download_note,
1341                 errnote='Download of %s failed' % player_url)
1342             res = self._parse_sig_js(code)
1343         elif player_type == 'swf':
1344             urlh = self._request_webpage(
1345                 player_url, video_id,
1346                 note=download_note,
1347                 errnote='Download of %s failed' % player_url)
1348             code = urlh.read()
1349             res = self._parse_sig_swf(code)
1350         else:
1351             assert False, 'Invalid player type %r' % player_type
1352
1353         test_string = ''.join(map(compat_chr, range(len(example_sig))))
1354         cache_res = res(test_string)
1355         cache_spec = [ord(c) for c in cache_res]
1356
1357         self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
1358         return res
1359
1360     def _print_sig_code(self, func, example_sig):
1361         def gen_sig_code(idxs):
1362             def _genslice(start, end, step):
1363                 starts = '' if start == 0 else str(start)
1364                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
1365                 steps = '' if step == 1 else (':%d' % step)
1366                 return 's[%s%s%s]' % (starts, ends, steps)
1367
1368             step = None
1369             # Quelch pyflakes warnings - start will be set when step is set
1370             start = '(Never used)'
1371             for i, prev in zip(idxs[1:], idxs[:-1]):
1372                 if step is not None:
1373                     if i - prev == step:
1374                         continue
1375                     yield _genslice(start, prev, step)
1376                     step = None
1377                     continue
1378                 if i - prev in [-1, 1]:
1379                     step = i - prev
1380                     start = prev
1381                     continue
1382                 else:
1383                     yield 's[%d]' % prev
1384             if step is None:
1385                 yield 's[%d]' % i
1386             else:
1387                 yield _genslice(start, i, step)
1388
1389         test_string = ''.join(map(compat_chr, range(len(example_sig))))
1390         cache_res = func(test_string)
1391         cache_spec = [ord(c) for c in cache_res]
1392         expr_code = ' + '.join(gen_sig_code(cache_spec))
1393         signature_id_tuple = '(%s)' % (
1394             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
1395         code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
1396                 '    return %s\n') % (signature_id_tuple, expr_code)
1397         self.to_screen('Extracted signature function:\n' + code)
1398
1399     def _parse_sig_js(self, jscode):
1400         funcname = self._search_regex(
1401             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1402              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1403              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1404              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
1405              # Obsolete patterns
1406              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1407              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
1408              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1409              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1410              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1411              r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1412              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
1413              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
1414             jscode, 'Initial JS player signature function name', group='sig')
1415
1416         jsi = JSInterpreter(jscode)
1417         initial_function = jsi.extract_function(funcname)
1418         return lambda s: initial_function([s])
1419
1420     def _parse_sig_swf(self, file_contents):
1421         swfi = SWFInterpreter(file_contents)
1422         TARGET_CLASSNAME = 'SignatureDecipher'
1423         searched_class = swfi.extract_class(TARGET_CLASSNAME)
1424         initial_function = swfi.extract_function(searched_class, 'decipher')
1425         return lambda s: initial_function([s])
1426
1427     def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
1428         """Turn the encrypted s field into a working signature"""
1429
1430         if player_url is None:
1431             raise ExtractorError('Cannot decrypt signature without player_url')
1432
1433         if player_url.startswith('//'):
1434             player_url = 'https:' + player_url
1435         elif not re.match(r'https?://', player_url):
1436             player_url = compat_urlparse.urljoin(
1437                 'https://www.youtube.com', player_url)
1438         try:
1439             player_id = (player_url, self._signature_cache_id(s))
1440             if player_id not in self._player_cache:
1441                 func = self._extract_signature_function(
1442                     video_id, player_url, s
1443                 )
1444                 self._player_cache[player_id] = func
1445             func = self._player_cache[player_id]
1446             if self._downloader.params.get('youtube_print_sig_code'):
1447                 self._print_sig_code(func, s)
1448             return func(s)
1449         except Exception as e:
1450             tb = traceback.format_exc()
1451             raise ExtractorError(
1452                 'Signature extraction failed: ' + tb, cause=e)
1453
1454     def _get_subtitles(self, video_id, webpage):
1455         try:
1456             subs_doc = self._download_xml(
1457                 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
1458                 video_id, note=False)
1459         except ExtractorError as err:
1460             self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
1461             return {}
1462
1463         sub_lang_list = {}
1464         for track in subs_doc.findall('track'):
1465             lang = track.attrib['lang_code']
1466             if lang in sub_lang_list:
1467                 continue
1468             sub_formats = []
1469             for ext in self._SUBTITLE_FORMATS:
1470                 params = compat_urllib_parse_urlencode({
1471                     'lang': lang,
1472                     'v': video_id,
1473                     'fmt': ext,
1474                     'name': track.attrib['name'].encode('utf-8'),
1475                 })
1476                 sub_formats.append({
1477                     'url': 'https://www.youtube.com/api/timedtext?' + params,
1478                     'ext': ext,
1479                 })
1480             sub_lang_list[lang] = sub_formats
1481         if not sub_lang_list:
1482             self._downloader.report_warning('video doesn\'t have subtitles')
1483             return {}
1484         return sub_lang_list
1485
1486     def _get_ytplayer_config(self, video_id, webpage):
1487         patterns = (
1488             # User data may contain arbitrary character sequences that may affect
1489             # JSON extraction with regex, e.g. when '};' is contained the second
1490             # regex won't capture the whole JSON. Yet working around by trying more
1491             # concrete regex first keeping in mind proper quoted string handling
1492             # to be implemented in future that will replace this workaround (see
1493             # https://github.com/ytdl-org/youtube-dl/issues/7468,
1494             # https://github.com/ytdl-org/youtube-dl/pull/7599)
1495             r';ytplayer\.config\s*=\s*({.+?});ytplayer',
1496             r';ytplayer\.config\s*=\s*({.+?});',
1497         )
1498         config = self._search_regex(
1499             patterns, webpage, 'ytplayer.config', default=None)
1500         if config:
1501             return self._parse_json(
1502                 uppercase_escape(config), video_id, fatal=False)
1503
1504     def _get_automatic_captions(self, video_id, webpage):
1505         """We need the webpage for getting the captions url, pass it as an
1506            argument to speed up the process."""
1507         self.to_screen('%s: Looking for automatic captions' % video_id)
1508         player_config = self._get_ytplayer_config(video_id, webpage)
1509         err_msg = 'Couldn\'t find automatic captions for %s' % video_id
1510         if not player_config:
1511             self._downloader.report_warning(err_msg)
1512             return {}
1513         try:
1514             args = player_config['args']
1515             caption_url = args.get('ttsurl')
1516             if caption_url:
1517                 timestamp = args['timestamp']
1518                 # We get the available subtitles
1519                 list_params = compat_urllib_parse_urlencode({
1520                     'type': 'list',
1521                     'tlangs': 1,
1522                     'asrs': 1,
1523                 })
1524                 list_url = caption_url + '&' + list_params
1525                 caption_list = self._download_xml(list_url, video_id)
1526                 original_lang_node = caption_list.find('track')
1527                 if original_lang_node is None:
1528                     self._downloader.report_warning('Video doesn\'t have automatic captions')
1529                     return {}
1530                 original_lang = original_lang_node.attrib['lang_code']
1531                 caption_kind = original_lang_node.attrib.get('kind', '')
1532
1533                 sub_lang_list = {}
1534                 for lang_node in caption_list.findall('target'):
1535                     sub_lang = lang_node.attrib['lang_code']
1536                     sub_formats = []
1537                     for ext in self._SUBTITLE_FORMATS:
1538                         params = compat_urllib_parse_urlencode({
1539                             'lang': original_lang,
1540                             'tlang': sub_lang,
1541                             'fmt': ext,
1542                             'ts': timestamp,
1543                             'kind': caption_kind,
1544                         })
1545                         sub_formats.append({
1546                             'url': caption_url + '&' + params,
1547                             'ext': ext,
1548                         })
1549                     sub_lang_list[sub_lang] = sub_formats
1550                 return sub_lang_list
1551
1552             def make_captions(sub_url, sub_langs):
1553                 parsed_sub_url = compat_urllib_parse_urlparse(sub_url)
1554                 caption_qs = compat_parse_qs(parsed_sub_url.query)
1555                 captions = {}
1556                 for sub_lang in sub_langs:
1557                     sub_formats = []
1558                     for ext in self._SUBTITLE_FORMATS:
1559                         caption_qs.update({
1560                             'tlang': [sub_lang],
1561                             'fmt': [ext],
1562                         })
1563                         sub_url = compat_urlparse.urlunparse(parsed_sub_url._replace(
1564                             query=compat_urllib_parse_urlencode(caption_qs, True)))
1565                         sub_formats.append({
1566                             'url': sub_url,
1567                             'ext': ext,
1568                         })
1569                     captions[sub_lang] = sub_formats
1570                 return captions
1571
1572             # New captions format as of 22.06.2017
1573             player_response = args.get('player_response')
1574             if player_response and isinstance(player_response, compat_str):
1575                 player_response = self._parse_json(
1576                     player_response, video_id, fatal=False)
1577                 if player_response:
1578                     renderer = player_response['captions']['playerCaptionsTracklistRenderer']
1579                     base_url = renderer['captionTracks'][0]['baseUrl']
1580                     sub_lang_list = []
1581                     for lang in renderer['translationLanguages']:
1582                         lang_code = lang.get('languageCode')
1583                         if lang_code:
1584                             sub_lang_list.append(lang_code)
1585                     return make_captions(base_url, sub_lang_list)
1586
1587             # Some videos don't provide ttsurl but rather caption_tracks and
1588             # caption_translation_languages (e.g. 20LmZk1hakA)
1589             # Does not used anymore as of 22.06.2017
1590             caption_tracks = args['caption_tracks']
1591             caption_translation_languages = args['caption_translation_languages']
1592             caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
1593             sub_lang_list = []
1594             for lang in caption_translation_languages.split(','):
1595                 lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
1596                 sub_lang = lang_qs.get('lc', [None])[0]
1597                 if sub_lang:
1598                     sub_lang_list.append(sub_lang)
1599             return make_captions(caption_url, sub_lang_list)
1600         # An extractor error can be raise by the download process if there are
1601         # no automatic captions but there are subtitles
1602         except (KeyError, IndexError, ExtractorError):
1603             self._downloader.report_warning(err_msg)
1604             return {}
1605
1606     def _mark_watched(self, video_id, video_info, player_response):
1607         playback_url = url_or_none(try_get(
1608             player_response,
1609             lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl']) or try_get(
1610             video_info, lambda x: x['videostats_playback_base_url'][0]))
1611         if not playback_url:
1612             return
1613         parsed_playback_url = compat_urlparse.urlparse(playback_url)
1614         qs = compat_urlparse.parse_qs(parsed_playback_url.query)
1615
1616         # cpn generation algorithm is reverse engineered from base.js.
1617         # In fact it works even with dummy cpn.
1618         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
1619         cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
1620
1621         qs.update({
1622             'ver': ['2'],
1623             'cpn': [cpn],
1624         })
1625         playback_url = compat_urlparse.urlunparse(
1626             parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
1627
1628         self._download_webpage(
1629             playback_url, video_id, 'Marking watched',
1630             'Unable to mark watched', fatal=False)
1631
1632     @staticmethod
1633     def _extract_urls(webpage):
1634         # Embedded YouTube player
1635         entries = [
1636             unescapeHTML(mobj.group('url'))
1637             for mobj in re.finditer(r'''(?x)
1638             (?:
1639                 <iframe[^>]+?src=|
1640                 data-video-url=|
1641                 <embed[^>]+?src=|
1642                 embedSWF\(?:\s*|
1643                 <object[^>]+data=|
1644                 new\s+SWFObject\(
1645             )
1646             (["\'])
1647                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
1648                 (?:embed|v|p)/[0-9A-Za-z_-]{11}.*?)
1649             \1''', webpage)]
1650
1651         # lazyYT YouTube embed
1652         entries.extend(list(map(
1653             unescapeHTML,
1654             re.findall(r'class="lazyYT" data-youtube-id="([^"]+)"', webpage))))
1655
1656         # Wordpress "YouTube Video Importer" plugin
1657         matches = re.findall(r'''(?x)<div[^>]+
1658             class=(?P<q1>[\'"])[^\'"]*\byvii_single_video_player\b[^\'"]*(?P=q1)[^>]+
1659             data-video_id=(?P<q2>[\'"])([^\'"]+)(?P=q2)''', webpage)
1660         entries.extend(m[-1] for m in matches)
1661
1662         return entries
1663
1664     @staticmethod
1665     def _extract_url(webpage):
1666         urls = YoutubeIE._extract_urls(webpage)
1667         return urls[0] if urls else None
1668
1669     @classmethod
1670     def extract_id(cls, url):
1671         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
1672         if mobj is None:
1673             raise ExtractorError('Invalid URL: %s' % url)
1674         video_id = mobj.group(2)
1675         return video_id
1676
1677     def _extract_chapters_from_json(self, webpage, video_id, duration):
1678         if not webpage:
1679             return
1680         player = self._parse_json(
1681             self._search_regex(
1682                 r'RELATED_PLAYER_ARGS["\']\s*:\s*({.+})\s*,?\s*\n', webpage,
1683                 'player args', default='{}'),
1684             video_id, fatal=False)
1685         if not player or not isinstance(player, dict):
1686             return
1687         watch_next_response = player.get('watch_next_response')
1688         if not isinstance(watch_next_response, compat_str):
1689             return
1690         response = self._parse_json(watch_next_response, video_id, fatal=False)
1691         if not response or not isinstance(response, dict):
1692             return
1693         chapters_list = try_get(
1694             response,
1695             lambda x: x['playerOverlays']
1696                        ['playerOverlayRenderer']
1697                        ['decoratedPlayerBarRenderer']
1698                        ['decoratedPlayerBarRenderer']
1699                        ['playerBar']
1700                        ['chapteredPlayerBarRenderer']
1701                        ['chapters'],
1702             list)
1703         if not chapters_list:
1704             return
1705
1706         def chapter_time(chapter):
1707             return float_or_none(
1708                 try_get(
1709                     chapter,
1710                     lambda x: x['chapterRenderer']['timeRangeStartMillis'],
1711                     int),
1712                 scale=1000)
1713         chapters = []
1714         for next_num, chapter in enumerate(chapters_list, start=1):
1715             start_time = chapter_time(chapter)
1716             if start_time is None:
1717                 continue
1718             end_time = (chapter_time(chapters_list[next_num])
1719                         if next_num < len(chapters_list) else duration)
1720             if end_time is None:
1721                 continue
1722             title = try_get(
1723                 chapter, lambda x: x['chapterRenderer']['title']['simpleText'],
1724                 compat_str)
1725             chapters.append({
1726                 'start_time': start_time,
1727                 'end_time': end_time,
1728                 'title': title,
1729             })
1730         return chapters
1731
1732     @staticmethod
1733     def _extract_chapters_from_description(description, duration):
1734         if not description:
1735             return None
1736         chapter_lines = re.findall(
1737             r'(?:^|<br\s*/>)([^<]*<a[^>]+onclick=["\']yt\.www\.watch\.player\.seekTo[^>]+>(\d{1,2}:\d{1,2}(?::\d{1,2})?)</a>[^>]*)(?=$|<br\s*/>)',
1738             description)
1739         if not chapter_lines:
1740             return None
1741         chapters = []
1742         for next_num, (chapter_line, time_point) in enumerate(
1743                 chapter_lines, start=1):
1744             start_time = parse_duration(time_point)
1745             if start_time is None:
1746                 continue
1747             if start_time > duration:
1748                 break
1749             end_time = (duration if next_num == len(chapter_lines)
1750                         else parse_duration(chapter_lines[next_num][1]))
1751             if end_time is None:
1752                 continue
1753             if end_time > duration:
1754                 end_time = duration
1755             if start_time > end_time:
1756                 break
1757             chapter_title = re.sub(
1758                 r'<a[^>]+>[^<]+</a>', '', chapter_line).strip(' \t-')
1759             chapter_title = re.sub(r'\s+', ' ', chapter_title)
1760             chapters.append({
1761                 'start_time': start_time,
1762                 'end_time': end_time,
1763                 'title': chapter_title,
1764             })
1765         return chapters
1766
1767     def _extract_chapters(self, webpage, description, video_id, duration):
1768         return (self._extract_chapters_from_json(webpage, video_id, duration)
1769                 or self._extract_chapters_from_description(description, duration))
1770
1771     def _real_extract(self, url):
1772         url, smuggled_data = unsmuggle_url(url, {})
1773
1774         proto = (
1775             'http' if self._downloader.params.get('prefer_insecure', False)
1776             else 'https')
1777
1778         start_time = None
1779         end_time = None
1780         parsed_url = compat_urllib_parse_urlparse(url)
1781         for component in [parsed_url.fragment, parsed_url.query]:
1782             query = compat_parse_qs(component)
1783             if start_time is None and 't' in query:
1784                 start_time = parse_duration(query['t'][0])
1785             if start_time is None and 'start' in query:
1786                 start_time = parse_duration(query['start'][0])
1787             if end_time is None and 'end' in query:
1788                 end_time = parse_duration(query['end'][0])
1789
1790         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1791         mobj = re.search(self._NEXT_URL_RE, url)
1792         if mobj:
1793             url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
1794         video_id = self.extract_id(url)
1795
1796         # Get video webpage
1797         url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
1798         video_webpage, urlh = self._download_webpage_handle(url, video_id)
1799
1800         qs = compat_parse_qs(compat_urllib_parse_urlparse(urlh.geturl()).query)
1801         video_id = qs.get('v', [None])[0] or video_id
1802
1803         # Attempt to extract SWF player URL
1804         mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1805         if mobj is not None:
1806             player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1807         else:
1808             player_url = None
1809
1810         dash_mpds = []
1811
1812         def add_dash_mpd(video_info):
1813             dash_mpd = video_info.get('dashmpd')
1814             if dash_mpd and dash_mpd[0] not in dash_mpds:
1815                 dash_mpds.append(dash_mpd[0])
1816
1817         def add_dash_mpd_pr(pl_response):
1818             dash_mpd = url_or_none(try_get(
1819                 pl_response, lambda x: x['streamingData']['dashManifestUrl'],
1820                 compat_str))
1821             if dash_mpd and dash_mpd not in dash_mpds:
1822                 dash_mpds.append(dash_mpd)
1823
1824         is_live = None
1825         view_count = None
1826
1827         def extract_view_count(v_info):
1828             return int_or_none(try_get(v_info, lambda x: x['view_count'][0]))
1829
1830         def extract_player_response(player_response, video_id):
1831             pl_response = str_or_none(player_response)
1832             if not pl_response:
1833                 return
1834             pl_response = self._parse_json(pl_response, video_id, fatal=False)
1835             if isinstance(pl_response, dict):
1836                 add_dash_mpd_pr(pl_response)
1837                 return pl_response
1838
1839         player_response = {}
1840
1841         # Get video info
1842         video_info = {}
1843         embed_webpage = None
1844         if (self._og_search_property('restrictions:age', video_webpage, default=None) == '18+'
1845                 or re.search(r'player-age-gate-content">', video_webpage) is not None):
1846             age_gate = True
1847             # We simulate the access to the video from www.youtube.com/v/{video_id}
1848             # this can be viewed without login into Youtube
1849             url = proto + '://www.youtube.com/embed/%s' % video_id
1850             embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
1851             data = compat_urllib_parse_urlencode({
1852                 'video_id': video_id,
1853                 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1854                 'sts': self._search_regex(
1855                     r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
1856             })
1857             video_info_url = proto + '://www.youtube.com/get_video_info?' + data
1858             try:
1859                 video_info_webpage = self._download_webpage(
1860                     video_info_url, video_id,
1861                     note='Refetching age-gated info webpage',
1862                     errnote='unable to download video info webpage')
1863             except ExtractorError:
1864                 video_info_webpage = None
1865             if video_info_webpage:
1866                 video_info = compat_parse_qs(video_info_webpage)
1867                 pl_response = video_info.get('player_response', [None])[0]
1868                 player_response = extract_player_response(pl_response, video_id)
1869                 add_dash_mpd(video_info)
1870                 view_count = extract_view_count(video_info)
1871         else:
1872             age_gate = False
1873             # Try looking directly into the video webpage
1874             ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
1875             if ytplayer_config:
1876                 args = ytplayer_config['args']
1877                 if args.get('url_encoded_fmt_stream_map') or args.get('hlsvp'):
1878                     # Convert to the same format returned by compat_parse_qs
1879                     video_info = dict((k, [v]) for k, v in args.items())
1880                     add_dash_mpd(video_info)
1881                 # Rental video is not rented but preview is available (e.g.
1882                 # https://www.youtube.com/watch?v=yYr8q0y5Jfg,
1883                 # https://github.com/ytdl-org/youtube-dl/issues/10532)
1884                 if not video_info and args.get('ypc_vid'):
1885                     return self.url_result(
1886                         args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
1887                 if args.get('livestream') == '1' or args.get('live_playback') == 1:
1888                     is_live = True
1889                 if not player_response:
1890                     player_response = extract_player_response(args.get('player_response'), video_id)
1891             if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
1892                 add_dash_mpd_pr(player_response)
1893
1894         def extract_unavailable_message():
1895             messages = []
1896             for tag, kind in (('h1', 'message'), ('div', 'submessage')):
1897                 msg = self._html_search_regex(
1898                     r'(?s)<{tag}[^>]+id=["\']unavailable-{kind}["\'][^>]*>(.+?)</{tag}>'.format(tag=tag, kind=kind),
1899                     video_webpage, 'unavailable %s' % kind, default=None)
1900                 if msg:
1901                     messages.append(msg)
1902             if messages:
1903                 return '\n'.join(messages)
1904
1905         if not video_info and not player_response:
1906             unavailable_message = extract_unavailable_message()
1907             if not unavailable_message:
1908                 unavailable_message = 'Unable to extract video data'
1909             raise ExtractorError(
1910                 'YouTube said: %s' % unavailable_message, expected=True, video_id=video_id)
1911
1912         if not isinstance(video_info, dict):
1913             video_info = {}
1914
1915         video_details = try_get(
1916             player_response, lambda x: x['videoDetails'], dict) or {}
1917
1918         microformat = try_get(
1919             player_response, lambda x: x['microformat']['playerMicroformatRenderer'], dict) or {}
1920
1921         video_title = video_info.get('title', [None])[0] or video_details.get('title')
1922         if not video_title:
1923             self._downloader.report_warning('Unable to extract video title')
1924             video_title = '_'
1925
1926         description_original = video_description = get_element_by_id("eow-description", video_webpage)
1927         if video_description:
1928
1929             def replace_url(m):
1930                 redir_url = compat_urlparse.urljoin(url, m.group(1))
1931                 parsed_redir_url = compat_urllib_parse_urlparse(redir_url)
1932                 if re.search(r'^(?:www\.)?(?:youtube(?:-nocookie)?\.com|youtu\.be)$', parsed_redir_url.netloc) and parsed_redir_url.path == '/redirect':
1933                     qs = compat_parse_qs(parsed_redir_url.query)
1934                     q = qs.get('q')
1935                     if q and q[0]:
1936                         return q[0]
1937                 return redir_url
1938
1939             description_original = video_description = re.sub(r'''(?x)
1940                 <a\s+
1941                     (?:[a-zA-Z-]+="[^"]*"\s+)*?
1942                     (?:title|href)="([^"]+)"\s+
1943                     (?:[a-zA-Z-]+="[^"]*"\s+)*?
1944                     class="[^"]*"[^>]*>
1945                 [^<]+\.{3}\s*
1946                 </a>
1947             ''', replace_url, video_description)
1948             video_description = clean_html(video_description)
1949         else:
1950             video_description = video_details.get('shortDescription')
1951             if video_description is None:
1952                 video_description = self._html_search_meta('description', video_webpage)
1953
1954         if not smuggled_data.get('force_singlefeed', False):
1955             if not self._downloader.params.get('noplaylist'):
1956                 multifeed_metadata_list = try_get(
1957                     player_response,
1958                     lambda x: x['multicamera']['playerLegacyMulticameraRenderer']['metadataList'],
1959                     compat_str) or try_get(
1960                     video_info, lambda x: x['multifeed_metadata_list'][0], compat_str)
1961                 if multifeed_metadata_list:
1962                     entries = []
1963                     feed_ids = []
1964                     for feed in multifeed_metadata_list.split(','):
1965                         # Unquote should take place before split on comma (,) since textual
1966                         # fields may contain comma as well (see
1967                         # https://github.com/ytdl-org/youtube-dl/issues/8536)
1968                         feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
1969
1970                         def feed_entry(name):
1971                             return try_get(feed_data, lambda x: x[name][0], compat_str)
1972
1973                         feed_id = feed_entry('id')
1974                         if not feed_id:
1975                             continue
1976                         feed_title = feed_entry('title')
1977                         title = video_title
1978                         if feed_title:
1979                             title += ' (%s)' % feed_title
1980                         entries.append({
1981                             '_type': 'url_transparent',
1982                             'ie_key': 'Youtube',
1983                             'url': smuggle_url(
1984                                 '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
1985                                 {'force_singlefeed': True}),
1986                             'title': title,
1987                         })
1988                         feed_ids.append(feed_id)
1989                     self.to_screen(
1990                         'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
1991                         % (', '.join(feed_ids), video_id))
1992                     return self.playlist_result(entries, video_id, video_title, video_description)
1993             else:
1994                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1995
1996         if view_count is None:
1997             view_count = extract_view_count(video_info)
1998         if view_count is None and video_details:
1999             view_count = int_or_none(video_details.get('viewCount'))
2000         if view_count is None and microformat:
2001             view_count = int_or_none(microformat.get('viewCount'))
2002
2003         if is_live is None:
2004             is_live = bool_or_none(video_details.get('isLive'))
2005
2006         # Check for "rental" videos
2007         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
2008             raise ExtractorError('"rental" videos not supported. See https://github.com/ytdl-org/youtube-dl/issues/359 for more information.', expected=True)
2009
2010         def _extract_filesize(media_url):
2011             return int_or_none(self._search_regex(
2012                 r'\bclen[=/](\d+)', media_url, 'filesize', default=None))
2013
2014         streaming_formats = try_get(player_response, lambda x: x['streamingData']['formats'], list) or []
2015         streaming_formats.extend(try_get(player_response, lambda x: x['streamingData']['adaptiveFormats'], list) or [])
2016
2017         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
2018             self.report_rtmp_download()
2019             formats = [{
2020                 'format_id': '_rtmp',
2021                 'protocol': 'rtmp',
2022                 'url': video_info['conn'][0],
2023                 'player_url': player_url,
2024             }]
2025         elif not is_live and (streaming_formats or len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1):
2026             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
2027             if 'rtmpe%3Dyes' in encoded_url_map:
2028                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/ytdl-org/youtube-dl/issues/343 for more information.', expected=True)
2029             formats = []
2030             formats_spec = {}
2031             fmt_list = video_info.get('fmt_list', [''])[0]
2032             if fmt_list:
2033                 for fmt in fmt_list.split(','):
2034                     spec = fmt.split('/')
2035                     if len(spec) > 1:
2036                         width_height = spec[1].split('x')
2037                         if len(width_height) == 2:
2038                             formats_spec[spec[0]] = {
2039                                 'resolution': spec[1],
2040                                 'width': int_or_none(width_height[0]),
2041                                 'height': int_or_none(width_height[1]),
2042                             }
2043             for fmt in streaming_formats:
2044                 itag = str_or_none(fmt.get('itag'))
2045                 if not itag:
2046                     continue
2047                 quality = fmt.get('quality')
2048                 quality_label = fmt.get('qualityLabel') or quality
2049                 formats_spec[itag] = {
2050                     'asr': int_or_none(fmt.get('audioSampleRate')),
2051                     'filesize': int_or_none(fmt.get('contentLength')),
2052                     'format_note': quality_label,
2053                     'fps': int_or_none(fmt.get('fps')),
2054                     'height': int_or_none(fmt.get('height')),
2055                     # bitrate for itag 43 is always 2147483647
2056                     'tbr': float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000) if itag != '43' else None,
2057                     'width': int_or_none(fmt.get('width')),
2058                 }
2059
2060             for fmt in streaming_formats:
2061                 if fmt.get('drmFamilies') or fmt.get('drm_families'):
2062                     continue
2063                 url = url_or_none(fmt.get('url'))
2064
2065                 if not url:
2066                     cipher = fmt.get('cipher') or fmt.get('signatureCipher')
2067                     if not cipher:
2068                         continue
2069                     url_data = compat_parse_qs(cipher)
2070                     url = url_or_none(try_get(url_data, lambda x: x['url'][0], compat_str))
2071                     if not url:
2072                         continue
2073                 else:
2074                     cipher = None
2075                     url_data = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
2076
2077                 stream_type = int_or_none(try_get(url_data, lambda x: x['stream_type'][0]))
2078                 # Unsupported FORMAT_STREAM_TYPE_OTF
2079                 if stream_type == 3:
2080                     continue
2081
2082                 format_id = fmt.get('itag') or url_data['itag'][0]
2083                 if not format_id:
2084                     continue
2085                 format_id = compat_str(format_id)
2086
2087                 if cipher:
2088                     if 's' in url_data or self._downloader.params.get('youtube_include_dash_manifest', True):
2089                         ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
2090                         jsplayer_url_json = self._search_regex(
2091                             ASSETS_RE,
2092                             embed_webpage if age_gate else video_webpage,
2093                             'JS player URL (1)', default=None)
2094                         if not jsplayer_url_json and not age_gate:
2095                             # We need the embed website after all
2096                             if embed_webpage is None:
2097                                 embed_url = proto + '://www.youtube.com/embed/%s' % video_id
2098                                 embed_webpage = self._download_webpage(
2099                                     embed_url, video_id, 'Downloading embed webpage')
2100                             jsplayer_url_json = self._search_regex(
2101                                 ASSETS_RE, embed_webpage, 'JS player URL')
2102
2103                         player_url = json.loads(jsplayer_url_json)
2104                         if player_url is None:
2105                             player_url_json = self._search_regex(
2106                                 r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
2107                                 video_webpage, 'age gate player URL')
2108                             player_url = json.loads(player_url_json)
2109
2110                     if 'sig' in url_data:
2111                         url += '&signature=' + url_data['sig'][0]
2112                     elif 's' in url_data:
2113                         encrypted_sig = url_data['s'][0]
2114
2115                         if self._downloader.params.get('verbose'):
2116                             if player_url is None:
2117                                 player_desc = 'unknown'
2118                             else:
2119                                 player_type, player_version = self._extract_player_info(player_url)
2120                                 player_desc = '%s player %s' % ('flash' if player_type == 'swf' else 'html5', player_version)
2121                             parts_sizes = self._signature_cache_id(encrypted_sig)
2122                             self.to_screen('{%s} signature length %s, %s' %
2123                                            (format_id, parts_sizes, player_desc))
2124
2125                         signature = self._decrypt_signature(
2126                             encrypted_sig, video_id, player_url, age_gate)
2127                         sp = try_get(url_data, lambda x: x['sp'][0], compat_str) or 'signature'
2128                         url += '&%s=%s' % (sp, signature)
2129                 if 'ratebypass' not in url:
2130                     url += '&ratebypass=yes'
2131
2132                 dct = {
2133                     'format_id': format_id,
2134                     'url': url,
2135                     'player_url': player_url,
2136                 }
2137                 if format_id in self._formats:
2138                     dct.update(self._formats[format_id])
2139                 if format_id in formats_spec:
2140                     dct.update(formats_spec[format_id])
2141
2142                 # Some itags are not included in DASH manifest thus corresponding formats will
2143                 # lack metadata (see https://github.com/ytdl-org/youtube-dl/pull/5993).
2144                 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
2145                 mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
2146                 width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
2147
2148                 if width is None:
2149                     width = int_or_none(fmt.get('width'))
2150                 if height is None:
2151                     height = int_or_none(fmt.get('height'))
2152
2153                 filesize = int_or_none(url_data.get(
2154                     'clen', [None])[0]) or _extract_filesize(url)
2155
2156                 quality = url_data.get('quality', [None])[0] or fmt.get('quality')
2157                 quality_label = url_data.get('quality_label', [None])[0] or fmt.get('qualityLabel')
2158
2159                 tbr = (float_or_none(url_data.get('bitrate', [None])[0], 1000)
2160                        or float_or_none(fmt.get('bitrate'), 1000)) if format_id != '43' else None
2161                 fps = int_or_none(url_data.get('fps', [None])[0]) or int_or_none(fmt.get('fps'))
2162
2163                 more_fields = {
2164                     'filesize': filesize,
2165                     'tbr': tbr,
2166                     'width': width,
2167                     'height': height,
2168                     'fps': fps,
2169                     'format_note': quality_label or quality,
2170                 }
2171                 for key, value in more_fields.items():
2172                     if value:
2173                         dct[key] = value
2174                 type_ = url_data.get('type', [None])[0] or fmt.get('mimeType')
2175                 if type_:
2176                     type_split = type_.split(';')
2177                     kind_ext = type_split[0].split('/')
2178                     if len(kind_ext) == 2:
2179                         kind, _ = kind_ext
2180                         dct['ext'] = mimetype2ext(type_split[0])
2181                         if kind in ('audio', 'video'):
2182                             codecs = None
2183                             for mobj in re.finditer(
2184                                     r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
2185                                 if mobj.group('key') == 'codecs':
2186                                     codecs = mobj.group('val')
2187                                     break
2188                             if codecs:
2189                                 dct.update(parse_codecs(codecs))
2190                 if dct.get('acodec') == 'none' or dct.get('vcodec') == 'none':
2191                     dct['downloader_options'] = {
2192                         # Youtube throttles chunks >~10M
2193                         'http_chunk_size': 10485760,
2194                     }
2195                 formats.append(dct)
2196         else:
2197             manifest_url = (
2198                 url_or_none(try_get(
2199                     player_response,
2200                     lambda x: x['streamingData']['hlsManifestUrl'],
2201                     compat_str))
2202                 or url_or_none(try_get(
2203                     video_info, lambda x: x['hlsvp'][0], compat_str)))
2204             if manifest_url:
2205                 formats = []
2206                 m3u8_formats = self._extract_m3u8_formats(
2207                     manifest_url, video_id, 'mp4', fatal=False)
2208                 for a_format in m3u8_formats:
2209                     itag = self._search_regex(
2210                         r'/itag/(\d+)/', a_format['url'], 'itag', default=None)
2211                     if itag:
2212                         a_format['format_id'] = itag
2213                         if itag in self._formats:
2214                             dct = self._formats[itag].copy()
2215                             dct.update(a_format)
2216                             a_format = dct
2217                     a_format['player_url'] = player_url
2218                     # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
2219                     a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
2220                     formats.append(a_format)
2221             else:
2222                 error_message = extract_unavailable_message()
2223                 if not error_message:
2224                     error_message = clean_html(try_get(
2225                         player_response, lambda x: x['playabilityStatus']['reason'],
2226                         compat_str))
2227                 if not error_message:
2228                     error_message = clean_html(
2229                         try_get(video_info, lambda x: x['reason'][0], compat_str))
2230                 if error_message:
2231                     raise ExtractorError(error_message, expected=True)
2232                 raise ExtractorError('no conn, hlsvp, hlsManifestUrl or url_encoded_fmt_stream_map information found in video info')
2233
2234         # uploader
2235         video_uploader = try_get(
2236             video_info, lambda x: x['author'][0],
2237             compat_str) or str_or_none(video_details.get('author'))
2238         if video_uploader:
2239             video_uploader = compat_urllib_parse_unquote_plus(video_uploader)
2240         else:
2241             self._downloader.report_warning('unable to extract uploader name')
2242
2243         # uploader_id
2244         video_uploader_id = None
2245         video_uploader_url = None
2246         mobj = re.search(
2247             r'<link itemprop="url" href="(?P<uploader_url>https?://www\.youtube\.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
2248             video_webpage)
2249         if mobj is not None:
2250             video_uploader_id = mobj.group('uploader_id')
2251             video_uploader_url = mobj.group('uploader_url')
2252         else:
2253             owner_profile_url = url_or_none(microformat.get('ownerProfileUrl'))
2254             if owner_profile_url:
2255                 video_uploader_id = self._search_regex(
2256                     r'(?:user|channel)/([^/]+)', owner_profile_url, 'uploader id',
2257                     default=None)
2258                 video_uploader_url = owner_profile_url
2259
2260         channel_id = (
2261             str_or_none(video_details.get('channelId'))
2262             or self._html_search_meta(
2263                 'channelId', video_webpage, 'channel id', default=None)
2264             or self._search_regex(
2265                 r'data-channel-external-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
2266                 video_webpage, 'channel id', default=None, group='id'))
2267         channel_url = 'http://www.youtube.com/channel/%s' % channel_id if channel_id else None
2268
2269         thumbnails = []
2270         thumbnails_list = try_get(
2271             video_details, lambda x: x['thumbnail']['thumbnails'], list) or []
2272         for t in thumbnails_list:
2273             if not isinstance(t, dict):
2274                 continue
2275             thumbnail_url = url_or_none(t.get('url'))
2276             if not thumbnail_url:
2277                 continue
2278             thumbnails.append({
2279                 'url': thumbnail_url,
2280                 'width': int_or_none(t.get('width')),
2281                 'height': int_or_none(t.get('height')),
2282             })
2283
2284         if not thumbnails:
2285             video_thumbnail = None
2286             # We try first to get a high quality image:
2287             m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
2288                                 video_webpage, re.DOTALL)
2289             if m_thumb is not None:
2290                 video_thumbnail = m_thumb.group(1)
2291             thumbnail_url = try_get(video_info, lambda x: x['thumbnail_url'][0], compat_str)
2292             if thumbnail_url:
2293                 video_thumbnail = compat_urllib_parse_unquote_plus(thumbnail_url)
2294             if video_thumbnail:
2295                 thumbnails.append({'url': video_thumbnail})
2296
2297         # upload date
2298         upload_date = self._html_search_meta(
2299             'datePublished', video_webpage, 'upload date', default=None)
2300         if not upload_date:
2301             upload_date = self._search_regex(
2302                 [r'(?s)id="eow-date.*?>(.*?)</span>',
2303                  r'(?:id="watch-uploader-info".*?>.*?|["\']simpleText["\']\s*:\s*["\'])(?:Published|Uploaded|Streamed live|Started) on (.+?)[<"\']'],
2304                 video_webpage, 'upload date', default=None)
2305         if not upload_date:
2306             upload_date = microformat.get('publishDate') or microformat.get('uploadDate')
2307         upload_date = unified_strdate(upload_date)
2308
2309         video_license = self._html_search_regex(
2310             r'<h4[^>]+class="title"[^>]*>\s*License\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li',
2311             video_webpage, 'license', default=None)
2312
2313         m_music = re.search(
2314             r'''(?x)
2315                 <h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*
2316                 <ul[^>]*>\s*
2317                 <li>(?P<title>.+?)
2318                 by (?P<creator>.+?)
2319                 (?:
2320                     \(.+?\)|
2321                     <a[^>]*
2322                         (?:
2323                             \bhref=["\']/red[^>]*>|             # drop possible
2324                             >\s*Listen ad-free with YouTube Red # YouTube Red ad
2325                         )
2326                     .*?
2327                 )?</li
2328             ''',
2329             video_webpage)
2330         if m_music:
2331             video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))
2332             video_creator = clean_html(m_music.group('creator'))
2333         else:
2334             video_alt_title = video_creator = None
2335
2336         def extract_meta(field):
2337             return self._html_search_regex(
2338                 r'<h4[^>]+class="title"[^>]*>\s*%s\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li>\s*' % field,
2339                 video_webpage, field, default=None)
2340
2341         track = extract_meta('Song')
2342         artist = extract_meta('Artist')
2343         album = extract_meta('Album')
2344
2345         # Youtube Music Auto-generated description
2346         release_date = release_year = None
2347         if video_description:
2348             mobj = re.search(r'(?s)Provided to YouTube by [^\n]+\n+(?P<track>[^·]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?', video_description)
2349             if mobj:
2350                 if not track:
2351                     track = mobj.group('track').strip()
2352                 if not artist:
2353                     artist = mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·'))
2354                 if not album:
2355                     album = mobj.group('album'.strip())
2356                 release_year = mobj.group('release_year')
2357                 release_date = mobj.group('release_date')
2358                 if release_date:
2359                     release_date = release_date.replace('-', '')
2360                     if not release_year:
2361                         release_year = int(release_date[:4])
2362                 if release_year:
2363                     release_year = int(release_year)
2364
2365         m_episode = re.search(
2366             r'<div[^>]+id="watch7-headline"[^>]*>\s*<span[^>]*>.*?>(?P<series>[^<]+)</a></b>\s*S(?P<season>\d+)\s*•\s*E(?P<episode>\d+)</span>',
2367             video_webpage)
2368         if m_episode:
2369             series = unescapeHTML(m_episode.group('series'))
2370             season_number = int(m_episode.group('season'))
2371             episode_number = int(m_episode.group('episode'))
2372         else:
2373             series = season_number = episode_number = None
2374
2375         m_cat_container = self._search_regex(
2376             r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
2377             video_webpage, 'categories', default=None)
2378         category = None
2379         if m_cat_container:
2380             category = self._html_search_regex(
2381                 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
2382                 default=None)
2383         if not category:
2384             category = try_get(
2385                 microformat, lambda x: x['category'], compat_str)
2386         video_categories = None if category is None else [category]
2387
2388         video_tags = [
2389             unescapeHTML(m.group('content'))
2390             for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
2391         if not video_tags:
2392             video_tags = try_get(video_details, lambda x: x['keywords'], list)
2393
2394         def _extract_count(count_name):
2395             return str_to_int(self._search_regex(
2396                 r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
2397                 % re.escape(count_name),
2398                 video_webpage, count_name, default=None))
2399
2400         like_count = _extract_count('like')
2401         dislike_count = _extract_count('dislike')
2402
2403         if view_count is None:
2404             view_count = str_to_int(self._search_regex(
2405                 r'<[^>]+class=["\']watch-view-count[^>]+>\s*([\d,\s]+)', video_webpage,
2406                 'view count', default=None))
2407
2408         average_rating = (
2409             float_or_none(video_details.get('averageRating'))
2410             or try_get(video_info, lambda x: float_or_none(x['avg_rating'][0])))
2411
2412         # subtitles
2413         video_subtitles = self.extract_subtitles(video_id, video_webpage)
2414         automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
2415
2416         video_duration = try_get(
2417             video_info, lambda x: int_or_none(x['length_seconds'][0]))
2418         if not video_duration:
2419             video_duration = int_or_none(video_details.get('lengthSeconds'))
2420         if not video_duration:
2421             video_duration = parse_duration(self._html_search_meta(
2422                 'duration', video_webpage, 'video duration'))
2423
2424         # annotations
2425         video_annotations = None
2426         if self._downloader.params.get('writeannotations', False):
2427             xsrf_token = self._search_regex(
2428                 r'([\'"])XSRF_TOKEN\1\s*:\s*([\'"])(?P<xsrf_token>[A-Za-z0-9+/=]+)\2',
2429                 video_webpage, 'xsrf token', group='xsrf_token', fatal=False)
2430             invideo_url = try_get(
2431                 player_response, lambda x: x['annotations'][0]['playerAnnotationsUrlsRenderer']['invideoUrl'], compat_str)
2432             if xsrf_token and invideo_url:
2433                 xsrf_field_name = self._search_regex(
2434                     r'([\'"])XSRF_FIELD_NAME\1\s*:\s*([\'"])(?P<xsrf_field_name>\w+)\2',
2435                     video_webpage, 'xsrf field name',
2436                     group='xsrf_field_name', default='session_token')
2437                 video_annotations = self._download_webpage(
2438                     self._proto_relative_url(invideo_url),
2439                     video_id, note='Downloading annotations',
2440                     errnote='Unable to download video annotations', fatal=False,
2441                     data=urlencode_postdata({xsrf_field_name: xsrf_token}))
2442
2443         chapters = self._extract_chapters(video_webpage, description_original, video_id, video_duration)
2444
2445         # Look for the DASH manifest
2446         if self._downloader.params.get('youtube_include_dash_manifest', True):
2447             dash_mpd_fatal = True
2448             for mpd_url in dash_mpds:
2449                 dash_formats = {}
2450                 try:
2451                     def decrypt_sig(mobj):
2452                         s = mobj.group(1)
2453                         dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
2454                         return '/signature/%s' % dec_s
2455
2456                     mpd_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, mpd_url)
2457
2458                     for df in self._extract_mpd_formats(
2459                             mpd_url, video_id, fatal=dash_mpd_fatal,
2460                             formats_dict=self._formats):
2461                         if not df.get('filesize'):
2462                             df['filesize'] = _extract_filesize(df['url'])
2463                         # Do not overwrite DASH format found in some previous DASH manifest
2464                         if df['format_id'] not in dash_formats:
2465                             dash_formats[df['format_id']] = df
2466                         # Additional DASH manifests may end up in HTTP Error 403 therefore
2467                         # allow them to fail without bug report message if we already have
2468                         # some DASH manifest succeeded. This is temporary workaround to reduce
2469                         # burst of bug reports until we figure out the reason and whether it
2470                         # can be fixed at all.
2471                         dash_mpd_fatal = False
2472                 except (ExtractorError, KeyError) as e:
2473                     self.report_warning(
2474                         'Skipping DASH manifest: %r' % e, video_id)
2475                 if dash_formats:
2476                     # Remove the formats we found through non-DASH, they
2477                     # contain less info and it can be wrong, because we use
2478                     # fixed values (for example the resolution). See
2479                     # https://github.com/ytdl-org/youtube-dl/issues/5774 for an
2480                     # example.
2481                     formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
2482                     formats.extend(dash_formats.values())
2483
2484         # Check for malformed aspect ratio
2485         stretched_m = re.search(
2486             r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
2487             video_webpage)
2488         if stretched_m:
2489             w = float(stretched_m.group('w'))
2490             h = float(stretched_m.group('h'))
2491             # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
2492             # We will only process correct ratios.
2493             if w > 0 and h > 0:
2494                 ratio = w / h
2495                 for f in formats:
2496                     if f.get('vcodec') != 'none':
2497                         f['stretched_ratio'] = ratio
2498
2499         if not formats:
2500             if 'reason' in video_info:
2501                 if 'The uploader has not made this video available in your country.' in video_info['reason']:
2502                     regions_allowed = self._html_search_meta(
2503                         'regionsAllowed', video_webpage, default=None)
2504                     countries = regions_allowed.split(',') if regions_allowed else None
2505                     self.raise_geo_restricted(
2506                         msg=video_info['reason'][0], countries=countries)
2507                 reason = video_info['reason'][0]
2508                 if 'Invalid parameters' in reason:
2509                     unavailable_message = extract_unavailable_message()
2510                     if unavailable_message:
2511                         reason = unavailable_message
2512                 raise ExtractorError(
2513                     'YouTube said: %s' % reason,
2514                     expected=True, video_id=video_id)
2515             if video_info.get('license_info') or try_get(player_response, lambda x: x['streamingData']['licenseInfos']):
2516                 raise ExtractorError('This video is DRM protected.', expected=True)
2517
2518         self._sort_formats(formats)
2519
2520         self.mark_watched(video_id, video_info, player_response)
2521
2522         return {
2523             'id': video_id,
2524             'uploader': video_uploader,
2525             'uploader_id': video_uploader_id,
2526             'uploader_url': video_uploader_url,
2527             'channel_id': channel_id,
2528             'channel_url': channel_url,
2529             'upload_date': upload_date,
2530             'license': video_license,
2531             'creator': video_creator or artist,
2532             'title': video_title,
2533             'alt_title': video_alt_title or track,
2534             'thumbnails': thumbnails,
2535             'description': video_description,
2536             'categories': video_categories,
2537             'tags': video_tags,
2538             'subtitles': video_subtitles,
2539             'automatic_captions': automatic_captions,
2540             'duration': video_duration,
2541             'age_limit': 18 if age_gate else 0,
2542             'annotations': video_annotations,
2543             'chapters': chapters,
2544             'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
2545             'view_count': view_count,
2546             'like_count': like_count,
2547             'dislike_count': dislike_count,
2548             'average_rating': average_rating,
2549             'formats': formats,
2550             'is_live': is_live,
2551             'start_time': start_time,
2552             'end_time': end_time,
2553             'series': series,
2554             'season_number': season_number,
2555             'episode_number': episode_number,
2556             'track': track,
2557             'artist': artist,
2558             'album': album,
2559             'release_date': release_date,
2560             'release_year': release_year,
2561         }
2562
2563
2564 class YoutubePlaylistIE(YoutubePlaylistBaseInfoExtractor):
2565     IE_DESC = 'YouTube.com playlists'
2566     _VALID_URL = r"""(?x)(?:
2567                         (?:https?://)?
2568                         (?:\w+\.)?
2569                         (?:
2570                             (?:
2571                                 youtube(?:kids)?\.com|
2572                                 invidio\.us
2573                             )
2574                             /
2575                             (?:
2576                                (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/(?:videoseries|[0-9A-Za-z_-]{11}))
2577                                \? (?:.*?[&;])*? (?:p|a|list)=
2578                             |  p/
2579                             )|
2580                             youtu\.be/[0-9A-Za-z_-]{11}\?.*?\blist=
2581                         )
2582                         (
2583                             (?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)?[0-9A-Za-z-_]{10,}
2584                             # Top tracks, they can also include dots
2585                             |(?:MC)[\w\.]*
2586                         )
2587                         .*
2588                      |
2589                         (%(playlist_id)s)
2590                      )""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
2591     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
2592     _VIDEO_RE_TPL = r'href="\s*/watch\?v=%s(?:&amp;(?:[^"]*?index=(?P<index>\d+))?(?:[^>]+>(?P<title>[^<]+))?)?'
2593     _VIDEO_RE = _VIDEO_RE_TPL % r'(?P<id>[0-9A-Za-z_-]{11})'
2594     IE_NAME = 'youtube:playlist'
2595     _TESTS = [{
2596         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
2597         'info_dict': {
2598             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
2599             'uploader': 'Sergey M.',
2600             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
2601             'title': 'youtube-dl public playlist',
2602         },
2603         'playlist_count': 1,
2604     }, {
2605         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
2606         'info_dict': {
2607             'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
2608             'uploader': 'Sergey M.',
2609             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
2610             'title': 'youtube-dl empty playlist',
2611         },
2612         'playlist_count': 0,
2613     }, {
2614         'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
2615         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2616         'info_dict': {
2617             'title': '29C3: Not my department',
2618             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
2619             'uploader': 'Christiaan008',
2620             'uploader_id': 'ChRiStIaAn008',
2621         },
2622         'playlist_count': 96,
2623     }, {
2624         'note': 'issue #673',
2625         'url': 'PLBB231211A4F62143',
2626         'info_dict': {
2627             'title': '[OLD]Team Fortress 2 (Class-based LP)',
2628             'id': 'PLBB231211A4F62143',
2629             'uploader': 'Wickydoo',
2630             'uploader_id': 'Wickydoo',
2631         },
2632         'playlist_mincount': 26,
2633     }, {
2634         'note': 'Large playlist',
2635         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
2636         'info_dict': {
2637             'title': 'Uploads from Cauchemar',
2638             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
2639             'uploader': 'Cauchemar',
2640             'uploader_id': 'Cauchemar89',
2641         },
2642         'playlist_mincount': 799,
2643     }, {
2644         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2645         'info_dict': {
2646             'title': 'YDL_safe_search',
2647             'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
2648         },
2649         'playlist_count': 2,
2650         'skip': 'This playlist is private',
2651     }, {
2652         'note': 'embedded',
2653         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2654         'playlist_count': 4,
2655         'info_dict': {
2656             'title': 'JODA15',
2657             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
2658             'uploader': 'milan',
2659             'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
2660         }
2661     }, {
2662         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2663         'playlist_mincount': 485,
2664         'info_dict': {
2665             'title': '2018 Chinese New Singles (11/6 updated)',
2666             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
2667             'uploader': 'LBK',
2668             'uploader_id': 'sdragonfang',
2669         }
2670     }, {
2671         'note': 'Embedded SWF player',
2672         'url': 'https://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
2673         'playlist_count': 4,
2674         'info_dict': {
2675             'title': 'JODA7',
2676             'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
2677         },
2678         'skip': 'This playlist does not exist',
2679     }, {
2680         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
2681         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
2682         'info_dict': {
2683             'title': 'Uploads from Interstellar Movie',
2684             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
2685             'uploader': 'Interstellar Movie',
2686             'uploader_id': 'InterstellarMovie1',
2687         },
2688         'playlist_mincount': 21,
2689     }, {
2690         # Playlist URL that does not actually serve a playlist
2691         'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
2692         'info_dict': {
2693             'id': 'FqZTN594JQw',
2694             'ext': 'webm',
2695             'title': "Smiley's People 01 detective, Adventure Series, Action",
2696             'uploader': 'STREEM',
2697             'uploader_id': 'UCyPhqAZgwYWZfxElWVbVJng',
2698             'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCyPhqAZgwYWZfxElWVbVJng',
2699             'upload_date': '20150526',
2700             'license': 'Standard YouTube License',
2701             'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
2702             'categories': ['People & Blogs'],
2703             'tags': list,
2704             'view_count': int,
2705             'like_count': int,
2706             'dislike_count': int,
2707         },
2708         'params': {
2709             'skip_download': True,
2710         },
2711         'skip': 'This video is not available.',
2712         'add_ie': [YoutubeIE.ie_key()],
2713     }, {
2714         'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
2715         'info_dict': {
2716             'id': 'yeWKywCrFtk',
2717             'ext': 'mp4',
2718             'title': 'Small Scale Baler and Braiding Rugs',
2719             'uploader': 'Backus-Page House Museum',
2720             'uploader_id': 'backuspagemuseum',
2721             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
2722             'upload_date': '20161008',
2723             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
2724             'categories': ['Nonprofits & Activism'],
2725             'tags': list,
2726             'like_count': int,
2727             'dislike_count': int,
2728         },
2729         'params': {
2730             'noplaylist': True,
2731             'skip_download': True,
2732         },
2733     }, {
2734         # https://github.com/ytdl-org/youtube-dl/issues/21844
2735         'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
2736         'info_dict': {
2737             'title': 'Data Analysis with Dr Mike Pound',
2738             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
2739             'uploader_id': 'Computerphile',
2740             'uploader': 'Computerphile',
2741         },
2742         'playlist_mincount': 11,
2743     }, {
2744         'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
2745         'only_matching': True,
2746     }, {
2747         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
2748         'only_matching': True,
2749     }, {
2750         # music album playlist
2751         'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
2752         'only_matching': True,
2753     }, {
2754         'url': 'https://invidio.us/playlist?list=PLDIoUOhQQPlXr63I_vwF9GD8sAKh77dWU',
2755         'only_matching': True,
2756     }, {
2757         'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
2758         'only_matching': True,
2759     }]
2760
2761     def _real_initialize(self):
2762         self._login()
2763
2764     def extract_videos_from_page(self, page):
2765         ids_in_page = []
2766         titles_in_page = []
2767
2768         for item in re.findall(
2769                 r'(<[^>]*\bdata-video-id\s*=\s*["\'][0-9A-Za-z_-]{11}[^>]+>)', page):
2770             attrs = extract_attributes(item)
2771             video_id = attrs['data-video-id']
2772             video_title = unescapeHTML(attrs.get('data-title'))
2773             if video_title:
2774                 video_title = video_title.strip()
2775             ids_in_page.append(video_id)
2776             titles_in_page.append(video_title)
2777
2778         # Fallback with old _VIDEO_RE
2779         self.extract_videos_from_page_impl(
2780             self._VIDEO_RE, page, ids_in_page, titles_in_page)
2781
2782         # Relaxed fallbacks
2783         self.extract_videos_from_page_impl(
2784             r'href="\s*/watch\?v\s*=\s*(?P<id>[0-9A-Za-z_-]{11})', page,
2785             ids_in_page, titles_in_page)
2786         self.extract_videos_from_page_impl(
2787             r'data-video-ids\s*=\s*["\'](?P<id>[0-9A-Za-z_-]{11})', page,
2788             ids_in_page, titles_in_page)
2789
2790         return zip(ids_in_page, titles_in_page)
2791
2792     def _extract_mix(self, playlist_id):
2793         # The mixes are generated from a single video
2794         # the id of the playlist is just 'RD' + video_id
2795         ids = []
2796         last_id = playlist_id[-11:]
2797         for n in itertools.count(1):
2798             url = 'https://www.youtube.com/watch?v=%s&list=%s' % (last_id, playlist_id)
2799             webpage = self._download_webpage(
2800                 url, playlist_id, 'Downloading page {0} of Youtube mix'.format(n))
2801             new_ids = orderedSet(re.findall(
2802                 r'''(?xs)data-video-username=".*?".*?
2803                            href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s''' % re.escape(playlist_id),
2804                 webpage))
2805             # Fetch new pages until all the videos are repeated, it seems that
2806             # there are always 51 unique videos.
2807             new_ids = [_id for _id in new_ids if _id not in ids]
2808             if not new_ids:
2809                 break
2810             ids.extend(new_ids)
2811             last_id = ids[-1]
2812
2813         url_results = self._ids_to_results(ids)
2814
2815         search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
2816         title_span = (
2817             search_title('playlist-title')
2818             or search_title('title long-title')
2819             or search_title('title'))
2820         title = clean_html(title_span)
2821
2822         return self.playlist_result(url_results, playlist_id, title)
2823
2824     def _extract_playlist(self, playlist_id):
2825         url = self._TEMPLATE_URL % playlist_id
2826         page = self._download_webpage(url, playlist_id)
2827
2828         # the yt-alert-message now has tabindex attribute (see https://github.com/ytdl-org/youtube-dl/issues/11604)
2829         for match in re.findall(r'<div class="yt-alert-message"[^>]*>([^<]+)</div>', page):
2830             match = match.strip()
2831             # Check if the playlist exists or is private
2832             mobj = re.match(r'[^<]*(?:The|This) playlist (?P<reason>does not exist|is private)[^<]*', match)
2833             if mobj:
2834                 reason = mobj.group('reason')
2835                 message = 'This playlist %s' % reason
2836                 if 'private' in reason:
2837                     message += ', use --username or --netrc to access it'
2838                 message += '.'
2839                 raise ExtractorError(message, expected=True)
2840             elif re.match(r'[^<]*Invalid parameters[^<]*', match):
2841                 raise ExtractorError(
2842                     'Invalid parameters. Maybe URL is incorrect.',
2843                     expected=True)
2844             elif re.match(r'[^<]*Choose your language[^<]*', match):
2845                 continue
2846             else:
2847                 self.report_warning('Youtube gives an alert message: ' + match)
2848
2849         playlist_title = self._html_search_regex(
2850             r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
2851             page, 'title', default=None)
2852
2853         _UPLOADER_BASE = r'class=["\']pl-header-details[^>]+>\s*<li>\s*<a[^>]+\bhref='
2854         uploader = self._html_search_regex(
2855             r'%s["\']/(?:user|channel)/[^>]+>([^<]+)' % _UPLOADER_BASE,
2856             page, 'uploader', default=None)
2857         mobj = re.search(
2858             r'%s(["\'])(?P<path>/(?:user|channel)/(?P<uploader_id>.+?))\1' % _UPLOADER_BASE,
2859             page)
2860         if mobj:
2861             uploader_id = mobj.group('uploader_id')
2862             uploader_url = compat_urlparse.urljoin(url, mobj.group('path'))
2863         else:
2864             uploader_id = uploader_url = None
2865
2866         has_videos = True
2867
2868         if not playlist_title:
2869             try:
2870                 # Some playlist URLs don't actually serve a playlist (e.g.
2871                 # https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4)
2872                 next(self._entries(page, playlist_id))
2873             except StopIteration:
2874                 has_videos = False
2875
2876         playlist = self.playlist_result(
2877             self._entries(page, playlist_id), playlist_id, playlist_title)
2878         playlist.update({
2879             'uploader': uploader,
2880             'uploader_id': uploader_id,
2881             'uploader_url': uploader_url,
2882         })
2883
2884         return has_videos, playlist
2885
2886     def _check_download_just_video(self, url, playlist_id):
2887         # Check if it's a video-specific URL
2888         query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
2889         video_id = query_dict.get('v', [None])[0] or self._search_regex(
2890             r'(?:(?:^|//)youtu\.be/|youtube\.com/embed/(?!videoseries))([0-9A-Za-z_-]{11})', url,
2891             'video id', default=None)
2892         if video_id:
2893             if self._downloader.params.get('noplaylist'):
2894                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
2895                 return video_id, self.url_result(video_id, 'Youtube', video_id=video_id)
2896             else:
2897                 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
2898                 return video_id, None
2899         return None, None
2900
2901     def _real_extract(self, url):
2902         # Extract playlist id
2903         mobj = re.match(self._VALID_URL, url)
2904         if mobj is None:
2905             raise ExtractorError('Invalid URL: %s' % url)
2906         playlist_id = mobj.group(1) or mobj.group(2)
2907
2908         video_id, video = self._check_download_just_video(url, playlist_id)
2909         if video:
2910             return video
2911
2912         if playlist_id.startswith(('RD', 'UL', 'PU')):
2913             # Mixes require a custom extraction process
2914             return self._extract_mix(playlist_id)
2915
2916         has_videos, playlist = self._extract_playlist(playlist_id)
2917         if has_videos or not video_id:
2918             return playlist
2919
2920         # Some playlist URLs don't actually serve a playlist (see
2921         # https://github.com/ytdl-org/youtube-dl/issues/10537).
2922         # Fallback to plain video extraction if there is a video id
2923         # along with playlist id.
2924         return self.url_result(video_id, 'Youtube', video_id=video_id)
2925
2926
2927 class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
2928     IE_DESC = 'YouTube.com channels'
2929     _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie|kids)?\.com|(?:www\.)?invidio\.us)/channel/(?P<id>[0-9A-Za-z_-]+)'
2930     _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
2931     _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
2932     IE_NAME = 'youtube:channel'
2933     _TESTS = [{
2934         'note': 'paginated channel',
2935         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
2936         'playlist_mincount': 91,
2937         'info_dict': {
2938             'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
2939             'title': 'Uploads from lex will',
2940             'uploader': 'lex will',
2941             'uploader_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
2942         }
2943     }, {
2944         'note': 'Age restricted channel',
2945         # from https://www.youtube.com/user/DeusExOfficial
2946         'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
2947         'playlist_mincount': 64,
2948         'info_dict': {
2949             'id': 'UUs0ifCMCm1icqRbqhUINa0w',
2950             'title': 'Uploads from Deus Ex',
2951             'uploader': 'Deus Ex',
2952             'uploader_id': 'DeusExOfficial',
2953         },
2954     }, {
2955         'url': 'https://invidio.us/channel/UC23qupoDRn9YOAVzeoxjOQA',
2956         'only_matching': True,
2957     }, {
2958         'url': 'https://www.youtubekids.com/channel/UCyu8StPfZWapR6rfW_JgqcA',
2959         'only_matching': True,
2960     }]
2961
2962     @classmethod
2963     def suitable(cls, url):
2964         return (False if YoutubePlaylistsIE.suitable(url) or YoutubeLiveIE.suitable(url)
2965                 else super(YoutubeChannelIE, cls).suitable(url))
2966
2967     def _build_template_url(self, url, channel_id):
2968         return self._TEMPLATE_URL % channel_id
2969
2970     def _real_extract(self, url):
2971         channel_id = self._match_id(url)
2972
2973         url = self._build_template_url(url, channel_id)
2974
2975         # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
2976         # Workaround by extracting as a playlist if managed to obtain channel playlist URL
2977         # otherwise fallback on channel by page extraction
2978         channel_page = self._download_webpage(
2979             url + '?view=57', channel_id,
2980             'Downloading channel page', fatal=False)
2981         if channel_page is False:
2982             channel_playlist_id = False
2983         else:
2984             channel_playlist_id = self._html_search_meta(
2985                 'channelId', channel_page, 'channel id', default=None)
2986             if not channel_playlist_id:
2987                 channel_url = self._html_search_meta(
2988                     ('al:ios:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad'),
2989                     channel_page, 'channel url', default=None)
2990                 if channel_url:
2991                     channel_playlist_id = self._search_regex(
2992                         r'vnd\.youtube://user/([0-9A-Za-z_-]+)',
2993                         channel_url, 'channel id', default=None)
2994         if channel_playlist_id and channel_playlist_id.startswith('UC'):
2995             playlist_id = 'UU' + channel_playlist_id[2:]
2996             return self.url_result(
2997                 compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')
2998
2999         channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
3000         autogenerated = re.search(r'''(?x)
3001                 class="[^"]*?(?:
3002                     channel-header-autogenerated-label|
3003                     yt-channel-title-autogenerated
3004                 )[^"]*"''', channel_page) is not None
3005
3006         if autogenerated:
3007             # The videos are contained in a single page
3008             # the ajax pages can't be used, they are empty
3009             entries = [
3010                 self.url_result(
3011                     video_id, 'Youtube', video_id=video_id,
3012                     video_title=video_title)
3013                 for video_id, video_title in self.extract_videos_from_page(channel_page)]
3014             return self.playlist_result(entries, channel_id)
3015
3016         try:
3017             next(self._entries(channel_page, channel_id))
3018         except StopIteration:
3019             alert_message = self._html_search_regex(
3020                 r'(?s)<div[^>]+class=(["\']).*?\byt-alert-message\b.*?\1[^>]*>(?P<alert>[^<]+)</div>',
3021                 channel_page, 'alert', default=None, group='alert')
3022             if alert_message:
3023                 raise ExtractorError('Youtube said: %s' % alert_message, expected=True)
3024
3025         return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
3026
3027
3028 class YoutubeUserIE(YoutubeChannelIE):
3029     IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
3030     _VALID_URL = r'(?:(?:https?://(?:\w+\.)?youtube\.com/(?:(?P<user>user|c)/)?(?!(?:attribution_link|watch|results|shared)(?:$|[^a-z_A-Z0-9%-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_%-]+)'
3031     _TEMPLATE_URL = 'https://www.youtube.com/%s/%s/videos'
3032     IE_NAME = 'youtube:user'
3033
3034     _TESTS = [{
3035         'url': 'https://www.youtube.com/user/TheLinuxFoundation',
3036         'playlist_mincount': 320,
3037         'info_dict': {
3038             'id': 'UUfX55Sx5hEFjoC3cNs6mCUQ',
3039             'title': 'Uploads from The Linux Foundation',
3040             'uploader': 'The Linux Foundation',
3041             'uploader_id': 'TheLinuxFoundation',
3042         }
3043     }, {
3044         # Only available via https://www.youtube.com/c/12minuteathlete/videos
3045         # but not https://www.youtube.com/user/12minuteathlete/videos
3046         'url': 'https://www.youtube.com/c/12minuteathlete/videos',
3047         'playlist_mincount': 249,
3048         'info_dict': {
3049             'id': 'UUVjM-zV6_opMDx7WYxnjZiQ',
3050             'title': 'Uploads from 12 Minute Athlete',
3051             'uploader': '12 Minute Athlete',
3052             'uploader_id': 'the12minuteathlete',
3053         }
3054     }, {
3055         'url': 'ytuser:phihag',
3056         'only_matching': True,
3057     }, {
3058         'url': 'https://www.youtube.com/c/gametrailers',
3059         'only_matching': True,
3060     }, {
3061         'url': 'https://www.youtube.com/c/Pawe%C5%82Zadro%C5%BCniak',
3062         'only_matching': True,
3063     }, {
3064         'url': 'https://www.youtube.com/gametrailers',
3065         'only_matching': True,
3066     }, {
3067         # This channel is not available, geo restricted to JP
3068         'url': 'https://www.youtube.com/user/kananishinoSMEJ/videos',
3069         'only_matching': True,
3070     }]
3071
3072     @classmethod
3073     def suitable(cls, url):
3074         # Don't return True if the url can be extracted with other youtube
3075         # extractor, the regex would is too permissive and it would match.
3076         other_yt_ies = iter(klass for (name, klass) in globals().items() if name.startswith('Youtube') and name.endswith('IE') and klass is not cls)
3077         if any(ie.suitable(url) for ie in other_yt_ies):
3078             return False
3079         else:
3080             return super(YoutubeUserIE, cls).suitable(url)
3081
3082     def _build_template_url(self, url, channel_id):
3083         mobj = re.match(self._VALID_URL, url)
3084         return self._TEMPLATE_URL % (mobj.group('user') or 'user', mobj.group('id'))
3085
3086
3087 class YoutubeLiveIE(YoutubeBaseInfoExtractor):
3088     IE_DESC = 'YouTube.com live streams'
3089     _VALID_URL = r'(?P<base_url>https?://(?:\w+\.)?youtube\.com/(?:(?:user|channel|c)/)?(?P<id>[^/]+))/live'
3090     IE_NAME = 'youtube:live'
3091
3092     _TESTS = [{
3093         'url': 'https://www.youtube.com/user/TheYoungTurks/live',
3094         'info_dict': {
3095             'id': 'a48o2S1cPoo',
3096             'ext': 'mp4',
3097             'title': 'The Young Turks - Live Main Show',
3098             'uploader': 'The Young Turks',
3099             'uploader_id': 'TheYoungTurks',
3100             'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheYoungTurks',
3101             'upload_date': '20150715',
3102             'license': 'Standard YouTube License',
3103             'description': 'md5:438179573adcdff3c97ebb1ee632b891',
3104             'categories': ['News & Politics'],
3105             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
3106             'like_count': int,
3107             'dislike_count': int,
3108         },
3109         'params': {
3110             'skip_download': True,
3111         },
3112     }, {
3113         'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
3114         'only_matching': True,
3115     }, {
3116         'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
3117         'only_matching': True,
3118     }, {
3119         'url': 'https://www.youtube.com/TheYoungTurks/live',
3120         'only_matching': True,
3121     }]
3122
3123     def _real_extract(self, url):
3124         mobj = re.match(self._VALID_URL, url)
3125         channel_id = mobj.group('id')
3126         base_url = mobj.group('base_url')
3127         webpage = self._download_webpage(url, channel_id, fatal=False)
3128         if webpage:
3129             page_type = self._og_search_property(
3130                 'type', webpage, 'page type', default='')
3131             video_id = self._html_search_meta(
3132                 'videoId', webpage, 'video id', default=None)
3133             if page_type.startswith('video') and video_id and re.match(
3134                     r'^[0-9A-Za-z_-]{11}$', video_id):
3135                 return self.url_result(video_id, YoutubeIE.ie_key())
3136         return self.url_result(base_url)
3137
3138
3139 class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
3140     IE_DESC = 'YouTube.com user/channel playlists'
3141     _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel|c)/(?P<id>[^/]+)/playlists'
3142     IE_NAME = 'youtube:playlists'
3143
3144     _TESTS = [{
3145         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
3146         'playlist_mincount': 4,
3147         'info_dict': {
3148             'id': 'ThirstForScience',
3149             'title': 'ThirstForScience',
3150         },
3151     }, {
3152         # with "Load more" button
3153         'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
3154         'playlist_mincount': 70,
3155         'info_dict': {
3156             'id': 'igorkle1',
3157             'title': 'Игорь Клейнер',
3158         },
3159     }, {
3160         'url': 'https://www.youtube.com/channel/UCiU1dHvZObB2iP6xkJ__Icw/playlists',
3161         'playlist_mincount': 17,
3162         'info_dict': {
3163             'id': 'UCiU1dHvZObB2iP6xkJ__Icw',
3164             'title': 'Chem Player',
3165         },
3166         'skip': 'Blocked',
3167     }, {
3168         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
3169         'only_matching': True,
3170     }]
3171
3172
3173 class YoutubeSearchBaseInfoExtractor(YoutubePlaylistBaseInfoExtractor):
3174     _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})(?:[^"]*"[^>]+\btitle="(?P<title>[^"]+))?'
3175
3176
3177 class YoutubeSearchIE(SearchInfoExtractor, YoutubeSearchBaseInfoExtractor):
3178     IE_DESC = 'YouTube.com searches'
3179     # there doesn't appear to be a real limit, for example if you search for
3180     # 'python' you get more than 8.000.000 results
3181     _MAX_RESULTS = float('inf')
3182     IE_NAME = 'youtube:search'
3183     _SEARCH_KEY = 'ytsearch'
3184     _SEARCH_PARAMS = None
3185     _TESTS = []
3186
3187     def _entries(self, query, n):
3188         data = {
3189             'context': {
3190                 'client': {
3191                     'clientName': 'WEB',
3192                     'clientVersion': '2.20201021.03.00',
3193                 }
3194             },
3195             'query': query,
3196         }
3197         if self._SEARCH_PARAMS:
3198             data['params'] = self._SEARCH_PARAMS
3199         total = 0
3200         for page_num in itertools.count(1):
3201             search = self._download_json(
3202                 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
3203                 video_id='query "%s"' % query,
3204                 note='Downloading page %s' % page_num,
3205                 errnote='Unable to download API page', fatal=False,
3206                 data=json.dumps(data).encode('utf8'),
3207                 headers={'content-type': 'application/json'})
3208             if not search:
3209                 break
3210             slr_contents = try_get(
3211                 search,
3212                 (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
3213                  lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
3214                 list)
3215             if not slr_contents:
3216                 break
3217             isr_contents = try_get(
3218                 slr_contents,
3219                 lambda x: x[0]['itemSectionRenderer']['contents'],
3220                 list)
3221             if not isr_contents:
3222                 break
3223             for content in isr_contents:
3224                 if not isinstance(content, dict):
3225                     continue
3226                 video = content.get('videoRenderer')
3227                 if not isinstance(video, dict):
3228                     continue
3229                 video_id = video.get('videoId')
3230                 if not video_id:
3231                     continue
3232                 title = try_get(video, lambda x: x['title']['runs'][0]['text'], compat_str)
3233                 description = try_get(video, lambda x: x['descriptionSnippet']['runs'][0]['text'], compat_str)
3234                 duration = parse_duration(try_get(video, lambda x: x['lengthText']['simpleText'], compat_str))
3235                 view_count_text = try_get(video, lambda x: x['viewCountText']['simpleText'], compat_str) or ''
3236                 view_count = int_or_none(self._search_regex(
3237                     r'^(\d+)', re.sub(r'\s', '', view_count_text),
3238                     'view count', default=None))
3239                 uploader = try_get(video, lambda x: x['ownerText']['runs'][0]['text'], compat_str)
3240                 total += 1
3241                 yield {
3242                     '_type': 'url_transparent',
3243                     'ie_key': YoutubeIE.ie_key(),
3244                     'id': video_id,
3245                     'url': video_id,
3246                     'title': title,
3247                     'description': description,
3248                     'duration': duration,
3249                     'view_count': view_count,
3250                     'uploader': uploader,
3251                 }
3252                 if total == n:
3253                     return
3254             token = try_get(
3255                 slr_contents,
3256                 lambda x: x[1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'],
3257                 compat_str)
3258             if not token:
3259                 break
3260             data['continuation'] = token
3261
3262     def _get_n_results(self, query, n):
3263         """Get a specified number of results for a query"""
3264         return self.playlist_result(self._entries(query, n), query)
3265
3266
3267 class YoutubeSearchDateIE(YoutubeSearchIE):
3268     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
3269     _SEARCH_KEY = 'ytsearchdate'
3270     IE_DESC = 'YouTube.com searches, newest videos first'
3271     _SEARCH_PARAMS = 'CAI%3D'
3272
3273
3274 class YoutubeSearchURLIE(YoutubeSearchBaseInfoExtractor):
3275     IE_DESC = 'YouTube.com search URLs'
3276     IE_NAME = 'youtube:search_url'
3277     _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
3278     _TESTS = [{
3279         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
3280         'playlist_mincount': 5,
3281         'info_dict': {
3282             'title': 'youtube-dl test video',
3283         }
3284     }, {
3285         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
3286         'only_matching': True,
3287     }]
3288
3289     def _real_extract(self, url):
3290         mobj = re.match(self._VALID_URL, url)
3291         query = compat_urllib_parse_unquote_plus(mobj.group('query'))
3292         webpage = self._download_webpage(url, query)
3293         return self.playlist_result(self._process_page(webpage), playlist_title=query)
3294
3295
3296 class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
3297     IE_DESC = 'YouTube.com (multi-season) shows'
3298     _VALID_URL = r'https?://(?:www\.)?youtube\.com/show/(?P<id>[^?#]*)'
3299     IE_NAME = 'youtube:show'
3300     _TESTS = [{
3301         'url': 'https://www.youtube.com/show/airdisasters',
3302         'playlist_mincount': 5,
3303         'info_dict': {
3304             'id': 'airdisasters',
3305             'title': 'Air Disasters',
3306         }
3307     }]
3308
3309     def _real_extract(self, url):
3310         playlist_id = self._match_id(url)
3311         return super(YoutubeShowIE, self)._real_extract(
3312             'https://www.youtube.com/show/%s/playlists' % playlist_id)
3313
3314
3315 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
3316     """
3317     Base class for feed extractors
3318     Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
3319     """
3320     _LOGIN_REQUIRED = True
3321
3322     @property
3323     def IE_NAME(self):
3324         return 'youtube:%s' % self._FEED_NAME
3325
3326     def _real_initialize(self):
3327         self._login()
3328
3329     def _entries(self, page):
3330         # The extraction process is the same as for playlists, but the regex
3331         # for the video ids doesn't contain an index
3332         ids = []
3333         more_widget_html = content_html = page
3334         for page_num in itertools.count(1):
3335             matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
3336
3337             # 'recommended' feed has infinite 'load more' and each new portion spins
3338             # the same videos in (sometimes) slightly different order, so we'll check
3339             # for unicity and break when portion has no new videos
3340             new_ids = list(filter(lambda video_id: video_id not in ids, orderedSet(matches)))
3341             if not new_ids:
3342                 break
3343
3344             ids.extend(new_ids)
3345
3346             for entry in self._ids_to_results(new_ids):
3347                 yield entry
3348
3349             mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
3350             if not mobj:
3351                 break
3352
3353             more = self._download_json(
3354                 'https://www.youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
3355                 'Downloading page #%s' % page_num,
3356                 transform_source=uppercase_escape,
3357                 headers=self._YOUTUBE_CLIENT_HEADERS)
3358             content_html = more['content_html']
3359             more_widget_html = more['load_more_widget_html']
3360
3361     def _real_extract(self, url):
3362         page = self._download_webpage(
3363             'https://www.youtube.com/feed/%s' % self._FEED_NAME,
3364             self._PLAYLIST_TITLE)
3365         return self.playlist_result(
3366             self._entries(page), playlist_title=self._PLAYLIST_TITLE)
3367
3368
3369 class YoutubeWatchLaterIE(YoutubePlaylistIE):
3370     IE_NAME = 'youtube:watchlater'
3371     IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
3372     _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:feed/watch_later|(?:playlist|watch)\?(?:.+&)?list=WL)|:ytwatchlater'
3373
3374     _TESTS = [{
3375         'url': 'https://www.youtube.com/playlist?list=WL',
3376         'only_matching': True,
3377     }, {
3378         'url': 'https://www.youtube.com/watch?v=bCNU9TrbiRk&index=1&list=WL',
3379         'only_matching': True,
3380     }]
3381
3382     def _real_extract(self, url):
3383         _, video = self._check_download_just_video(url, 'WL')
3384         if video:
3385             return video
3386         _, playlist = self._extract_playlist('WL')
3387         return playlist
3388
3389
3390 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
3391     IE_NAME = 'youtube:favorites'
3392     IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
3393     _VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
3394     _LOGIN_REQUIRED = True
3395
3396     def _real_extract(self, url):
3397         webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
3398         playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, 'favourites playlist id')
3399         return self.url_result(playlist_id, 'YoutubePlaylist')
3400
3401
3402 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
3403     IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
3404     _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/recommended|:ytrec(?:ommended)?'
3405     _FEED_NAME = 'recommended'
3406     _PLAYLIST_TITLE = 'Youtube Recommended videos'
3407
3408
3409 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
3410     IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
3411     _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
3412     _FEED_NAME = 'subscriptions'
3413     _PLAYLIST_TITLE = 'Youtube Subscriptions'
3414
3415
3416 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
3417     IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
3418     _VALID_URL = r'https?://(?:www\.)?youtube\.com/feed/history|:ythistory'
3419     _FEED_NAME = 'history'
3420     _PLAYLIST_TITLE = 'Youtube History'
3421
3422
3423 class YoutubeTruncatedURLIE(InfoExtractor):
3424     IE_NAME = 'youtube:truncated_url'
3425     IE_DESC = False  # Do not list
3426     _VALID_URL = r'''(?x)
3427         (?:https?://)?
3428         (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
3429         (?:watch\?(?:
3430             feature=[a-z_]+|
3431             annotation_id=annotation_[^&]+|
3432             x-yt-cl=[0-9]+|
3433             hl=[^&]*|
3434             t=[0-9]+
3435         )?
3436         |
3437             attribution_link\?a=[^&]+
3438         )
3439         $
3440     '''
3441
3442     _TESTS = [{
3443         'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
3444         'only_matching': True,
3445     }, {
3446         'url': 'https://www.youtube.com/watch?',
3447         'only_matching': True,
3448     }, {
3449         'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
3450         'only_matching': True,
3451     }, {
3452         'url': 'https://www.youtube.com/watch?feature=foo',
3453         'only_matching': True,
3454     }, {
3455         'url': 'https://www.youtube.com/watch?hl=en-GB',
3456         'only_matching': True,
3457     }, {
3458         'url': 'https://www.youtube.com/watch?t=2372',
3459         'only_matching': True,
3460     }]
3461
3462     def _real_extract(self, url):
3463         raise ExtractorError(
3464             'Did you forget to quote the URL? Remember that & is a meta '
3465             'character in most shells, so you want to put the URL in quotes, '
3466             'like  youtube-dl '
3467             '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
3468             ' or simply  youtube-dl BaW_jenozKc  .',
3469             expected=True)
3470
3471
3472 class YoutubeTruncatedIDIE(InfoExtractor):
3473     IE_NAME = 'youtube:truncated_id'
3474     IE_DESC = False  # Do not list
3475     _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
3476
3477     _TESTS = [{
3478         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
3479         'only_matching': True,
3480     }]
3481
3482     def _real_extract(self, url):
3483         video_id = self._match_id(url)
3484         raise ExtractorError(
3485             'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
3486             expected=True)