_ Git - youtube-dl/blob - youtube_dl/extractor/youtube.py

   1 # coding: utf-8
   2
   3 import collections
   4 import errno
   5 import io
   6 import itertools
   7 import json
   8 import os.path
   9 import re
  10 import string
  11 import struct
  12 import traceback
  13 import zlib
  14
  15 from .common import InfoExtractor, SearchInfoExtractor
  16 from .subtitles import SubtitlesInfoExtractor
  17 from ..utils import (
  18     compat_chr,
  19     compat_parse_qs,
  20     compat_urllib_parse,
  21     compat_urllib_request,
  22     compat_urlparse,
  23     compat_str,
  24
  25     clean_html,
  26     get_cachedir,
  27     get_element_by_id,
  28     get_element_by_attribute,
  29     ExtractorError,
  30     unescapeHTML,
  31     unified_strdate,
  32     orderedSet,
  33     write_json_file,
  34 )
  35
  36 class YoutubeBaseInfoExtractor(InfoExtractor):
  37     """Provide base functions for Youtube extractors"""
  38     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
  39     _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
  40     _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
  41     _NETRC_MACHINE = 'youtube'
  42     # If True it will raise an error if no login info is provided
  43     _LOGIN_REQUIRED = False
  44
  45     def _set_language(self):
  46         return bool(self._download_webpage(
  47             self._LANG_URL, None,
  48             note=u'Setting language', errnote='unable to set language',
  49             fatal=False))
  50
  51     def _login(self):
  52         (username, password) = self._get_login_info()
  53         # No authentication to be performed
  54         if username is None:
  55             if self._LOGIN_REQUIRED:
  56                 raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True)
  57             return False
  58
  59         login_page = self._download_webpage(
  60             self._LOGIN_URL, None,
  61             note=u'Downloading login page',
  62             errnote=u'unable to fetch login page', fatal=False)
  63         if login_page is False:
  64             return
  65
  66         galx = self._search_regex(r'(?s)<input.+?name="GALX".+?value="(.+?)"',
  67                                   login_page, u'Login GALX parameter')
  68
  69         # Log in
  70         login_form_strs = {
  71                 u'continue': u'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
  72                 u'Email': username,
  73                 u'GALX': galx,
  74                 u'Passwd': password,
  75                 u'PersistentCookie': u'yes',
  76                 u'_utf8': u'霱',
  77                 u'bgresponse': u'js_disabled',
  78                 u'checkConnection': u'',
  79                 u'checkedDomains': u'youtube',
  80                 u'dnConn': u'',
  81                 u'pstMsg': u'0',
  82                 u'rmShown': u'1',
  83                 u'secTok': u'',
  84                 u'signIn': u'Sign in',
  85                 u'timeStmp': u'',
  86                 u'service': u'youtube',
  87                 u'uilel': u'3',
  88                 u'hl': u'en_US',
  89         }
  90         # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
  91         # chokes on unicode
  92         login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items())
  93         login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
  94
  95         req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
  96         login_results = self._download_webpage(
  97             req, None,
  98             note=u'Logging in', errnote=u'unable to log in', fatal=False)
  99         if login_results is False:
 100             return False
 101         if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
 102             self._downloader.report_warning(u'unable to log in: bad username or password')
 103             return False
 104         return True
 105
 106     def _confirm_age(self):
 107         age_form = {
 108             'next_url': '/',
 109             'action_confirm': 'Confirm',
 110         }
 111         req = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form))
 112
 113         self._download_webpage(
 114             req, None,
 115             note=u'Confirming age', errnote=u'Unable to confirm age')
 116         return True
 117
 118     def _real_initialize(self):
 119         if self._downloader is None:
 120             return
 121         if not self._set_language():
 122             return
 123         if not self._login():
 124             return
 125         self._confirm_age()
 126
 127
 128 class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 129     IE_DESC = u'YouTube.com'
 130     _VALID_URL = r"""(?x)^
 131                      (
 132                          (?:https?://|//)?                                    # http(s):// or protocol-independent URL (optional)
 133                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
 134                             tube\.majestyc\.net/|
 135                             youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
 136                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
 137                          (?:                                                  # the various things that can precede the ID:
 138                              (?:(?:v|embed|e)/)                               # v/ or embed/ or e/
 139                              |(?:                                             # or the v= param in all its forms
 140                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?)?    # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
 141                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
 142                                  (?:.*?&)?                                    # any other preceding param (like /?s=tuff&v=xxxx)
 143                                  v=
 144                              )
 145                          ))
 146                          |youtu\.be/                                          # just youtu.be/xxxx
 147                          )
 148                      )?                                                       # all until now is optional -> you can pass the naked ID
 149                      ([0-9A-Za-z_-]{11})                                      # here is it! the YouTube video ID
 150                      (?(1).+)?                                                # if we found the ID, everything can follow
 151                      $"""
 152     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
 153     _formats = {
 154         '5': {'ext': 'flv', 'width': 400, 'height': 240},
 155         '6': {'ext': 'flv', 'width': 450, 'height': 270},
 156         '13': {'ext': '3gp'},
 157         '17': {'ext': '3gp', 'width': 176, 'height': 144},
 158         '18': {'ext': 'mp4', 'width': 640, 'height': 360},
 159         '22': {'ext': 'mp4', 'width': 1280, 'height': 720},
 160         '34': {'ext': 'flv', 'width': 640, 'height': 360},
 161         '35': {'ext': 'flv', 'width': 854, 'height': 480},
 162         '36': {'ext': '3gp', 'width': 320, 'height': 240},
 163         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080},
 164         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072},
 165         '43': {'ext': 'webm', 'width': 640, 'height': 360},
 166         '44': {'ext': 'webm', 'width': 854, 'height': 480},
 167         '45': {'ext': 'webm', 'width': 1280, 'height': 720},
 168         '46': {'ext': 'webm', 'width': 1920, 'height': 1080},
 169
 170
 171         # 3d videos
 172         '82': {'ext': 'mp4', 'height': 360, 'resolution': '360p', 'format_note': '3D', 'preference': -20},
 173         '83': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': '3D', 'preference': -20},
 174         '84': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': '3D', 'preference': -20},
 175         '85': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': '3D', 'preference': -20},
 176         '100': {'ext': 'webm', 'height': 360, 'resolution': '360p', 'format_note': '3D', 'preference': -20},
 177         '101': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': '3D', 'preference': -20},
 178         '102': {'ext': 'webm', 'height': 720, 'resolution': '720p', 'format_note': '3D', 'preference': -20},
 179
 180         # Apple HTTP Live Streaming
 181         '92': {'ext': 'mp4', 'height': 240, 'resolution': '240p', 'format_note': 'HLS', 'preference': -10},
 182         '93': {'ext': 'mp4', 'height': 360, 'resolution': '360p', 'format_note': 'HLS', 'preference': -10},
 183         '94': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': 'HLS', 'preference': -10},
 184         '95': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': 'HLS', 'preference': -10},
 185         '96': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'HLS', 'preference': -10},
 186         '132': {'ext': 'mp4', 'height': 240, 'resolution': '240p', 'format_note': 'HLS', 'preference': -10},
 187         '151': {'ext': 'mp4', 'height': 72, 'resolution': '72p', 'format_note': 'HLS', 'preference': -10},
 188
 189         # DASH mp4 video
 190         '133': {'ext': 'mp4', 'height': 240, 'resolution': '240p', 'format_note': 'DASH video', 'preference': -40},
 191         '134': {'ext': 'mp4', 'height': 360, 'resolution': '360p', 'format_note': 'DASH video', 'preference': -40},
 192         '135': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': 'DASH video', 'preference': -40},
 193         '136': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': 'DASH video', 'preference': -40},
 194         '137': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH video', 'preference': -40},
 195         '138': {'ext': 'mp4', 'height': 1081, 'resolution': '>1080p', 'format_note': 'DASH video', 'preference': -40},
 196         '160': {'ext': 'mp4', 'height': 192, 'resolution': '192p', 'format_note': 'DASH video', 'preference': -40},
 197         '264': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH video', 'preference': -40},
 198
 199         # Dash mp4 audio
 200         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
 201         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 128, 'preference': -50},
 202         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 256, 'preference': -50},
 203
 204         # Dash webm
 205         '242': {'ext': 'webm', 'height': 240, 'resolution': '240p', 'format_note': 'DASH webm', 'preference': -40},
 206         '243': {'ext': 'webm', 'height': 360, 'resolution': '360p', 'format_note': 'DASH webm', 'preference': -40},
 207         '244': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40},
 208         '245': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40},
 209         '246': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40},
 210         '247': {'ext': 'webm', 'height': 720, 'resolution': '720p', 'format_note': 'DASH webm', 'preference': -40},
 211         '248': {'ext': 'webm', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH webm', 'preference': -40},
 212
 213         # Dash webm audio
 214         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 48, 'preference': -50},
 215         '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 256, 'preference': -50},
 216     }
 217
 218     IE_NAME = u'youtube'
 219     _TESTS = [
 220         {
 221             u"url":  u"http://www.youtube.com/watch?v=BaW_jenozKc",
 222             u"file":  u"BaW_jenozKc.mp4",
 223             u"info_dict": {
 224                 u"title": u"youtube-dl test video \"'/\\ä↭𝕐",
 225                 u"uploader": u"Philipp Hagemeister",
 226                 u"uploader_id": u"phihag",
 227                 u"upload_date": u"20121002",
 228                 u"description": u"test chars:  \"'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
 229             }
 230         },
 231         {
 232             u"url":  u"http://www.youtube.com/watch?v=UxxajLWwzqY",
 233             u"file":  u"UxxajLWwzqY.mp4",
 234             u"note": u"Test generic use_cipher_signature video (#897)",
 235             u"info_dict": {
 236                 u"upload_date": u"20120506",
 237                 u"title": u"Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]",
 238                 u"description": u"md5:5b292926389560516e384ac437c0ec07",
 239                 u"uploader": u"Icona Pop",
 240                 u"uploader_id": u"IconaPop"
 241             }
 242         },
 243         {
 244             u"url":  u"https://www.youtube.com/watch?v=07FYdnEawAQ",
 245             u"file":  u"07FYdnEawAQ.mp4",
 246             u"note": u"Test VEVO video with age protection (#956)",
 247             u"info_dict": {
 248                 u"upload_date": u"20130703",
 249                 u"title": u"Justin Timberlake - Tunnel Vision (Explicit)",
 250                 u"description": u"md5:64249768eec3bc4276236606ea996373",
 251                 u"uploader": u"justintimberlakeVEVO",
 252                 u"uploader_id": u"justintimberlakeVEVO"
 253             }
 254         },
 255         {
 256             u"url":  u"//www.YouTube.com/watch?v=yZIXLfi8CZQ",
 257             u"file":  u"yZIXLfi8CZQ.mp4",
 258             u"note": u"Embed-only video (#1746)",
 259             u"info_dict": {
 260                 u"upload_date": u"20120608",
 261                 u"title": u"Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012",
 262                 u"description": u"md5:09b78bd971f1e3e289601dfba15ca4f7",
 263                 u"uploader": u"SET India",
 264                 u"uploader_id": u"setindia"
 265             }
 266         },
 267     ]
 268
 269
 270     @classmethod
 271     def suitable(cls, url):
 272         """Receives a URL and returns True if suitable for this IE."""
 273         if YoutubePlaylistIE.suitable(url): return False
 274         return re.match(cls._VALID_URL, url) is not None
 275
 276     def __init__(self, *args, **kwargs):
 277         super(YoutubeIE, self).__init__(*args, **kwargs)
 278         self._player_cache = {}
 279
 280     def report_video_info_webpage_download(self, video_id):
 281         """Report attempt to download video info webpage."""
 282         self.to_screen(u'%s: Downloading video info webpage' % video_id)
 283
 284     def report_information_extraction(self, video_id):
 285         """Report attempt to extract video information."""
 286         self.to_screen(u'%s: Extracting video information' % video_id)
 287
 288     def report_unavailable_format(self, video_id, format):
 289         """Report extracted video URL."""
 290         self.to_screen(u'%s: Format %s not available' % (video_id, format))
 291
 292     def report_rtmp_download(self):
 293         """Indicate the download will use the RTMP protocol."""
 294         self.to_screen(u'RTMP download detected')
 295
 296     def _extract_signature_function(self, video_id, player_url, slen):
 297         id_m = re.match(r'.*-(?P<id>[a-zA-Z0-9_-]+)\.(?P<ext>[a-z]+)$',
 298                         player_url)
 299         player_type = id_m.group('ext')
 300         player_id = id_m.group('id')
 301
 302         # Read from filesystem cache
 303         func_id = '%s_%s_%d' % (player_type, player_id, slen)
 304         assert os.path.basename(func_id) == func_id
 305         cache_dir = get_cachedir(self._downloader.params)
 306
 307         cache_enabled = cache_dir is not None
 308         if cache_enabled:
 309             cache_fn = os.path.join(os.path.expanduser(cache_dir),
 310                                     u'youtube-sigfuncs',
 311                                     func_id + '.json')
 312             try:
 313                 with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
 314                     cache_spec = json.load(cachef)
 315                 return lambda s: u''.join(s[i] for i in cache_spec)
 316             except IOError:
 317                 pass  # No cache available
 318
 319         if player_type == 'js':
 320             code = self._download_webpage(
 321                 player_url, video_id,
 322                 note=u'Downloading %s player %s' % (player_type, player_id),
 323                 errnote=u'Download of %s failed' % player_url)
 324             res = self._parse_sig_js(code)
 325         elif player_type == 'swf':
 326             urlh = self._request_webpage(
 327                 player_url, video_id,
 328                 note=u'Downloading %s player %s' % (player_type, player_id),
 329                 errnote=u'Download of %s failed' % player_url)
 330             code = urlh.read()
 331             res = self._parse_sig_swf(code)
 332         else:
 333             assert False, 'Invalid player type %r' % player_type
 334
 335         if cache_enabled:
 336             try:
 337                 test_string = u''.join(map(compat_chr, range(slen)))
 338                 cache_res = res(test_string)
 339                 cache_spec = [ord(c) for c in cache_res]
 340                 try:
 341                     os.makedirs(os.path.dirname(cache_fn))
 342                 except OSError as ose:
 343                     if ose.errno != errno.EEXIST:
 344                         raise
 345                 write_json_file(cache_spec, cache_fn)
 346             except Exception:
 347                 tb = traceback.format_exc()
 348                 self._downloader.report_warning(
 349                     u'Writing cache to %r failed: %s' % (cache_fn, tb))
 350
 351         return res
 352
 353     def _print_sig_code(self, func, slen):
 354         def gen_sig_code(idxs):
 355             def _genslice(start, end, step):
 356                 starts = u'' if start == 0 else str(start)
 357                 ends = (u':%d' % (end+step)) if end + step >= 0 else u':'
 358                 steps = u'' if step == 1 else (u':%d' % step)
 359                 return u's[%s%s%s]' % (starts, ends, steps)
 360
 361             step = None
 362             start = '(Never used)'  # Quelch pyflakes warnings - start will be
 363                                     # set as soon as step is set
 364             for i, prev in zip(idxs[1:], idxs[:-1]):
 365                 if step is not None:
 366                     if i - prev == step:
 367                         continue
 368                     yield _genslice(start, prev, step)
 369                     step = None
 370                     continue
 371                 if i - prev in [-1, 1]:
 372                     step = i - prev
 373                     start = prev
 374                     continue
 375                 else:
 376                     yield u's[%d]' % prev
 377             if step is None:
 378                 yield u's[%d]' % i
 379             else:
 380                 yield _genslice(start, i, step)
 381
 382         test_string = u''.join(map(compat_chr, range(slen)))
 383         cache_res = func(test_string)
 384         cache_spec = [ord(c) for c in cache_res]
 385         expr_code = u' + '.join(gen_sig_code(cache_spec))
 386         code = u'if len(s) == %d:\n    return %s\n' % (slen, expr_code)
 387         self.to_screen(u'Extracted signature function:\n' + code)
 388
 389     def _parse_sig_js(self, jscode):
 390         funcname = self._search_regex(
 391             r'signature=([a-zA-Z]+)', jscode,
 392             u'Initial JS player signature function name')
 393
 394         functions = {}
 395
 396         def argidx(varname):
 397             return string.lowercase.index(varname)
 398
 399         def interpret_statement(stmt, local_vars, allow_recursion=20):
 400             if allow_recursion < 0:
 401                 raise ExtractorError(u'Recursion limit reached')
 402
 403             if stmt.startswith(u'var '):
 404                 stmt = stmt[len(u'var '):]
 405             ass_m = re.match(r'^(?P<out>[a-z]+)(?:\[(?P<index>[^\]]+)\])?' +
 406                              r'=(?P<expr>.*)$', stmt)
 407             if ass_m:
 408                 if ass_m.groupdict().get('index'):
 409                     def assign(val):
 410                         lvar = local_vars[ass_m.group('out')]
 411                         idx = interpret_expression(ass_m.group('index'),
 412                                                    local_vars, allow_recursion)
 413                         assert isinstance(idx, int)
 414                         lvar[idx] = val
 415                         return val
 416                     expr = ass_m.group('expr')
 417                 else:
 418                     def assign(val):
 419                         local_vars[ass_m.group('out')] = val
 420                         return val
 421                     expr = ass_m.group('expr')
 422             elif stmt.startswith(u'return '):
 423                 assign = lambda v: v
 424                 expr = stmt[len(u'return '):]
 425             else:
 426                 raise ExtractorError(
 427                     u'Cannot determine left side of statement in %r' % stmt)
 428
 429             v = interpret_expression(expr, local_vars, allow_recursion)
 430             return assign(v)
 431
 432         def interpret_expression(expr, local_vars, allow_recursion):
 433             if expr.isdigit():
 434                 return int(expr)
 435
 436             if expr.isalpha():
 437                 return local_vars[expr]
 438
 439             m = re.match(r'^(?P<in>[a-z]+)\.(?P<member>.*)$', expr)
 440             if m:
 441                 member = m.group('member')
 442                 val = local_vars[m.group('in')]
 443                 if member == 'split("")':
 444                     return list(val)
 445                 if member == 'join("")':
 446                     return u''.join(val)
 447                 if member == 'length':
 448                     return len(val)
 449                 if member == 'reverse()':
 450                     return val[::-1]
 451                 slice_m = re.match(r'slice\((?P<idx>.*)\)', member)
 452                 if slice_m:
 453                     idx = interpret_expression(
 454                         slice_m.group('idx'), local_vars, allow_recursion-1)
 455                     return val[idx:]
 456
 457             m = re.match(
 458                 r'^(?P<in>[a-z]+)\[(?P<idx>.+)\]$', expr)
 459             if m:
 460                 val = local_vars[m.group('in')]
 461                 idx = interpret_expression(m.group('idx'), local_vars,
 462                                            allow_recursion-1)
 463                 return val[idx]
 464
 465             m = re.match(r'^(?P<a>.+?)(?P<op>[%])(?P<b>.+?)$', expr)
 466             if m:
 467                 a = interpret_expression(m.group('a'),
 468                                          local_vars, allow_recursion)
 469                 b = interpret_expression(m.group('b'),
 470                                          local_vars, allow_recursion)
 471                 return a % b
 472
 473             m = re.match(
 474                 r'^(?P<func>[a-zA-Z]+)\((?P<args>[a-z0-9,]+)\)$', expr)
 475             if m:
 476                 fname = m.group('func')
 477                 if fname not in functions:
 478                     functions[fname] = extract_function(fname)
 479                 argvals = [int(v) if v.isdigit() else local_vars[v]
 480                            for v in m.group('args').split(',')]
 481                 return functions[fname](argvals)
 482             raise ExtractorError(u'Unsupported JS expression %r' % expr)
 483
 484         def extract_function(funcname):
 485             func_m = re.search(
 486                 r'function ' + re.escape(funcname) +
 487                 r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
 488                 jscode)
 489             argnames = func_m.group('args').split(',')
 490
 491             def resf(args):
 492                 local_vars = dict(zip(argnames, args))
 493                 for stmt in func_m.group('code').split(';'):
 494                     res = interpret_statement(stmt, local_vars)
 495                 return res
 496             return resf
 497
 498         initial_function = extract_function(funcname)
 499         return lambda s: initial_function([s])
 500
 501     def _parse_sig_swf(self, file_contents):
 502         if file_contents[1:3] != b'WS':
 503             raise ExtractorError(
 504                 u'Not an SWF file; header is %r' % file_contents[:3])
 505         if file_contents[:1] == b'C':
 506             content = zlib.decompress(file_contents[8:])
 507         else:
 508             raise NotImplementedError(u'Unsupported compression format %r' %
 509                                       file_contents[:1])
 510
 511         def extract_tags(content):
 512             pos = 0
 513             while pos < len(content):
 514                 header16 = struct.unpack('<H', content[pos:pos+2])[0]
 515                 pos += 2
 516                 tag_code = header16 >> 6
 517                 tag_len = header16 & 0x3f
 518                 if tag_len == 0x3f:
 519                     tag_len = struct.unpack('<I', content[pos:pos+4])[0]
 520                     pos += 4
 521                 assert pos+tag_len <= len(content)
 522                 yield (tag_code, content[pos:pos+tag_len])
 523                 pos += tag_len
 524
 525         code_tag = next(tag
 526                         for tag_code, tag in extract_tags(content)
 527                         if tag_code == 82)
 528         p = code_tag.index(b'\0', 4) + 1
 529         code_reader = io.BytesIO(code_tag[p:])
 530
 531         # Parse ABC (AVM2 ByteCode)
 532         def read_int(reader=None):
 533             if reader is None:
 534                 reader = code_reader
 535             res = 0
 536             shift = 0
 537             for _ in range(5):
 538                 buf = reader.read(1)
 539                 assert len(buf) == 1
 540                 b = struct.unpack('<B', buf)[0]
 541                 res = res | ((b & 0x7f) << shift)
 542                 if b & 0x80 == 0:
 543                     break
 544                 shift += 7
 545             return res
 546
 547         def u30(reader=None):
 548             res = read_int(reader)
 549             assert res & 0xf0000000 == 0
 550             return res
 551         u32 = read_int
 552
 553         def s32(reader=None):
 554             v = read_int(reader)
 555             if v & 0x80000000 != 0:
 556                 v = - ((v ^ 0xffffffff) + 1)
 557             return v
 558
 559         def read_string(reader=None):
 560             if reader is None:
 561                 reader = code_reader
 562             slen = u30(reader)
 563             resb = reader.read(slen)
 564             assert len(resb) == slen
 565             return resb.decode('utf-8')
 566
 567         def read_bytes(count, reader=None):
 568             if reader is None:
 569                 reader = code_reader
 570             resb = reader.read(count)
 571             assert len(resb) == count
 572             return resb
 573
 574         def read_byte(reader=None):
 575             resb = read_bytes(1, reader=reader)
 576             res = struct.unpack('<B', resb)[0]
 577             return res
 578
 579         # minor_version + major_version
 580         read_bytes(2 + 2)
 581
 582         # Constant pool
 583         int_count = u30()
 584         for _c in range(1, int_count):
 585             s32()
 586         uint_count = u30()
 587         for _c in range(1, uint_count):
 588             u32()
 589         double_count = u30()
 590         read_bytes((double_count-1) * 8)
 591         string_count = u30()
 592         constant_strings = [u'']
 593         for _c in range(1, string_count):
 594             s = read_string()
 595             constant_strings.append(s)
 596         namespace_count = u30()
 597         for _c in range(1, namespace_count):
 598             read_bytes(1)  # kind
 599             u30()  # name
 600         ns_set_count = u30()
 601         for _c in range(1, ns_set_count):
 602             count = u30()
 603             for _c2 in range(count):
 604                 u30()
 605         multiname_count = u30()
 606         MULTINAME_SIZES = {
 607             0x07: 2,  # QName
 608             0x0d: 2,  # QNameA
 609             0x0f: 1,  # RTQName
 610             0x10: 1,  # RTQNameA
 611             0x11: 0,  # RTQNameL
 612             0x12: 0,  # RTQNameLA
 613             0x09: 2,  # Multiname
 614             0x0e: 2,  # MultinameA
 615             0x1b: 1,  # MultinameL
 616             0x1c: 1,  # MultinameLA
 617         }
 618         multinames = [u'']
 619         for _c in range(1, multiname_count):
 620             kind = u30()
 621             assert kind in MULTINAME_SIZES, u'Invalid multiname kind %r' % kind
 622             if kind == 0x07:
 623                 u30()  # namespace_idx
 624                 name_idx = u30()
 625                 multinames.append(constant_strings[name_idx])
 626             else:
 627                 multinames.append('[MULTINAME kind: %d]' % kind)
 628                 for _c2 in range(MULTINAME_SIZES[kind]):
 629                     u30()
 630
 631         # Methods
 632         method_count = u30()
 633         MethodInfo = collections.namedtuple(
 634             'MethodInfo',
 635             ['NEED_ARGUMENTS', 'NEED_REST'])
 636         method_infos = []
 637         for method_id in range(method_count):
 638             param_count = u30()
 639             u30()  # return type
 640             for _ in range(param_count):
 641                 u30()  # param type
 642             u30()  # name index (always 0 for youtube)
 643             flags = read_byte()
 644             if flags & 0x08 != 0:
 645                 # Options present
 646                 option_count = u30()
 647                 for c in range(option_count):
 648                     u30()  # val
 649                     read_bytes(1)  # kind
 650             if flags & 0x80 != 0:
 651                 # Param names present
 652                 for _ in range(param_count):
 653                     u30()  # param name
 654             mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
 655             method_infos.append(mi)
 656
 657         # Metadata
 658         metadata_count = u30()
 659         for _c in range(metadata_count):
 660             u30()  # name
 661             item_count = u30()
 662             for _c2 in range(item_count):
 663                 u30()  # key
 664                 u30()  # value
 665
 666         def parse_traits_info():
 667             trait_name_idx = u30()
 668             kind_full = read_byte()
 669             kind = kind_full & 0x0f
 670             attrs = kind_full >> 4
 671             methods = {}
 672             if kind in [0x00, 0x06]:  # Slot or Const
 673                 u30()  # Slot id
 674                 u30()  # type_name_idx
 675                 vindex = u30()
 676                 if vindex != 0:
 677                     read_byte()  # vkind
 678             elif kind in [0x01, 0x02, 0x03]:  # Method / Getter / Setter
 679                 u30()  # disp_id
 680                 method_idx = u30()
 681                 methods[multinames[trait_name_idx]] = method_idx
 682             elif kind == 0x04:  # Class
 683                 u30()  # slot_id
 684                 u30()  # classi
 685             elif kind == 0x05:  # Function
 686                 u30()  # slot_id
 687                 function_idx = u30()
 688                 methods[function_idx] = multinames[trait_name_idx]
 689             else:
 690                 raise ExtractorError(u'Unsupported trait kind %d' % kind)
 691
 692             if attrs & 0x4 != 0:  # Metadata present
 693                 metadata_count = u30()
 694                 for _c3 in range(metadata_count):
 695                     u30()  # metadata index
 696
 697             return methods
 698
 699         # Classes
 700         TARGET_CLASSNAME = u'SignatureDecipher'
 701         searched_idx = multinames.index(TARGET_CLASSNAME)
 702         searched_class_id = None
 703         class_count = u30()
 704         for class_id in range(class_count):
 705             name_idx = u30()
 706             if name_idx == searched_idx:
 707                 # We found the class we're looking for!
 708                 searched_class_id = class_id
 709             u30()  # super_name idx
 710             flags = read_byte()
 711             if flags & 0x08 != 0:  # Protected namespace is present
 712                 u30()  # protected_ns_idx
 713             intrf_count = u30()
 714             for _c2 in range(intrf_count):
 715                 u30()
 716             u30()  # iinit
 717             trait_count = u30()
 718             for _c2 in range(trait_count):
 719                 parse_traits_info()
 720
 721         if searched_class_id is None:
 722             raise ExtractorError(u'Target class %r not found' %
 723                                  TARGET_CLASSNAME)
 724
 725         method_names = {}
 726         method_idxs = {}
 727         for class_id in range(class_count):
 728             u30()  # cinit
 729             trait_count = u30()
 730             for _c2 in range(trait_count):
 731                 trait_methods = parse_traits_info()
 732                 if class_id == searched_class_id:
 733                     method_names.update(trait_methods.items())
 734                     method_idxs.update(dict(
 735                         (idx, name)
 736                         for name, idx in trait_methods.items()))
 737
 738         # Scripts
 739         script_count = u30()
 740         for _c in range(script_count):
 741             u30()  # init
 742             trait_count = u30()
 743             for _c2 in range(trait_count):
 744                 parse_traits_info()
 745
 746         # Method bodies
 747         method_body_count = u30()
 748         Method = collections.namedtuple('Method', ['code', 'local_count'])
 749         methods = {}
 750         for _c in range(method_body_count):
 751             method_idx = u30()
 752             u30()  # max_stack
 753             local_count = u30()
 754             u30()  # init_scope_depth
 755             u30()  # max_scope_depth
 756             code_length = u30()
 757             code = read_bytes(code_length)
 758             if method_idx in method_idxs:
 759                 m = Method(code, local_count)
 760                 methods[method_idxs[method_idx]] = m
 761             exception_count = u30()
 762             for _c2 in range(exception_count):
 763                 u30()  # from
 764                 u30()  # to
 765                 u30()  # target
 766                 u30()  # exc_type
 767                 u30()  # var_name
 768             trait_count = u30()
 769             for _c2 in range(trait_count):
 770                 parse_traits_info()
 771
 772         assert p + code_reader.tell() == len(code_tag)
 773         assert len(methods) == len(method_idxs)
 774
 775         method_pyfunctions = {}
 776
 777         def extract_function(func_name):
 778             if func_name in method_pyfunctions:
 779                 return method_pyfunctions[func_name]
 780             if func_name not in methods:
 781                 raise ExtractorError(u'Cannot find function %r' % func_name)
 782             m = methods[func_name]
 783
 784             def resfunc(args):
 785                 registers = ['(this)'] + list(args) + [None] * m.local_count
 786                 stack = []
 787                 coder = io.BytesIO(m.code)
 788                 while True:
 789                     opcode = struct.unpack('!B', coder.read(1))[0]
 790                     if opcode == 36:  # pushbyte
 791                         v = struct.unpack('!B', coder.read(1))[0]
 792                         stack.append(v)
 793                     elif opcode == 44:  # pushstring
 794                         idx = u30(coder)
 795                         stack.append(constant_strings[idx])
 796                     elif opcode == 48:  # pushscope
 797                         # We don't implement the scope register, so we'll just
 798                         # ignore the popped value
 799                         stack.pop()
 800                     elif opcode == 70:  # callproperty
 801                         index = u30(coder)
 802                         mname = multinames[index]
 803                         arg_count = u30(coder)
 804                         args = list(reversed(
 805                             [stack.pop() for _ in range(arg_count)]))
 806                         obj = stack.pop()
 807                         if mname == u'split':
 808                             assert len(args) == 1
 809                             assert isinstance(args[0], compat_str)
 810                             assert isinstance(obj, compat_str)
 811                             if args[0] == u'':
 812                                 res = list(obj)
 813                             else:
 814                                 res = obj.split(args[0])
 815                             stack.append(res)
 816                         elif mname == u'slice':
 817                             assert len(args) == 1
 818                             assert isinstance(args[0], int)
 819                             assert isinstance(obj, list)
 820                             res = obj[args[0]:]
 821                             stack.append(res)
 822                         elif mname == u'join':
 823                             assert len(args) == 1
 824                             assert isinstance(args[0], compat_str)
 825                             assert isinstance(obj, list)
 826                             res = args[0].join(obj)
 827                             stack.append(res)
 828                         elif mname in method_pyfunctions:
 829                             stack.append(method_pyfunctions[mname](args))
 830                         else:
 831                             raise NotImplementedError(
 832                                 u'Unsupported property %r on %r'
 833                                 % (mname, obj))
 834                     elif opcode == 72:  # returnvalue
 835                         res = stack.pop()
 836                         return res
 837                     elif opcode == 79:  # callpropvoid
 838                         index = u30(coder)
 839                         mname = multinames[index]
 840                         arg_count = u30(coder)
 841                         args = list(reversed(
 842                             [stack.pop() for _ in range(arg_count)]))
 843                         obj = stack.pop()
 844                         if mname == u'reverse':
 845                             assert isinstance(obj, list)
 846                             obj.reverse()
 847                         else:
 848                             raise NotImplementedError(
 849                                 u'Unsupported (void) property %r on %r'
 850                                 % (mname, obj))
 851                     elif opcode == 93:  # findpropstrict
 852                         index = u30(coder)
 853                         mname = multinames[index]
 854                         res = extract_function(mname)
 855                         stack.append(res)
 856                     elif opcode == 97:  # setproperty
 857                         index = u30(coder)
 858                         value = stack.pop()
 859                         idx = stack.pop()
 860                         obj = stack.pop()
 861                         assert isinstance(obj, list)
 862                         assert isinstance(idx, int)
 863                         obj[idx] = value
 864                     elif opcode == 98:  # getlocal
 865                         index = u30(coder)
 866                         stack.append(registers[index])
 867                     elif opcode == 99:  # setlocal
 868                         index = u30(coder)
 869                         value = stack.pop()
 870                         registers[index] = value
 871                     elif opcode == 102:  # getproperty
 872                         index = u30(coder)
 873                         pname = multinames[index]
 874                         if pname == u'length':
 875                             obj = stack.pop()
 876                             assert isinstance(obj, list)
 877                             stack.append(len(obj))
 878                         else:  # Assume attribute access
 879                             idx = stack.pop()
 880                             assert isinstance(idx, int)
 881                             obj = stack.pop()
 882                             assert isinstance(obj, list)
 883                             stack.append(obj[idx])
 884                     elif opcode == 128:  # coerce
 885                         u30(coder)
 886                     elif opcode == 133:  # coerce_s
 887                         assert isinstance(stack[-1], (type(None), compat_str))
 888                     elif opcode == 164:  # modulo
 889                         value2 = stack.pop()
 890                         value1 = stack.pop()
 891                         res = value1 % value2
 892                         stack.append(res)
 893                     elif opcode == 208:  # getlocal_0
 894                         stack.append(registers[0])
 895                     elif opcode == 209:  # getlocal_1
 896                         stack.append(registers[1])
 897                     elif opcode == 210:  # getlocal_2
 898                         stack.append(registers[2])
 899                     elif opcode == 211:  # getlocal_3
 900                         stack.append(registers[3])
 901                     elif opcode == 214:  # setlocal_2
 902                         registers[2] = stack.pop()
 903                     elif opcode == 215:  # setlocal_3
 904                         registers[3] = stack.pop()
 905                     else:
 906                         raise NotImplementedError(
 907                             u'Unsupported opcode %d' % opcode)
 908
 909             method_pyfunctions[func_name] = resfunc
 910             return resfunc
 911
 912         initial_function = extract_function(u'decipher')
 913         return lambda s: initial_function([s])
 914
 915     def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
 916         """Turn the encrypted s field into a working signature"""
 917
 918         if player_url is not None:
 919             if player_url.startswith(u'//'):
 920                 player_url = u'https:' + player_url
 921             try:
 922                 player_id = (player_url, len(s))
 923                 if player_id not in self._player_cache:
 924                     func = self._extract_signature_function(
 925                         video_id, player_url, len(s)
 926                     )
 927                     self._player_cache[player_id] = func
 928                 func = self._player_cache[player_id]
 929                 if self._downloader.params.get('youtube_print_sig_code'):
 930                     self._print_sig_code(func, len(s))
 931                 return func(s)
 932             except Exception:
 933                 tb = traceback.format_exc()
 934                 self._downloader.report_warning(
 935                     u'Automatic signature extraction failed: ' + tb)
 936
 937             self._downloader.report_warning(
 938                 u'Warning: Falling back to static signature algorithm')
 939
 940         return self._static_decrypt_signature(
 941             s, video_id, player_url, age_gate)
 942
 943     def _static_decrypt_signature(self, s, video_id, player_url, age_gate):
 944         if age_gate:
 945             # The videos with age protection use another player, so the
 946             # algorithms can be different.
 947             if len(s) == 86:
 948                 return s[2:63] + s[82] + s[64:82] + s[63]
 949
 950         if len(s) == 93:
 951             return s[86:29:-1] + s[88] + s[28:5:-1]
 952         elif len(s) == 92:
 953             return s[25] + s[3:25] + s[0] + s[26:42] + s[79] + s[43:79] + s[91] + s[80:83]
 954         elif len(s) == 91:
 955             return s[84:27:-1] + s[86] + s[26:5:-1]
 956         elif len(s) == 90:
 957             return s[25] + s[3:25] + s[2] + s[26:40] + s[77] + s[41:77] + s[89] + s[78:81]
 958         elif len(s) == 89:
 959             return s[84:78:-1] + s[87] + s[77:60:-1] + s[0] + s[59:3:-1]
 960         elif len(s) == 88:
 961             return s[7:28] + s[87] + s[29:45] + s[55] + s[46:55] + s[2] + s[56:87] + s[28]
 962         elif len(s) == 87:
 963             return s[6:27] + s[4] + s[28:39] + s[27] + s[40:59] + s[2] + s[60:]
 964         elif len(s) == 86:
 965             return s[80:72:-1] + s[16] + s[71:39:-1] + s[72] + s[38:16:-1] + s[82] + s[15::-1]
 966         elif len(s) == 85:
 967             return s[3:11] + s[0] + s[12:55] + s[84] + s[56:84]
 968         elif len(s) == 84:
 969             return s[78:70:-1] + s[14] + s[69:37:-1] + s[70] + s[36:14:-1] + s[80] + s[:14][::-1]
 970         elif len(s) == 83:
 971             return s[80:63:-1] + s[0] + s[62:0:-1] + s[63]
 972         elif len(s) == 82:
 973             return s[80:37:-1] + s[7] + s[36:7:-1] + s[0] + s[6:0:-1] + s[37]
 974         elif len(s) == 81:
 975             return s[56] + s[79:56:-1] + s[41] + s[55:41:-1] + s[80] + s[40:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
 976         elif len(s) == 80:
 977             return s[1:19] + s[0] + s[20:68] + s[19] + s[69:80]
 978         elif len(s) == 79:
 979             return s[54] + s[77:54:-1] + s[39] + s[53:39:-1] + s[78] + s[38:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
 980
 981         else:
 982             raise ExtractorError(u'Unable to decrypt signature, key length %d not supported; retrying might work' % (len(s)))
 983
 984     def _get_available_subtitles(self, video_id, webpage):
 985         try:
 986             sub_list = self._download_webpage(
 987                 'http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
 988                 video_id, note=False)
 989         except ExtractorError as err:
 990             self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
 991             return {}
 992         lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
 993
 994         sub_lang_list = {}
 995         for l in lang_list:
 996             lang = l[1]
 997             params = compat_urllib_parse.urlencode({
 998                 'lang': lang,
 999                 'v': video_id,
1000                 'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
1001                 'name': l[0].encode('utf-8'),
1002             })
1003             url = u'http://www.youtube.com/api/timedtext?' + params
1004             sub_lang_list[lang] = url
1005         if not sub_lang_list:
1006             self._downloader.report_warning(u'video doesn\'t have subtitles')
1007             return {}
1008         return sub_lang_list
1009
1010     def _get_available_automatic_caption(self, video_id, webpage):
1011         """We need the webpage for getting the captions url, pass it as an
1012            argument to speed up the process."""
1013         sub_format = self._downloader.params.get('subtitlesformat', 'srt')
1014         self.to_screen(u'%s: Looking for automatic captions' % video_id)
1015         mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
1016         err_msg = u'Couldn\'t find automatic captions for %s' % video_id
1017         if mobj is None:
1018             self._downloader.report_warning(err_msg)
1019             return {}
1020         player_config = json.loads(mobj.group(1))
1021         try:
1022             args = player_config[u'args']
1023             caption_url = args[u'ttsurl']
1024             timestamp = args[u'timestamp']
1025             # We get the available subtitles
1026             list_params = compat_urllib_parse.urlencode({
1027                 'type': 'list',
1028                 'tlangs': 1,
1029                 'asrs': 1,
1030             })
1031             list_url = caption_url + '&' + list_params
1032             caption_list = self._download_xml(list_url, video_id)
1033             original_lang_node = caption_list.find('track')
1034             if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr' :
1035                 self._downloader.report_warning(u'Video doesn\'t have automatic captions')
1036                 return {}
1037             original_lang = original_lang_node.attrib['lang_code']
1038
1039             sub_lang_list = {}
1040             for lang_node in caption_list.findall('target'):
1041                 sub_lang = lang_node.attrib['lang_code']
1042                 params = compat_urllib_parse.urlencode({
1043                     'lang': original_lang,
1044                     'tlang': sub_lang,
1045                     'fmt': sub_format,
1046                     'ts': timestamp,
1047                     'kind': 'asr',
1048                 })
1049                 sub_lang_list[sub_lang] = caption_url + '&' + params
1050             return sub_lang_list
1051         # An extractor error can be raise by the download process if there are
1052         # no automatic captions but there are subtitles
1053         except (KeyError, ExtractorError):
1054             self._downloader.report_warning(err_msg)
1055             return {}
1056
1057     def _extract_id(self, url):
1058         mobj = re.match(self._VALID_URL, url, re.VERBOSE)
1059         if mobj is None:
1060             raise ExtractorError(u'Invalid URL: %s' % url)
1061         video_id = mobj.group(2)
1062         return video_id
1063
1064     def _get_video_url_list(self, url_map):
1065         """
1066         Transform a dictionary in the format {itag:url} to a list of (itag, url)
1067         with the requested formats.
1068         """
1069         existing_formats = [x for x in self._formats if x in url_map]
1070         if len(existing_formats) == 0:
1071             raise ExtractorError(u'no known formats available for video')
1072         video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
1073         video_url_list.reverse() # order worst to best
1074         return video_url_list
1075
1076     def _extract_from_m3u8(self, manifest_url, video_id):
1077         url_map = {}
1078         def _get_urls(_manifest):
1079             lines = _manifest.split('\n')
1080             urls = filter(lambda l: l and not l.startswith('#'),
1081                             lines)
1082             return urls
1083         manifest = self._download_webpage(manifest_url, video_id, u'Downloading formats manifest')
1084         formats_urls = _get_urls(manifest)
1085         for format_url in formats_urls:
1086             itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
1087             url_map[itag] = format_url
1088         return url_map
1089
1090     def _extract_annotations(self, video_id):
1091         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
1092         return self._download_webpage(url, video_id, note=u'Searching for annotations.', errnote=u'Unable to download video annotations.')
1093
1094     def _real_extract(self, url):
1095         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1096         mobj = re.search(self._NEXT_URL_RE, url)
1097         if mobj:
1098             url = 'https://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
1099         video_id = self._extract_id(url)
1100
1101         # Get video webpage
1102         url = 'https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
1103         video_webpage = self._download_webpage(url, video_id)
1104
1105         # Attempt to extract SWF player URL
1106         mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1107         if mobj is not None:
1108             player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1109         else:
1110             player_url = None
1111
1112         # Get video info
1113         self.report_video_info_webpage_download(video_id)
1114         if re.search(r'player-age-gate-content">', video_webpage) is not None:
1115             self.report_age_confirmation()
1116             age_gate = True
1117             # We simulate the access to the video from www.youtube.com/v/{video_id}
1118             # this can be viewed without login into Youtube
1119             data = compat_urllib_parse.urlencode({'video_id': video_id,
1120                                                   'el': 'player_embedded',
1121                                                   'gl': 'US',
1122                                                   'hl': 'en',
1123                                                   'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1124                                                   'asv': 3,
1125                                                   'sts':'1588',
1126                                                   })
1127             video_info_url = 'https://www.youtube.com/get_video_info?' + data
1128             video_info_webpage = self._download_webpage(video_info_url, video_id,
1129                                     note=False,
1130                                     errnote='unable to download video info webpage')
1131             video_info = compat_parse_qs(video_info_webpage)
1132         else:
1133             age_gate = False
1134             for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
1135                 video_info_url = ('https://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1136                         % (video_id, el_type))
1137                 video_info_webpage = self._download_webpage(video_info_url, video_id,
1138                                         note=False,
1139                                         errnote='unable to download video info webpage')
1140                 video_info = compat_parse_qs(video_info_webpage)
1141                 if 'token' in video_info:
1142                     break
1143         if 'token' not in video_info:
1144             if 'reason' in video_info:
1145                 raise ExtractorError(u'YouTube said: %s' % video_info['reason'][0], expected=True)
1146             else:
1147                 raise ExtractorError(u'"token" parameter not in video info for unknown reason')
1148
1149         if 'view_count' in video_info:
1150             view_count = int(video_info['view_count'][0])
1151         else:
1152             view_count = None
1153
1154         # Check for "rental" videos
1155         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1156             raise ExtractorError(u'"rental" videos not supported')
1157
1158         # Start extracting information
1159         self.report_information_extraction(video_id)
1160
1161         # uploader
1162         if 'author' not in video_info:
1163             raise ExtractorError(u'Unable to extract uploader name')
1164         video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
1165
1166         # uploader_id
1167         video_uploader_id = None
1168         mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
1169         if mobj is not None:
1170             video_uploader_id = mobj.group(1)
1171         else:
1172             self._downloader.report_warning(u'unable to extract uploader nickname')
1173
1174         # title
1175         if 'title' in video_info:
1176             video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
1177         else:
1178             self._downloader.report_warning(u'Unable to extract video title')
1179             video_title = u'_'
1180
1181         # thumbnail image
1182         # We try first to get a high quality image:
1183         m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
1184                             video_webpage, re.DOTALL)
1185         if m_thumb is not None:
1186             video_thumbnail = m_thumb.group(1)
1187         elif 'thumbnail_url' not in video_info:
1188             self._downloader.report_warning(u'unable to extract video thumbnail')
1189             video_thumbnail = None
1190         else:   # don't panic if we can't find it
1191             video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
1192
1193         # upload date
1194         upload_date = None
1195         mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
1196         if mobj is not None:
1197             upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1198             upload_date = unified_strdate(upload_date)
1199
1200         # description
1201         video_description = get_element_by_id("eow-description", video_webpage)
1202         if video_description:
1203             video_description = re.sub(r'''(?x)
1204                 <a\s+
1205                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
1206                     title="([^"]+)"\s+
1207                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
1208                     class="yt-uix-redirect-link"\s*>
1209                 [^<]+
1210                 </a>
1211             ''', r'\1', video_description)
1212             video_description = clean_html(video_description)
1213         else:
1214             fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
1215             if fd_mobj:
1216                 video_description = unescapeHTML(fd_mobj.group(1))
1217             else:
1218                 video_description = u''
1219
1220         def _extract_count(klass):
1221             count = self._search_regex(
1222                 r'class="%s">([\d,]+)</span>' % re.escape(klass),
1223                 video_webpage, klass, default=None)
1224             if count is not None:
1225                 return int(count.replace(',', ''))
1226             return None
1227         like_count = _extract_count(u'likes-count')
1228         dislike_count = _extract_count(u'dislikes-count')
1229
1230         # subtitles
1231         video_subtitles = self.extract_subtitles(video_id, video_webpage)
1232
1233         if self._downloader.params.get('listsubtitles', False):
1234             self._list_available_subtitles(video_id, video_webpage)
1235             return
1236
1237         if 'length_seconds' not in video_info:
1238             self._downloader.report_warning(u'unable to extract video duration')
1239             video_duration = None
1240         else:
1241             video_duration = int(compat_urllib_parse.unquote_plus(video_info['length_seconds'][0]))
1242
1243         # annotations
1244         video_annotations = None
1245         if self._downloader.params.get('writeannotations', False):
1246                 video_annotations = self._extract_annotations(video_id)
1247
1248         # Decide which formats to download
1249
1250         try:
1251             mobj = re.search(r';ytplayer.config = ({.*?});', video_webpage)
1252             if not mobj:
1253                 raise ValueError('Could not find vevo ID')
1254             info = json.loads(mobj.group(1))
1255             args = info['args']
1256             # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
1257             # this signatures are encrypted
1258             if 'url_encoded_fmt_stream_map' not in args:
1259                 raise ValueError(u'No stream_map present')  # caught below
1260             re_signature = re.compile(r'[&,]s=')
1261             m_s = re_signature.search(args['url_encoded_fmt_stream_map'])
1262             if m_s is not None:
1263                 self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
1264                 video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
1265             m_s = re_signature.search(args.get('adaptive_fmts', u''))
1266             if m_s is not None:
1267                 if 'adaptive_fmts' in video_info:
1268                     video_info['adaptive_fmts'][0] += ',' + args['adaptive_fmts']
1269                 else:
1270                     video_info['adaptive_fmts'] = [args['adaptive_fmts']]
1271         except ValueError:
1272             pass
1273
1274         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1275             self.report_rtmp_download()
1276             video_url_list = [(None, video_info['conn'][0])]
1277         elif len(video_info.get('url_encoded_fmt_stream_map', [])) >= 1 or len(video_info.get('adaptive_fmts', [])) >= 1:
1278             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts',[''])[0]
1279             if 'rtmpe%3Dyes' in encoded_url_map:
1280                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
1281             url_map = {}
1282             for url_data_str in encoded_url_map.split(','):
1283                 url_data = compat_parse_qs(url_data_str)
1284                 if 'itag' in url_data and 'url' in url_data:
1285                     url = url_data['url'][0]
1286                     if 'sig' in url_data:
1287                         url += '&signature=' + url_data['sig'][0]
1288                     elif 's' in url_data:
1289                         encrypted_sig = url_data['s'][0]
1290                         if self._downloader.params.get('verbose'):
1291                             if age_gate:
1292                                 if player_url is None:
1293                                     player_version = 'unknown'
1294                                 else:
1295                                     player_version = self._search_regex(
1296                                         r'-(.+)\.swf$', player_url,
1297                                         u'flash player', fatal=False)
1298                                 player_desc = 'flash player %s' % player_version
1299                             else:
1300                                 player_version = self._search_regex(
1301                                     r'html5player-(.+?)\.js', video_webpage,
1302                                     'html5 player', fatal=False)
1303                                 player_desc = u'html5 player %s' % player_version
1304
1305                             parts_sizes = u'.'.join(compat_str(len(part)) for part in encrypted_sig.split('.'))
1306                             self.to_screen(u'encrypted signature length %d (%s), itag %s, %s' %
1307                                 (len(encrypted_sig), parts_sizes, url_data['itag'][0], player_desc))
1308
1309                         if not age_gate:
1310                             jsplayer_url_json = self._search_regex(
1311                                 r'"assets":.+?"js":\s*("[^"]+")',
1312                                 video_webpage, u'JS player URL')
1313                             player_url = json.loads(jsplayer_url_json)
1314
1315                         signature = self._decrypt_signature(
1316                             encrypted_sig, video_id, player_url, age_gate)
1317                         url += '&signature=' + signature
1318                     if 'ratebypass' not in url:
1319                         url += '&ratebypass=yes'
1320                     url_map[url_data['itag'][0]] = url
1321             video_url_list = self._get_video_url_list(url_map)
1322         elif video_info.get('hlsvp'):
1323             manifest_url = video_info['hlsvp'][0]
1324             url_map = self._extract_from_m3u8(manifest_url, video_id)
1325             video_url_list = self._get_video_url_list(url_map)
1326         else:
1327             raise ExtractorError(u'no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
1328
1329         formats = []
1330         for itag, video_real_url in video_url_list:
1331             dct = {
1332                 'format_id': itag,
1333                 'url': video_real_url,
1334                 'player_url': player_url,
1335             }
1336             dct.update(self._formats[itag])
1337             formats.append(dct)
1338
1339         self._sort_formats(formats)
1340
1341         return {
1342             'id':           video_id,
1343             'uploader':     video_uploader,
1344             'uploader_id':  video_uploader_id,
1345             'upload_date':  upload_date,
1346             'title':        video_title,
1347             'thumbnail':    video_thumbnail,
1348             'description':  video_description,
1349             'subtitles':    video_subtitles,
1350             'duration':     video_duration,
1351             'age_limit':    18 if age_gate else 0,
1352             'annotations':  video_annotations,
1353             'webpage_url': 'https://www.youtube.com/watch?v=%s' % video_id,
1354             'view_count':   view_count,
1355             'like_count': like_count,
1356             'dislike_count': dislike_count,
1357             'formats':      formats,
1358         }
1359
1360 class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
1361     IE_DESC = u'YouTube.com playlists'
1362     _VALID_URL = r"""(?:
1363                         (?:https?://)?
1364                         (?:\w+\.)?
1365                         youtube\.com/
1366                         (?:
1367                            (?:course|view_play_list|my_playlists|artist|playlist|watch)
1368                            \? (?:.*?&)*? (?:p|a|list)=
1369                         |  p/
1370                         )
1371                         ((?:PL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,})
1372                         .*
1373                      |
1374                         ((?:PL|EC|UU|FL|RD)[0-9A-Za-z-_]{10,})
1375                      )"""
1376     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&page=%s'
1377     _MORE_PAGES_INDICATOR = r'data-link-type="next"'
1378     _VIDEO_RE = r'href="/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)'
1379     IE_NAME = u'youtube:playlist'
1380
1381     @classmethod
1382     def suitable(cls, url):
1383         """Receives a URL and returns True if suitable for this IE."""
1384         return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
1385
1386     def _real_initialize(self):
1387         self._login()
1388
1389     def _ids_to_results(self, ids):
1390         return [self.url_result(vid_id, 'Youtube', video_id=vid_id)
1391                        for vid_id in ids]
1392
1393     def _extract_mix(self, playlist_id):
1394         # The mixes are generated from a a single video
1395         # the id of the playlist is just 'RD' + video_id
1396         url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id)
1397         webpage = self._download_webpage(url, playlist_id, u'Downloading Youtube mix')
1398         title_span = (get_element_by_attribute('class', 'title long-title', webpage) or
1399             get_element_by_attribute('class', 'title ', webpage))
1400         title = clean_html(title_span)
1401         video_re = r'data-index="\d+".*?href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s' % re.escape(playlist_id)
1402         ids = orderedSet(re.findall(video_re, webpage))
1403         url_results = self._ids_to_results(ids)
1404
1405         return self.playlist_result(url_results, playlist_id, title)
1406
1407     def _real_extract(self, url):
1408         # Extract playlist id
1409         mobj = re.match(self._VALID_URL, url, re.VERBOSE)
1410         if mobj is None:
1411             raise ExtractorError(u'Invalid URL: %s' % url)
1412         playlist_id = mobj.group(1) or mobj.group(2)
1413
1414         # Check if it's a video-specific URL
1415         query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
1416         if 'v' in query_dict:
1417             video_id = query_dict['v'][0]
1418             if self._downloader.params.get('noplaylist'):
1419                 self.to_screen(u'Downloading just video %s because of --no-playlist' % video_id)
1420                 return self.url_result(video_id, 'Youtube', video_id=video_id)
1421             else:
1422                 self.to_screen(u'Downloading playlist PL%s - add --no-playlist to just download video %s' % (playlist_id, video_id))
1423
1424         if playlist_id.startswith('RD'):
1425             # Mixes require a custom extraction process
1426             return self._extract_mix(playlist_id)
1427         if playlist_id.startswith('TL'):
1428             raise ExtractorError(u'For downloading YouTube.com top lists, use '
1429                 u'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True)
1430
1431         # Extract the video ids from the playlist pages
1432         ids = []
1433
1434         for page_num in itertools.count(1):
1435             url = self._TEMPLATE_URL % (playlist_id, page_num)
1436             page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num)
1437             matches = re.finditer(self._VIDEO_RE, page)
1438             # We remove the duplicates and the link with index 0
1439             # (it's not the first video of the playlist)
1440             new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
1441             ids.extend(new_ids)
1442
1443             if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1444                 break
1445
1446         playlist_title = self._og_search_title(page)
1447
1448         url_results = self._ids_to_results(ids)
1449         return self.playlist_result(url_results, playlist_id, playlist_title)
1450
1451
1452 class YoutubeTopListIE(YoutubePlaylistIE):
1453     IE_NAME = u'youtube:toplist'
1454     IE_DESC = (u'YouTube.com top lists, "yttoplist:{channel}:{list title}"'
1455         u' (Example: "yttoplist:music:Top Tracks")')
1456     _VALID_URL = r'yttoplist:(?P<chann>.*?):(?P<title>.*?)$'
1457
1458     def _real_extract(self, url):
1459         mobj = re.match(self._VALID_URL, url)
1460         channel = mobj.group('chann')
1461         title = mobj.group('title')
1462         query = compat_urllib_parse.urlencode({'title': title})
1463         playlist_re = 'href="([^"]+?%s[^"]+?)"' % re.escape(query)
1464         channel_page = self._download_webpage('https://www.youtube.com/%s' % channel, title)
1465         link = self._html_search_regex(playlist_re, channel_page, u'list')
1466         url = compat_urlparse.urljoin('https://www.youtube.com/', link)
1467
1468         video_re = r'data-index="\d+".*?data-video-id="([0-9A-Za-z_-]{11})"'
1469         ids = []
1470         # sometimes the webpage doesn't contain the videos
1471         # retry until we get them
1472         for i in itertools.count(0):
1473             msg = u'Downloading Youtube mix'
1474             if i > 0:
1475                 msg += ', retry #%d' % i
1476             webpage = self._download_webpage(url, title, msg)
1477             ids = orderedSet(re.findall(video_re, webpage))
1478             if ids:
1479                 break
1480         url_results = self._ids_to_results(ids)
1481         return self.playlist_result(url_results, playlist_title=title)
1482
1483
1484 class YoutubeChannelIE(InfoExtractor):
1485     IE_DESC = u'YouTube.com channels'
1486     _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
1487     _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
1488     _MORE_PAGES_URL = 'http://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
1489     IE_NAME = u'youtube:channel'
1490
1491     def extract_videos_from_page(self, page):
1492         ids_in_page = []
1493         for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
1494             if mobj.group(1) not in ids_in_page:
1495                 ids_in_page.append(mobj.group(1))
1496         return ids_in_page
1497
1498     def _real_extract(self, url):
1499         # Extract channel id
1500         mobj = re.match(self._VALID_URL, url)
1501         if mobj is None:
1502             raise ExtractorError(u'Invalid URL: %s' % url)
1503
1504         # Download channel page
1505         channel_id = mobj.group(1)
1506         video_ids = []
1507         url = 'https://www.youtube.com/channel/%s/videos' % channel_id
1508         channel_page = self._download_webpage(url, channel_id)
1509         autogenerated = re.search(r'''(?x)
1510                 class="[^"]*?(?:
1511                     channel-header-autogenerated-label|
1512                     yt-channel-title-autogenerated
1513                 )[^"]*"''', channel_page) is not None
1514
1515         if autogenerated:
1516             # The videos are contained in a single page
1517             # the ajax pages can't be used, they are empty
1518             video_ids = self.extract_videos_from_page(channel_page)
1519         else:
1520             # Download all channel pages using the json-based channel_ajax query
1521             for pagenum in itertools.count(1):
1522                 url = self._MORE_PAGES_URL % (pagenum, channel_id)
1523                 page = self._download_webpage(url, channel_id,
1524                                               u'Downloading page #%s' % pagenum)
1525
1526                 page = json.loads(page)
1527
1528                 ids_in_page = self.extract_videos_from_page(page['content_html'])
1529                 video_ids.extend(ids_in_page)
1530
1531                 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
1532                     break
1533
1534         self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
1535
1536         url_entries = [self.url_result(video_id, 'Youtube', video_id=video_id)
1537                        for video_id in video_ids]
1538         return self.playlist_result(url_entries, channel_id)
1539
1540
1541 class YoutubeUserIE(InfoExtractor):
1542     IE_DESC = u'YouTube.com user videos (URL or "ytuser" keyword)'
1543     _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
1544     _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1545     _GDATA_PAGE_SIZE = 50
1546     _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
1547     IE_NAME = u'youtube:user'
1548
1549     @classmethod
1550     def suitable(cls, url):
1551         # Don't return True if the url can be extracted with other youtube
1552         # extractor, the regex would is too permissive and it would match.
1553         other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
1554         if any(ie.suitable(url) for ie in other_ies): return False
1555         else: return super(YoutubeUserIE, cls).suitable(url)
1556
1557     def _real_extract(self, url):
1558         # Extract username
1559         mobj = re.match(self._VALID_URL, url)
1560         if mobj is None:
1561             raise ExtractorError(u'Invalid URL: %s' % url)
1562
1563         username = mobj.group(1)
1564
1565         # Download video ids using YouTube Data API. Result size per
1566         # query is limited (currently to 50 videos) so we need to query
1567         # page by page until there are no video ids - it means we got
1568         # all of them.
1569
1570         url_results = []
1571
1572         for pagenum in itertools.count(0):
1573             start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1574
1575             gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
1576             page = self._download_webpage(gdata_url, username,
1577                                           u'Downloading video ids from %d to %d' % (start_index, start_index + self._GDATA_PAGE_SIZE))
1578
1579             try:
1580                 response = json.loads(page)
1581             except ValueError as err:
1582                 raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
1583             if 'entry' not in response['feed']:
1584                 # Number of videos is a multiple of self._MAX_RESULTS
1585                 break
1586
1587             # Extract video identifiers
1588             entries = response['feed']['entry']
1589             for entry in entries:
1590                 title = entry['title']['$t']
1591                 video_id = entry['id']['$t'].split('/')[-1]
1592                 url_results.append({
1593                     '_type': 'url',
1594                     'url': video_id,
1595                     'ie_key': 'Youtube',
1596                     'id': 'video_id',
1597                     'title': title,
1598                 })
1599
1600             # A little optimization - if current page is not
1601             # "full", ie. does not contain PAGE_SIZE video ids then
1602             # we can assume that this page is the last one - there
1603             # are no more ids on further pages - no need to query
1604             # again.
1605
1606             if len(entries) < self._GDATA_PAGE_SIZE:
1607                 break
1608
1609         return self.playlist_result(url_results, playlist_title=username)
1610
1611
1612 class YoutubeSearchIE(SearchInfoExtractor):
1613     IE_DESC = u'YouTube.com searches'
1614     _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1615     _MAX_RESULTS = 1000
1616     IE_NAME = u'youtube:search'
1617     _SEARCH_KEY = 'ytsearch'
1618
1619     def _get_n_results(self, query, n):
1620         """Get a specified number of results for a query"""
1621
1622         video_ids = []
1623         pagenum = 0
1624         limit = n
1625
1626         while (50 * pagenum) < limit:
1627             result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
1628             data_json = self._download_webpage(
1629                 result_url, video_id=u'query "%s"' % query,
1630                 note=u'Downloading page %s' % (pagenum + 1),
1631                 errnote=u'Unable to download API page')
1632             data = json.loads(data_json)
1633             api_response = data['data']
1634
1635             if 'items' not in api_response:
1636                 raise ExtractorError(u'[youtube] No video results')
1637
1638             new_ids = list(video['id'] for video in api_response['items'])
1639             video_ids += new_ids
1640
1641             limit = min(n, api_response['totalItems'])
1642             pagenum += 1
1643
1644         if len(video_ids) > n:
1645             video_ids = video_ids[:n]
1646         videos = [self.url_result(video_id, 'Youtube', video_id=video_id)
1647                   for video_id in video_ids]
1648         return self.playlist_result(videos, query)
1649
1650 class YoutubeSearchDateIE(YoutubeSearchIE):
1651     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
1652     _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published'
1653     _SEARCH_KEY = 'ytsearchdate'
1654     IE_DESC = u'YouTube.com searches, newest videos first'
1655
1656 class YoutubeShowIE(InfoExtractor):
1657     IE_DESC = u'YouTube.com (multi-season) shows'
1658     _VALID_URL = r'https?://www\.youtube\.com/show/(.*)'
1659     IE_NAME = u'youtube:show'
1660
1661     def _real_extract(self, url):
1662         mobj = re.match(self._VALID_URL, url)
1663         show_name = mobj.group(1)
1664         webpage = self._download_webpage(url, show_name, u'Downloading show webpage')
1665         # There's one playlist for each season of the show
1666         m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
1667         self.to_screen(u'%s: Found %s seasons' % (show_name, len(m_seasons)))
1668         return [self.url_result('https://www.youtube.com' + season.group(1), 'YoutubePlaylist') for season in m_seasons]
1669
1670
1671 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
1672     """
1673     Base class for extractors that fetch info from
1674     http://www.youtube.com/feed_ajax
1675     Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
1676     """
1677     _LOGIN_REQUIRED = True
1678     # use action_load_personal_feed instead of action_load_system_feed
1679     _PERSONAL_FEED = False
1680
1681     @property
1682     def _FEED_TEMPLATE(self):
1683         action = 'action_load_system_feed'
1684         if self._PERSONAL_FEED:
1685             action = 'action_load_personal_feed'
1686         return 'http://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME)
1687
1688     @property
1689     def IE_NAME(self):
1690         return u'youtube:%s' % self._FEED_NAME
1691
1692     def _real_initialize(self):
1693         self._login()
1694
1695     def _real_extract(self, url):
1696         feed_entries = []
1697         paging = 0
1698         for i in itertools.count(1):
1699             info = self._download_webpage(self._FEED_TEMPLATE % paging,
1700                                           u'%s feed' % self._FEED_NAME,
1701                                           u'Downloading page %s' % i)
1702             info = json.loads(info)
1703             feed_html = info['feed_html']
1704             m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
1705             ids = orderedSet(m.group(1) for m in m_ids)
1706             feed_entries.extend(
1707                 self.url_result(video_id, 'Youtube', video_id=video_id)
1708                 for video_id in ids)
1709             if info['paging'] is None:
1710                 break
1711             paging = info['paging']
1712         return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
1713
1714 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
1715     IE_DESC = u'YouTube.com subscriptions feed, "ytsubs" keyword(requires authentication)'
1716     _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
1717     _FEED_NAME = 'subscriptions'
1718     _PLAYLIST_TITLE = u'Youtube Subscriptions'
1719
1720 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
1721     IE_DESC = u'YouTube.com recommended videos, "ytrec" keyword (requires authentication)'
1722     _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
1723     _FEED_NAME = 'recommended'
1724     _PLAYLIST_TITLE = u'Youtube Recommended videos'
1725
1726 class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
1727     IE_DESC = u'Youtube watch later list, "ytwatchlater" keyword (requires authentication)'
1728     _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
1729     _FEED_NAME = 'watch_later'
1730     _PLAYLIST_TITLE = u'Youtube Watch Later'
1731     _PERSONAL_FEED = True
1732
1733 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
1734     IE_DESC = u'Youtube watch history, "ythistory" keyword (requires authentication)'
1735     _VALID_URL = u'https?://www\.youtube\.com/feed/history|:ythistory'
1736     _FEED_NAME = 'history'
1737     _PERSONAL_FEED = True
1738     _PLAYLIST_TITLE = u'Youtube Watch History'
1739
1740 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
1741     IE_NAME = u'youtube:favorites'
1742     IE_DESC = u'YouTube.com favourite videos, "ytfav" keyword (requires authentication)'
1743     _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
1744     _LOGIN_REQUIRED = True
1745
1746     def _real_extract(self, url):
1747         webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
1748         playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, u'favourites playlist id')
1749         return self.url_result(playlist_id, 'YoutubePlaylist')
1750
1751
1752 class YoutubeTruncatedURLIE(InfoExtractor):
1753     IE_NAME = 'youtube:truncated_url'
1754     IE_DESC = False  # Do not list
1755     _VALID_URL = r'(?:https?://)?[^/]+/watch\?feature=[a-z_]+$'
1756
1757     def _real_extract(self, url):
1758         raise ExtractorError(
1759             u'Did you forget to quote the URL? Remember that & is a meta '
1760             u'character in most shells, so you want to put the URL in quotes, '
1761             u'like  youtube-dl '
1762             u'\'http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc\''
1763             u' (or simply  youtube-dl BaW_jenozKc  ).',
1764             expected=True)