14 import xml.etree.ElementTree
17 from .common import InfoExtractor, SearchInfoExtractor
18 from .subtitles import SubtitlesInfoExtractor
25 compat_urllib_request,
39 class YoutubeBaseInfoExtractor(InfoExtractor):
40 """Provide base functions for Youtube extractors"""
41 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
42 _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
43 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
44 _NETRC_MACHINE = 'youtube'
45 # If True it will raise an error if no login info is provided
46 _LOGIN_REQUIRED = False
48 def report_lang(self):
49 """Report attempt to set language."""
50 self.to_screen(u'Setting language')
52 def _set_language(self):
53 request = compat_urllib_request.Request(self._LANG_URL)
56 compat_urllib_request.urlopen(request).read()
57 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
58 self._downloader.report_warning(u'unable to set language: %s' % compat_str(err))
63 (username, password) = self._get_login_info()
64 # No authentication to be performed
66 if self._LOGIN_REQUIRED:
67 raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True)
70 request = compat_urllib_request.Request(self._LOGIN_URL)
72 login_page = compat_urllib_request.urlopen(request).read().decode('utf-8')
73 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
74 self._downloader.report_warning(u'unable to fetch login page: %s' % compat_str(err))
79 match = re.search(re.compile(r'<input.+?name="GALX".+?value="(.+?)"', re.DOTALL), login_page)
82 match = re.search(re.compile(r'<input.+?name="dsh".+?value="(.+?)"', re.DOTALL), login_page)
88 u'continue': u'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
92 u'PersistentCookie': u'yes',
94 u'bgresponse': u'js_disabled',
95 u'checkConnection': u'',
96 u'checkedDomains': u'youtube',
102 u'signIn': u'Sign in',
104 u'service': u'youtube',
108 # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
110 login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items())
111 login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
112 request = compat_urllib_request.Request(self._LOGIN_URL, login_data)
115 login_results = compat_urllib_request.urlopen(request).read().decode('utf-8')
116 if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
117 self._downloader.report_warning(u'unable to log in: bad username or password')
119 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
120 self._downloader.report_warning(u'unable to log in: %s' % compat_str(err))
124 def _confirm_age(self):
127 'action_confirm': 'Confirm',
129 request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form))
131 self.report_age_confirmation()
132 compat_urllib_request.urlopen(request).read().decode('utf-8')
133 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
134 raise ExtractorError(u'Unable to confirm age: %s' % compat_str(err))
137 def _real_initialize(self):
138 if self._downloader is None:
140 if not self._set_language():
142 if not self._login():
147 class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
148 IE_DESC = u'YouTube.com'
151 (?:https?://)? # http(s):// (optional)
152 (?:(?:(?:(?:\w+\.)?youtube(?:-nocookie)?\.com/|
153 tube\.majestyc\.net/|
154 youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
155 (?:.*?\#/)? # handle anchor (#/) redirect urls
156 (?: # the various things that can precede the ID:
157 (?:(?:v|embed|e)/) # v/ or embed/ or e/
158 |(?: # or the v= param in all its forms
159 (?:(?:watch|movie)(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
160 (?:\?|\#!?) # the params delimiter ? or # or #!
161 (?:.*?&)? # any other preceding param (like /?s=tuff&v=xxxx)
165 |youtu\.be/ # just youtu.be/xxxx
167 )? # all until now is optional -> you can pass the naked ID
168 ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
169 (?(1).+)? # if we found the ID, everything can follow
171 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
172 # Listed in order of quality
173 _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '36', '17', '13',
174 # Apple HTTP Live Streaming
175 '96', '95', '94', '93', '92', '132', '151',
177 '85', '84', '102', '83', '101', '82', '100',
179 '138', '137', '248', '136', '247', '135', '246',
180 '245', '244', '134', '243', '133', '242', '160',
182 '141', '172', '140', '171', '139',
184 _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '36', '17', '13',
185 # Apple HTTP Live Streaming
186 '96', '95', '94', '93', '92', '132', '151',
188 '85', '102', '84', '101', '83', '100', '82',
190 '138', '248', '137', '247', '136', '246', '245',
191 '244', '135', '243', '134', '242', '133', '160',
193 '172', '141', '171', '140', '139',
195 _video_formats_map = {
196 'flv': ['35', '34', '6', '5'],
197 '3gp': ['36', '17', '13'],
198 'mp4': ['38', '37', '22', '18'],
199 'webm': ['46', '45', '44', '43'],
201 _video_extensions = {
223 # Apple HTTP Live Streaming
257 _video_dimensions = {
339 u"url": u"http://www.youtube.com/watch?v=BaW_jenozKc",
340 u"file": u"BaW_jenozKc.mp4",
342 u"title": u"youtube-dl test video \"'/\\ä↭𝕐",
343 u"uploader": u"Philipp Hagemeister",
344 u"uploader_id": u"phihag",
345 u"upload_date": u"20121002",
346 u"description": u"test chars: \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
350 u"url": u"http://www.youtube.com/watch?v=1ltcDfZMA3U",
351 u"file": u"1ltcDfZMA3U.mp4",
352 u"note": u"Test VEVO video (#897)",
354 u"upload_date": u"20070518",
355 u"title": u"Maps - It Will Find You",
356 u"description": u"Music video by Maps performing It Will Find You.",
357 u"uploader": u"MuteUSA",
358 u"uploader_id": u"MuteUSA"
362 u"url": u"http://www.youtube.com/watch?v=UxxajLWwzqY",
363 u"file": u"UxxajLWwzqY.mp4",
364 u"note": u"Test generic use_cipher_signature video (#897)",
366 u"upload_date": u"20120506",
367 u"title": u"Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]",
368 u"description": u"md5:5b292926389560516e384ac437c0ec07",
369 u"uploader": u"Icona Pop",
370 u"uploader_id": u"IconaPop"
374 u"url": u"https://www.youtube.com/watch?v=07FYdnEawAQ",
375 u"file": u"07FYdnEawAQ.mp4",
376 u"note": u"Test VEVO video with age protection (#956)",
378 u"upload_date": u"20130703",
379 u"title": u"Justin Timberlake - Tunnel Vision (Explicit)",
380 u"description": u"md5:64249768eec3bc4276236606ea996373",
381 u"uploader": u"justintimberlakeVEVO",
382 u"uploader_id": u"justintimberlakeVEVO"
389 def suitable(cls, url):
390 """Receives a URL and returns True if suitable for this IE."""
391 if YoutubePlaylistIE.suitable(url): return False
392 return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
394 def __init__(self, *args, **kwargs):
395 super(YoutubeIE, self).__init__(*args, **kwargs)
396 self._player_cache = {}
398 def report_video_webpage_download(self, video_id):
399 """Report attempt to download video webpage."""
400 self.to_screen(u'%s: Downloading video webpage' % video_id)
402 def report_video_info_webpage_download(self, video_id):
403 """Report attempt to download video info webpage."""
404 self.to_screen(u'%s: Downloading video info webpage' % video_id)
406 def report_information_extraction(self, video_id):
407 """Report attempt to extract video information."""
408 self.to_screen(u'%s: Extracting video information' % video_id)
410 def report_unavailable_format(self, video_id, format):
411 """Report extracted video URL."""
412 self.to_screen(u'%s: Format %s not available' % (video_id, format))
414 def report_rtmp_download(self):
415 """Indicate the download will use the RTMP protocol."""
416 self.to_screen(u'RTMP download detected')
418 def _extract_signature_function(self, video_id, player_url, slen):
419 id_m = re.match(r'.*-(?P<id>[a-zA-Z0-9_-]+)\.(?P<ext>[a-z]+)$',
421 player_type = id_m.group('ext')
422 player_id = id_m.group('id')
424 # Read from filesystem cache
425 func_id = '%s_%s_%d' % (player_type, player_id, slen)
426 assert os.path.basename(func_id) == func_id
427 cache_dir = get_cachedir(self._downloader.params)
429 cache_enabled = cache_dir is not None
431 cache_fn = os.path.join(os.path.expanduser(cache_dir),
435 with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
436 cache_spec = json.load(cachef)
437 return lambda s: u''.join(s[i] for i in cache_spec)
439 pass # No cache available
441 if player_type == 'js':
442 code = self._download_webpage(
443 player_url, video_id,
444 note=u'Downloading %s player %s' % (player_type, player_id),
445 errnote=u'Download of %s failed' % player_url)
446 res = self._parse_sig_js(code)
447 elif player_type == 'swf':
448 urlh = self._request_webpage(
449 player_url, video_id,
450 note=u'Downloading %s player %s' % (player_type, player_id),
451 errnote=u'Download of %s failed' % player_url)
453 res = self._parse_sig_swf(code)
455 assert False, 'Invalid player type %r' % player_type
459 test_string = u''.join(map(compat_chr, range(slen)))
460 cache_res = res(test_string)
461 cache_spec = [ord(c) for c in cache_res]
463 os.makedirs(os.path.dirname(cache_fn))
464 except OSError as ose:
465 if ose.errno != errno.EEXIST:
467 write_json_file(cache_spec, cache_fn)
469 tb = traceback.format_exc()
470 self._downloader.report_warning(
471 u'Writing cache to %r failed: %s' % (cache_fn, tb))
475 def _print_sig_code(self, func, slen):
476 def gen_sig_code(idxs):
477 def _genslice(start, end, step):
478 starts = u'' if start == 0 else str(start)
479 ends = (u':%d' % (end+step)) if end + step >= 0 else u':'
480 steps = u'' if step == 1 else (u':%d' % step)
481 return u's[%s%s%s]' % (starts, ends, steps)
484 start = '(Never used)' # Quelch pyflakes warnings - start will be
485 # set as soon as step is set
486 for i, prev in zip(idxs[1:], idxs[:-1]):
490 yield _genslice(start, prev, step)
493 if i - prev in [-1, 1]:
498 yield u's[%d]' % prev
502 yield _genslice(start, i, step)
504 test_string = u''.join(map(compat_chr, range(slen)))
505 cache_res = func(test_string)
506 cache_spec = [ord(c) for c in cache_res]
507 expr_code = u' + '.join(gen_sig_code(cache_spec))
508 code = u'if len(s) == %d:\n return %s\n' % (slen, expr_code)
509 self.to_screen(u'Extracted signature function:\n' + code)
511 def _parse_sig_js(self, jscode):
512 funcname = self._search_regex(
513 r'signature=([a-zA-Z]+)', jscode,
514 u'Initial JS player signature function name')
519 return string.lowercase.index(varname)
521 def interpret_statement(stmt, local_vars, allow_recursion=20):
522 if allow_recursion < 0:
523 raise ExtractorError(u'Recursion limit reached')
525 if stmt.startswith(u'var '):
526 stmt = stmt[len(u'var '):]
527 ass_m = re.match(r'^(?P<out>[a-z]+)(?:\[(?P<index>[^\]]+)\])?' +
528 r'=(?P<expr>.*)$', stmt)
530 if ass_m.groupdict().get('index'):
532 lvar = local_vars[ass_m.group('out')]
533 idx = interpret_expression(ass_m.group('index'),
534 local_vars, allow_recursion)
535 assert isinstance(idx, int)
538 expr = ass_m.group('expr')
541 local_vars[ass_m.group('out')] = val
543 expr = ass_m.group('expr')
544 elif stmt.startswith(u'return '):
546 expr = stmt[len(u'return '):]
548 raise ExtractorError(
549 u'Cannot determine left side of statement in %r' % stmt)
551 v = interpret_expression(expr, local_vars, allow_recursion)
554 def interpret_expression(expr, local_vars, allow_recursion):
559 return local_vars[expr]
561 m = re.match(r'^(?P<in>[a-z]+)\.(?P<member>.*)$', expr)
563 member = m.group('member')
564 val = local_vars[m.group('in')]
565 if member == 'split("")':
567 if member == 'join("")':
569 if member == 'length':
571 if member == 'reverse()':
573 slice_m = re.match(r'slice\((?P<idx>.*)\)', member)
575 idx = interpret_expression(
576 slice_m.group('idx'), local_vars, allow_recursion-1)
580 r'^(?P<in>[a-z]+)\[(?P<idx>.+)\]$', expr)
582 val = local_vars[m.group('in')]
583 idx = interpret_expression(m.group('idx'), local_vars,
587 m = re.match(r'^(?P<a>.+?)(?P<op>[%])(?P<b>.+?)$', expr)
589 a = interpret_expression(m.group('a'),
590 local_vars, allow_recursion)
591 b = interpret_expression(m.group('b'),
592 local_vars, allow_recursion)
596 r'^(?P<func>[a-zA-Z]+)\((?P<args>[a-z0-9,]+)\)$', expr)
598 fname = m.group('func')
599 if fname not in functions:
600 functions[fname] = extract_function(fname)
601 argvals = [int(v) if v.isdigit() else local_vars[v]
602 for v in m.group('args').split(',')]
603 return functions[fname](argvals)
604 raise ExtractorError(u'Unsupported JS expression %r' % expr)
606 def extract_function(funcname):
608 r'function ' + re.escape(funcname) +
609 r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
611 argnames = func_m.group('args').split(',')
614 local_vars = dict(zip(argnames, args))
615 for stmt in func_m.group('code').split(';'):
616 res = interpret_statement(stmt, local_vars)
620 initial_function = extract_function(funcname)
621 return lambda s: initial_function([s])
623 def _parse_sig_swf(self, file_contents):
624 if file_contents[1:3] != b'WS':
625 raise ExtractorError(
626 u'Not an SWF file; header is %r' % file_contents[:3])
627 if file_contents[:1] == b'C':
628 content = zlib.decompress(file_contents[8:])
630 raise NotImplementedError(u'Unsupported compression format %r' %
633 def extract_tags(content):
635 while pos < len(content):
636 header16 = struct.unpack('<H', content[pos:pos+2])[0]
638 tag_code = header16 >> 6
639 tag_len = header16 & 0x3f
641 tag_len = struct.unpack('<I', content[pos:pos+4])[0]
643 assert pos+tag_len <= len(content)
644 yield (tag_code, content[pos:pos+tag_len])
648 for tag_code, tag in extract_tags(content)
650 p = code_tag.index(b'\0', 4) + 1
651 code_reader = io.BytesIO(code_tag[p:])
653 # Parse ABC (AVM2 ByteCode)
654 def read_int(reader=None):
662 b = struct.unpack('<B', buf)[0]
663 res = res | ((b & 0x7f) << shift)
669 def u30(reader=None):
670 res = read_int(reader)
671 assert res & 0xf0000000 == 0
675 def s32(reader=None):
677 if v & 0x80000000 != 0:
678 v = - ((v ^ 0xffffffff) + 1)
681 def read_string(reader=None):
685 resb = reader.read(slen)
686 assert len(resb) == slen
687 return resb.decode('utf-8')
689 def read_bytes(count, reader=None):
692 resb = reader.read(count)
693 assert len(resb) == count
696 def read_byte(reader=None):
697 resb = read_bytes(1, reader=reader)
698 res = struct.unpack('<B', resb)[0]
701 # minor_version + major_version
706 for _c in range(1, int_count):
709 for _c in range(1, uint_count):
712 read_bytes((double_count-1) * 8)
714 constant_strings = [u'']
715 for _c in range(1, string_count):
717 constant_strings.append(s)
718 namespace_count = u30()
719 for _c in range(1, namespace_count):
723 for _c in range(1, ns_set_count):
725 for _c2 in range(count):
727 multiname_count = u30()
736 0x0e: 2, # MultinameA
737 0x1b: 1, # MultinameL
738 0x1c: 1, # MultinameLA
741 for _c in range(1, multiname_count):
743 assert kind in MULTINAME_SIZES, u'Invalid multiname kind %r' % kind
745 u30() # namespace_idx
747 multinames.append(constant_strings[name_idx])
749 multinames.append('[MULTINAME kind: %d]' % kind)
750 for _c2 in range(MULTINAME_SIZES[kind]):
755 MethodInfo = collections.namedtuple(
757 ['NEED_ARGUMENTS', 'NEED_REST'])
759 for method_id in range(method_count):
762 for _ in range(param_count):
764 u30() # name index (always 0 for youtube)
766 if flags & 0x08 != 0:
769 for c in range(option_count):
772 if flags & 0x80 != 0:
773 # Param names present
774 for _ in range(param_count):
776 mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
777 method_infos.append(mi)
780 metadata_count = u30()
781 for _c in range(metadata_count):
784 for _c2 in range(item_count):
788 def parse_traits_info():
789 trait_name_idx = u30()
790 kind_full = read_byte()
791 kind = kind_full & 0x0f
792 attrs = kind_full >> 4
794 if kind in [0x00, 0x06]: # Slot or Const
796 u30() # type_name_idx
800 elif kind in [0x01, 0x02, 0x03]: # Method / Getter / Setter
803 methods[multinames[trait_name_idx]] = method_idx
804 elif kind == 0x04: # Class
807 elif kind == 0x05: # Function
810 methods[function_idx] = multinames[trait_name_idx]
812 raise ExtractorError(u'Unsupported trait kind %d' % kind)
814 if attrs & 0x4 != 0: # Metadata present
815 metadata_count = u30()
816 for _c3 in range(metadata_count):
817 u30() # metadata index
822 TARGET_CLASSNAME = u'SignatureDecipher'
823 searched_idx = multinames.index(TARGET_CLASSNAME)
824 searched_class_id = None
826 for class_id in range(class_count):
828 if name_idx == searched_idx:
829 # We found the class we're looking for!
830 searched_class_id = class_id
831 u30() # super_name idx
833 if flags & 0x08 != 0: # Protected namespace is present
834 u30() # protected_ns_idx
836 for _c2 in range(intrf_count):
840 for _c2 in range(trait_count):
843 if searched_class_id is None:
844 raise ExtractorError(u'Target class %r not found' %
849 for class_id in range(class_count):
852 for _c2 in range(trait_count):
853 trait_methods = parse_traits_info()
854 if class_id == searched_class_id:
855 method_names.update(trait_methods.items())
856 method_idxs.update(dict(
858 for name, idx in trait_methods.items()))
862 for _c in range(script_count):
865 for _c2 in range(trait_count):
869 method_body_count = u30()
870 Method = collections.namedtuple('Method', ['code', 'local_count'])
872 for _c in range(method_body_count):
876 u30() # init_scope_depth
877 u30() # max_scope_depth
879 code = read_bytes(code_length)
880 if method_idx in method_idxs:
881 m = Method(code, local_count)
882 methods[method_idxs[method_idx]] = m
883 exception_count = u30()
884 for _c2 in range(exception_count):
891 for _c2 in range(trait_count):
894 assert p + code_reader.tell() == len(code_tag)
895 assert len(methods) == len(method_idxs)
897 method_pyfunctions = {}
899 def extract_function(func_name):
900 if func_name in method_pyfunctions:
901 return method_pyfunctions[func_name]
902 if func_name not in methods:
903 raise ExtractorError(u'Cannot find function %r' % func_name)
904 m = methods[func_name]
907 registers = ['(this)'] + list(args) + [None] * m.local_count
909 coder = io.BytesIO(m.code)
911 opcode = struct.unpack('!B', coder.read(1))[0]
912 if opcode == 36: # pushbyte
913 v = struct.unpack('!B', coder.read(1))[0]
915 elif opcode == 44: # pushstring
917 stack.append(constant_strings[idx])
918 elif opcode == 48: # pushscope
919 # We don't implement the scope register, so we'll just
920 # ignore the popped value
922 elif opcode == 70: # callproperty
924 mname = multinames[index]
925 arg_count = u30(coder)
926 args = list(reversed(
927 [stack.pop() for _ in range(arg_count)]))
929 if mname == u'split':
930 assert len(args) == 1
931 assert isinstance(args[0], compat_str)
932 assert isinstance(obj, compat_str)
936 res = obj.split(args[0])
938 elif mname == u'slice':
939 assert len(args) == 1
940 assert isinstance(args[0], int)
941 assert isinstance(obj, list)
944 elif mname == u'join':
945 assert len(args) == 1
946 assert isinstance(args[0], compat_str)
947 assert isinstance(obj, list)
948 res = args[0].join(obj)
950 elif mname in method_pyfunctions:
951 stack.append(method_pyfunctions[mname](args))
953 raise NotImplementedError(
954 u'Unsupported property %r on %r'
956 elif opcode == 72: # returnvalue
959 elif opcode == 79: # callpropvoid
961 mname = multinames[index]
962 arg_count = u30(coder)
963 args = list(reversed(
964 [stack.pop() for _ in range(arg_count)]))
966 if mname == u'reverse':
967 assert isinstance(obj, list)
970 raise NotImplementedError(
971 u'Unsupported (void) property %r on %r'
973 elif opcode == 93: # findpropstrict
975 mname = multinames[index]
976 res = extract_function(mname)
978 elif opcode == 97: # setproperty
983 assert isinstance(obj, list)
984 assert isinstance(idx, int)
986 elif opcode == 98: # getlocal
988 stack.append(registers[index])
989 elif opcode == 99: # setlocal
992 registers[index] = value
993 elif opcode == 102: # getproperty
995 pname = multinames[index]
996 if pname == u'length':
998 assert isinstance(obj, list)
999 stack.append(len(obj))
1000 else: # Assume attribute access
1002 assert isinstance(idx, int)
1004 assert isinstance(obj, list)
1005 stack.append(obj[idx])
1006 elif opcode == 128: # coerce
1008 elif opcode == 133: # coerce_s
1009 assert isinstance(stack[-1], (type(None), compat_str))
1010 elif opcode == 164: # modulo
1011 value2 = stack.pop()
1012 value1 = stack.pop()
1013 res = value1 % value2
1015 elif opcode == 208: # getlocal_0
1016 stack.append(registers[0])
1017 elif opcode == 209: # getlocal_1
1018 stack.append(registers[1])
1019 elif opcode == 210: # getlocal_2
1020 stack.append(registers[2])
1021 elif opcode == 211: # getlocal_3
1022 stack.append(registers[3])
1023 elif opcode == 214: # setlocal_2
1024 registers[2] = stack.pop()
1025 elif opcode == 215: # setlocal_3
1026 registers[3] = stack.pop()
1028 raise NotImplementedError(
1029 u'Unsupported opcode %d' % opcode)
1031 method_pyfunctions[func_name] = resfunc
1034 initial_function = extract_function(u'decipher')
1035 return lambda s: initial_function([s])
1037 def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
1038 """Turn the encrypted s field into a working signature"""
1040 if player_url is not None:
1042 player_id = (player_url, len(s))
1043 if player_id not in self._player_cache:
1044 func = self._extract_signature_function(
1045 video_id, player_url, len(s)
1047 self._player_cache[player_id] = func
1048 func = self._player_cache[player_id]
1049 if self._downloader.params.get('youtube_print_sig_code'):
1050 self._print_sig_code(func, len(s))
1053 tb = traceback.format_exc()
1054 self._downloader.report_warning(
1055 u'Automatic signature extraction failed: ' + tb)
1057 self._downloader.report_warning(
1058 u'Warning: Falling back to static signature algorithm')
1060 return self._static_decrypt_signature(
1061 s, video_id, player_url, age_gate)
1063 def _static_decrypt_signature(self, s, video_id, player_url, age_gate):
1065 # The videos with age protection use another player, so the
1066 # algorithms can be different.
1068 return s[2:63] + s[82] + s[64:82] + s[63]
1071 return s[86:29:-1] + s[88] + s[28:5:-1]
1073 return s[25] + s[3:25] + s[0] + s[26:42] + s[79] + s[43:79] + s[91] + s[80:83]
1075 return s[84:27:-1] + s[86] + s[26:5:-1]
1077 return s[25] + s[3:25] + s[2] + s[26:40] + s[77] + s[41:77] + s[89] + s[78:81]
1079 return s[84:78:-1] + s[87] + s[77:60:-1] + s[0] + s[59:3:-1]
1081 return s[7:28] + s[87] + s[29:45] + s[55] + s[46:55] + s[2] + s[56:87] + s[28]
1083 return s[6:27] + s[4] + s[28:39] + s[27] + s[40:59] + s[2] + s[60:]
1085 return s[80:72:-1] + s[16] + s[71:39:-1] + s[72] + s[38:16:-1] + s[82] + s[15::-1]
1087 return s[3:11] + s[0] + s[12:55] + s[84] + s[56:84]
1089 return s[78:70:-1] + s[14] + s[69:37:-1] + s[70] + s[36:14:-1] + s[80] + s[:14][::-1]
1091 return s[80:63:-1] + s[0] + s[62:0:-1] + s[63]
1093 return s[80:37:-1] + s[7] + s[36:7:-1] + s[0] + s[6:0:-1] + s[37]
1095 return s[56] + s[79:56:-1] + s[41] + s[55:41:-1] + s[80] + s[40:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
1097 return s[1:19] + s[0] + s[20:68] + s[19] + s[69:80]
1099 return s[54] + s[77:54:-1] + s[39] + s[53:39:-1] + s[78] + s[38:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
1102 raise ExtractorError(u'Unable to decrypt signature, key length %d not supported; retrying might work' % (len(s)))
1104 def _get_available_subtitles(self, video_id):
1106 sub_list = self._download_webpage(
1107 'http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
1108 video_id, note=False)
1109 except ExtractorError as err:
1110 self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
1112 lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
1117 params = compat_urllib_parse.urlencode({
1120 'fmt': self._downloader.params.get('subtitlesformat'),
1123 url = u'http://www.youtube.com/api/timedtext?' + params
1124 sub_lang_list[lang] = url
1125 if not sub_lang_list:
1126 self._downloader.report_warning(u'video doesn\'t have subtitles')
1128 return sub_lang_list
1130 def _get_available_automatic_caption(self, video_id, webpage):
1131 """We need the webpage for getting the captions url, pass it as an
1132 argument to speed up the process."""
1133 sub_format = self._downloader.params.get('subtitlesformat')
1134 self.to_screen(u'%s: Looking for automatic captions' % video_id)
1135 mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
1136 err_msg = u'Couldn\'t find automatic captions for %s' % video_id
1138 self._downloader.report_warning(err_msg)
1140 player_config = json.loads(mobj.group(1))
1142 args = player_config[u'args']
1143 caption_url = args[u'ttsurl']
1144 timestamp = args[u'timestamp']
1145 # We get the available subtitles
1146 list_params = compat_urllib_parse.urlencode({
1151 list_url = caption_url + '&' + list_params
1152 list_page = self._download_webpage(list_url, video_id)
1153 caption_list = xml.etree.ElementTree.fromstring(list_page.encode('utf-8'))
1154 original_lang_node = caption_list.find('track')
1155 if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr' :
1156 self._downloader.report_warning(u'Video doesn\'t have automatic captions')
1158 original_lang = original_lang_node.attrib['lang_code']
1161 for lang_node in caption_list.findall('target'):
1162 sub_lang = lang_node.attrib['lang_code']
1163 params = compat_urllib_parse.urlencode({
1164 'lang': original_lang,
1170 sub_lang_list[sub_lang] = caption_url + '&' + params
1171 return sub_lang_list
1172 # An extractor error can be raise by the download process if there are
1173 # no automatic captions but there are subtitles
1174 except (KeyError, ExtractorError):
1175 self._downloader.report_warning(err_msg)
1178 def _print_formats(self, formats):
1179 print('Available formats:')
1181 print('%s\t:\t%s\t[%s]%s' %(x, self._video_extensions.get(x, 'flv'),
1182 self._video_dimensions.get(x, '???'),
1183 ' ('+self._special_itags[x]+')' if x in self._special_itags else ''))
1185 def _extract_id(self, url):
1186 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
1188 raise ExtractorError(u'Invalid URL: %s' % url)
1189 video_id = mobj.group(2)
1192 def _get_video_url_list(self, url_map):
1194 Transform a dictionary in the format {itag:url} to a list of (itag, url)
1195 with the requested formats.
1197 req_format = self._downloader.params.get('format', None)
1198 format_limit = self._downloader.params.get('format_limit', None)
1199 available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
1200 if format_limit is not None and format_limit in available_formats:
1201 format_list = available_formats[available_formats.index(format_limit):]
1203 format_list = available_formats
1204 existing_formats = [x for x in format_list if x in url_map]
1205 if len(existing_formats) == 0:
1206 raise ExtractorError(u'no known formats available for video')
1207 if self._downloader.params.get('listformats', None):
1208 self._print_formats(existing_formats)
1210 if req_format is None or req_format == 'best':
1211 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
1212 elif req_format == 'worst':
1213 video_url_list = [(existing_formats[-1], url_map[existing_formats[-1]])] # worst quality
1214 elif req_format in ('-1', 'all'):
1215 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
1217 # Specific formats. We pick the first in a slash-delimeted sequence.
1218 # Format can be specified as itag or 'mp4' or 'flv' etc. We pick the highest quality
1219 # available in the specified format. For example,
1220 # if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
1221 # if '1/mp4/3/4' is requested and '1' and '5' (is a mp4) are available, we pick '1'.
1222 # if '1/mp4/3/4' is requested and '4' and '5' (is a mp4) are available, we pick '5'.
1223 req_formats = req_format.split('/')
1224 video_url_list = None
1225 for rf in req_formats:
1227 video_url_list = [(rf, url_map[rf])]
1229 if rf in self._video_formats_map:
1230 for srf in self._video_formats_map[rf]:
1232 video_url_list = [(srf, url_map[srf])]
1237 if video_url_list is None:
1238 raise ExtractorError(u'requested format not available')
1239 return video_url_list
1241 def _extract_from_m3u8(self, manifest_url, video_id):
1243 def _get_urls(_manifest):
1244 lines = _manifest.split('\n')
1245 urls = filter(lambda l: l and not l.startswith('#'),
1248 manifest = self._download_webpage(manifest_url, video_id, u'Downloading formats manifest')
1249 formats_urls = _get_urls(manifest)
1250 for format_url in formats_urls:
1251 itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
1252 url_map[itag] = format_url
1255 def _extract_annotations(self, video_id):
1256 url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
1257 return self._download_webpage(url, video_id, note=u'Searching for annotations.', errnote=u'Unable to download video annotations.')
1259 def _real_extract(self, url):
1260 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1261 mobj = re.search(self._NEXT_URL_RE, url)
1263 url = 'https://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
1264 video_id = self._extract_id(url)
1267 self.report_video_webpage_download(video_id)
1268 url = 'https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
1269 request = compat_urllib_request.Request(url)
1271 video_webpage_bytes = compat_urllib_request.urlopen(request).read()
1272 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1273 raise ExtractorError(u'Unable to download video webpage: %s' % compat_str(err))
1275 video_webpage = video_webpage_bytes.decode('utf-8', 'ignore')
1277 # Attempt to extract SWF player URL
1278 mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1279 if mobj is not None:
1280 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1285 self.report_video_info_webpage_download(video_id)
1286 if re.search(r'player-age-gate-content">', video_webpage) is not None:
1287 self.report_age_confirmation()
1289 # We simulate the access to the video from www.youtube.com/v/{video_id}
1290 # this can be viewed without login into Youtube
1291 data = compat_urllib_parse.urlencode({'video_id': video_id,
1295 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1299 video_info_url = 'https://www.youtube.com/get_video_info?' + data
1300 video_info_webpage = self._download_webpage(video_info_url, video_id,
1302 errnote='unable to download video info webpage')
1303 video_info = compat_parse_qs(video_info_webpage)
1306 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
1307 video_info_url = ('https://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1308 % (video_id, el_type))
1309 video_info_webpage = self._download_webpage(video_info_url, video_id,
1311 errnote='unable to download video info webpage')
1312 video_info = compat_parse_qs(video_info_webpage)
1313 if 'token' in video_info:
1315 if 'token' not in video_info:
1316 if 'reason' in video_info:
1317 raise ExtractorError(u'YouTube said: %s' % video_info['reason'][0], expected=True)
1319 raise ExtractorError(u'"token" parameter not in video info for unknown reason')
1321 # Check for "rental" videos
1322 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1323 raise ExtractorError(u'"rental" videos not supported')
1325 # Start extracting information
1326 self.report_information_extraction(video_id)
1329 if 'author' not in video_info:
1330 raise ExtractorError(u'Unable to extract uploader name')
1331 video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
1334 video_uploader_id = None
1335 mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
1336 if mobj is not None:
1337 video_uploader_id = mobj.group(1)
1339 self._downloader.report_warning(u'unable to extract uploader nickname')
1342 if 'title' in video_info:
1343 video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
1345 self._downloader.report_warning(u'Unable to extract video title')
1349 # We try first to get a high quality image:
1350 m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
1351 video_webpage, re.DOTALL)
1352 if m_thumb is not None:
1353 video_thumbnail = m_thumb.group(1)
1354 elif 'thumbnail_url' not in video_info:
1355 self._downloader.report_warning(u'unable to extract video thumbnail')
1356 video_thumbnail = None
1357 else: # don't panic if we can't find it
1358 video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
1362 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
1363 if mobj is not None:
1364 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1365 upload_date = unified_strdate(upload_date)
1368 video_description = get_element_by_id("eow-description", video_webpage)
1369 if video_description:
1370 video_description = clean_html(video_description)
1372 fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
1374 video_description = unescapeHTML(fd_mobj.group(1))
1376 video_description = u''
1379 video_subtitles = self.extract_subtitles(video_id, video_webpage)
1381 if self._downloader.params.get('listsubtitles', False):
1382 self._list_available_subtitles(video_id, video_webpage)
1385 if 'length_seconds' not in video_info:
1386 self._downloader.report_warning(u'unable to extract video duration')
1389 video_duration = compat_urllib_parse.unquote_plus(video_info['length_seconds'][0])
1392 video_annotations = None
1393 if self._downloader.params.get('writeannotations', False):
1394 video_annotations = self._extract_annotations(video_id)
1396 # Decide which formats to download
1399 mobj = re.search(r';ytplayer.config = ({.*?});', video_webpage)
1401 raise ValueError('Could not find vevo ID')
1402 info = json.loads(mobj.group(1))
1404 # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
1405 # this signatures are encrypted
1406 if 'url_encoded_fmt_stream_map' not in args:
1407 raise ValueError(u'No stream_map present') # caught below
1408 re_signature = re.compile(r'[&,]s=')
1409 m_s = re_signature.search(args['url_encoded_fmt_stream_map'])
1411 self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
1412 video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
1413 m_s = re_signature.search(args.get('adaptive_fmts', u''))
1415 if 'adaptive_fmts' in video_info:
1416 video_info['adaptive_fmts'][0] += ',' + args['adaptive_fmts']
1418 video_info['adaptive_fmts'] = [args['adaptive_fmts']]
1422 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1423 self.report_rtmp_download()
1424 video_url_list = [(None, video_info['conn'][0])]
1425 elif len(video_info.get('url_encoded_fmt_stream_map', [])) >= 1 or len(video_info.get('adaptive_fmts', [])) >= 1:
1426 encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts',[''])[0]
1427 if 'rtmpe%3Dyes' in encoded_url_map:
1428 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
1430 for url_data_str in encoded_url_map.split(','):
1431 url_data = compat_parse_qs(url_data_str)
1432 if 'itag' in url_data and 'url' in url_data:
1433 url = url_data['url'][0]
1434 if 'sig' in url_data:
1435 url += '&signature=' + url_data['sig'][0]
1436 elif 's' in url_data:
1437 encrypted_sig = url_data['s'][0]
1438 if self._downloader.params.get('verbose'):
1440 if player_url is None:
1441 player_version = 'unknown'
1443 player_version = self._search_regex(
1444 r'-(.+)\.swf$', player_url,
1445 u'flash player', fatal=False)
1446 player_desc = 'flash player %s' % player_version
1448 player_version = self._search_regex(
1449 r'html5player-(.+?)\.js', video_webpage,
1450 'html5 player', fatal=False)
1451 player_desc = u'html5 player %s' % player_version
1453 parts_sizes = u'.'.join(compat_str(len(part)) for part in encrypted_sig.split('.'))
1454 self.to_screen(u'encrypted signature length %d (%s), itag %s, %s' %
1455 (len(encrypted_sig), parts_sizes, url_data['itag'][0], player_desc))
1458 jsplayer_url_json = self._search_regex(
1459 r'"assets":.+?"js":\s*("[^"]+")',
1460 video_webpage, u'JS player URL')
1461 player_url = json.loads(jsplayer_url_json)
1463 signature = self._decrypt_signature(
1464 encrypted_sig, video_id, player_url, age_gate)
1465 url += '&signature=' + signature
1466 if 'ratebypass' not in url:
1467 url += '&ratebypass=yes'
1468 url_map[url_data['itag'][0]] = url
1469 video_url_list = self._get_video_url_list(url_map)
1470 if not video_url_list:
1472 elif video_info.get('hlsvp'):
1473 manifest_url = video_info['hlsvp'][0]
1474 url_map = self._extract_from_m3u8(manifest_url, video_id)
1475 video_url_list = self._get_video_url_list(url_map)
1476 if not video_url_list:
1480 raise ExtractorError(u'no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
1483 for itag, video_real_url in video_url_list:
1485 video_extension = self._video_extensions.get(itag, 'flv')
1487 video_format = '{0} - {1}{2}'.format(itag if itag else video_extension,
1488 self._video_dimensions.get(itag, '???'),
1489 ' ('+self._special_itags[itag]+')' if itag in self._special_itags else '')
1493 'url': video_real_url,
1494 'uploader': video_uploader,
1495 'uploader_id': video_uploader_id,
1496 'upload_date': upload_date,
1497 'title': video_title,
1498 'ext': video_extension,
1499 'format': video_format,
1501 'thumbnail': video_thumbnail,
1502 'description': video_description,
1503 'player_url': player_url,
1504 'subtitles': video_subtitles,
1505 'duration': video_duration,
1506 'age_limit': 18 if age_gate else 0,
1507 'annotations': video_annotations
1511 class YoutubePlaylistIE(InfoExtractor):
1512 IE_DESC = u'YouTube.com playlists'
1513 _VALID_URL = r"""(?:
1518 (?:course|view_play_list|my_playlists|artist|playlist|watch)
1519 \? (?:.*?&)*? (?:p|a|list)=
1522 ((?:PL|EC|UU|FL)?[0-9A-Za-z-_]{10,})
1525 ((?:PL|EC|UU|FL)[0-9A-Za-z-_]{10,})
1527 _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/playlists/%s?max-results=%i&start-index=%i&v=2&alt=json&safeSearch=none'
1529 IE_NAME = u'youtube:playlist'
1532 def suitable(cls, url):
1533 """Receives a URL and returns True if suitable for this IE."""
1534 return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
1536 def _real_extract(self, url):
1537 # Extract playlist id
1538 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
1540 raise ExtractorError(u'Invalid URL: %s' % url)
1541 playlist_id = mobj.group(1) or mobj.group(2)
1543 # Check if it's a video-specific URL
1544 query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
1545 if 'v' in query_dict:
1546 video_id = query_dict['v'][0]
1547 if self._downloader.params.get('noplaylist'):
1548 self.to_screen(u'Downloading just video %s because of --no-playlist' % video_id)
1549 return self.url_result('https://www.youtube.com/watch?v=' + video_id, 'Youtube')
1551 self.to_screen(u'Downloading playlist PL%s - add --no-playlist to just download video %s' % (playlist_id, video_id))
1553 # Download playlist videos from API
1556 for page_num in itertools.count(1):
1557 start_index = self._MAX_RESULTS * (page_num - 1) + 1
1558 if start_index >= 1000:
1559 self._downloader.report_warning(u'Max number of results reached')
1561 url = self._TEMPLATE_URL % (playlist_id, self._MAX_RESULTS, start_index)
1562 page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num)
1565 response = json.loads(page)
1566 except ValueError as err:
1567 raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
1569 if 'feed' not in response:
1570 raise ExtractorError(u'Got a malformed response from YouTube API')
1571 playlist_title = response['feed']['title']['$t']
1572 if 'entry' not in response['feed']:
1573 # Number of videos is a multiple of self._MAX_RESULTS
1576 for entry in response['feed']['entry']:
1577 index = entry['yt$position']['$t']
1578 if 'media$group' in entry and 'yt$videoid' in entry['media$group']:
1581 'https://www.youtube.com/watch?v=' + entry['media$group']['yt$videoid']['$t']
1584 videos = [v[1] for v in sorted(videos)]
1586 url_results = [self.url_result(vurl, 'Youtube') for vurl in videos]
1587 return [self.playlist_result(url_results, playlist_id, playlist_title)]
1590 class YoutubeChannelIE(InfoExtractor):
1591 IE_DESC = u'YouTube.com channels'
1592 _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
1593 _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
1594 _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
1595 _MORE_PAGES_URL = 'http://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
1596 IE_NAME = u'youtube:channel'
1598 def extract_videos_from_page(self, page):
1600 for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
1601 if mobj.group(1) not in ids_in_page:
1602 ids_in_page.append(mobj.group(1))
1605 def _real_extract(self, url):
1606 # Extract channel id
1607 mobj = re.match(self._VALID_URL, url)
1609 raise ExtractorError(u'Invalid URL: %s' % url)
1611 # Download channel page
1612 channel_id = mobj.group(1)
1616 url = self._TEMPLATE_URL % (channel_id, pagenum)
1617 page = self._download_webpage(url, channel_id,
1618 u'Downloading page #%s' % pagenum)
1620 # Extract video identifiers
1621 ids_in_page = self.extract_videos_from_page(page)
1622 video_ids.extend(ids_in_page)
1624 # Download any subsequent channel pages using the json-based channel_ajax query
1625 if self._MORE_PAGES_INDICATOR in page:
1626 for pagenum in itertools.count(1):
1627 url = self._MORE_PAGES_URL % (pagenum, channel_id)
1628 page = self._download_webpage(url, channel_id,
1629 u'Downloading page #%s' % pagenum)
1631 page = json.loads(page)
1633 ids_in_page = self.extract_videos_from_page(page['content_html'])
1634 video_ids.extend(ids_in_page)
1636 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
1639 self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
1641 urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids]
1642 url_entries = [self.url_result(eurl, 'Youtube') for eurl in urls]
1643 return [self.playlist_result(url_entries, channel_id)]
1646 class YoutubeUserIE(InfoExtractor):
1647 IE_DESC = u'YouTube.com user videos (URL or "ytuser" keyword)'
1648 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
1649 _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1650 _GDATA_PAGE_SIZE = 50
1651 _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
1652 IE_NAME = u'youtube:user'
1655 def suitable(cls, url):
1656 # Don't return True if the url can be extracted with other youtube
1657 # extractor, the regex would is too permissive and it would match.
1658 other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
1659 if any(ie.suitable(url) for ie in other_ies): return False
1660 else: return super(YoutubeUserIE, cls).suitable(url)
1662 def _real_extract(self, url):
1664 mobj = re.match(self._VALID_URL, url)
1666 raise ExtractorError(u'Invalid URL: %s' % url)
1668 username = mobj.group(1)
1670 # Download video ids using YouTube Data API. Result size per
1671 # query is limited (currently to 50 videos) so we need to query
1672 # page by page until there are no video ids - it means we got
1677 for pagenum in itertools.count(0):
1678 start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1680 gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
1681 page = self._download_webpage(gdata_url, username,
1682 u'Downloading video ids from %d to %d' % (start_index, start_index + self._GDATA_PAGE_SIZE))
1685 response = json.loads(page)
1686 except ValueError as err:
1687 raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
1688 if 'entry' not in response['feed']:
1689 # Number of videos is a multiple of self._MAX_RESULTS
1692 # Extract video identifiers
1694 for entry in response['feed']['entry']:
1695 ids_in_page.append(entry['id']['$t'].split('/')[-1])
1696 video_ids.extend(ids_in_page)
1698 # A little optimization - if current page is not
1699 # "full", ie. does not contain PAGE_SIZE video ids then
1700 # we can assume that this page is the last one - there
1701 # are no more ids on further pages - no need to query
1704 if len(ids_in_page) < self._GDATA_PAGE_SIZE:
1707 urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids]
1708 url_results = [self.url_result(rurl, 'Youtube') for rurl in urls]
1709 return [self.playlist_result(url_results, playlist_title = username)]
1711 class YoutubeSearchIE(SearchInfoExtractor):
1712 IE_DESC = u'YouTube.com searches'
1713 _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1715 IE_NAME = u'youtube:search'
1716 _SEARCH_KEY = 'ytsearch'
1718 def report_download_page(self, query, pagenum):
1719 """Report attempt to download search page with given number."""
1720 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1722 def _get_n_results(self, query, n):
1723 """Get a specified number of results for a query"""
1729 while (50 * pagenum) < limit:
1730 self.report_download_page(query, pagenum+1)
1731 result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
1732 request = compat_urllib_request.Request(result_url)
1734 data = compat_urllib_request.urlopen(request).read().decode('utf-8')
1735 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1736 raise ExtractorError(u'Unable to download API page: %s' % compat_str(err))
1737 api_response = json.loads(data)['data']
1739 if not 'items' in api_response:
1740 raise ExtractorError(u'[youtube] No video results')
1742 new_ids = list(video['id'] for video in api_response['items'])
1743 video_ids += new_ids
1745 limit = min(n, api_response['totalItems'])
1748 if len(video_ids) > n:
1749 video_ids = video_ids[:n]
1750 videos = [self.url_result('http://www.youtube.com/watch?v=%s' % id, 'Youtube') for id in video_ids]
1751 return self.playlist_result(videos, query)
1754 class YoutubeShowIE(InfoExtractor):
1755 IE_DESC = u'YouTube.com (multi-season) shows'
1756 _VALID_URL = r'https?://www\.youtube\.com/show/(.*)'
1757 IE_NAME = u'youtube:show'
1759 def _real_extract(self, url):
1760 mobj = re.match(self._VALID_URL, url)
1761 show_name = mobj.group(1)
1762 webpage = self._download_webpage(url, show_name, u'Downloading show webpage')
1763 # There's one playlist for each season of the show
1764 m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
1765 self.to_screen(u'%s: Found %s seasons' % (show_name, len(m_seasons)))
1766 return [self.url_result('https://www.youtube.com' + season.group(1), 'YoutubePlaylist') for season in m_seasons]
1769 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
1771 Base class for extractors that fetch info from
1772 http://www.youtube.com/feed_ajax
1773 Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
1775 _LOGIN_REQUIRED = True
1777 # use action_load_personal_feed instead of action_load_system_feed
1778 _PERSONAL_FEED = False
1781 def _FEED_TEMPLATE(self):
1782 action = 'action_load_system_feed'
1783 if self._PERSONAL_FEED:
1784 action = 'action_load_personal_feed'
1785 return 'http://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME)
1789 return u'youtube:%s' % self._FEED_NAME
1791 def _real_initialize(self):
1794 def _real_extract(self, url):
1796 # The step argument is available only in 2.7 or higher
1797 for i in itertools.count(0):
1798 paging = i*self._PAGING_STEP
1799 info = self._download_webpage(self._FEED_TEMPLATE % paging,
1800 u'%s feed' % self._FEED_NAME,
1801 u'Downloading page %s' % i)
1802 info = json.loads(info)
1803 feed_html = info['feed_html']
1804 m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
1805 ids = orderedSet(m.group(1) for m in m_ids)
1806 feed_entries.extend(self.url_result(id, 'Youtube') for id in ids)
1807 if info['paging'] is None:
1809 return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
1811 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
1812 IE_DESC = u'YouTube.com subscriptions feed, "ytsubs" keyword(requires authentication)'
1813 _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
1814 _FEED_NAME = 'subscriptions'
1815 _PLAYLIST_TITLE = u'Youtube Subscriptions'
1817 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
1818 IE_DESC = u'YouTube.com recommended videos, "ytrec" keyword (requires authentication)'
1819 _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
1820 _FEED_NAME = 'recommended'
1821 _PLAYLIST_TITLE = u'Youtube Recommended videos'
1823 class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
1824 IE_DESC = u'Youtube watch later list, "ytwatchlater" keyword (requires authentication)'
1825 _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
1826 _FEED_NAME = 'watch_later'
1827 _PLAYLIST_TITLE = u'Youtube Watch Later'
1829 _PERSONAL_FEED = True
1831 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
1832 IE_NAME = u'youtube:favorites'
1833 IE_DESC = u'YouTube.com favourite videos, "ytfav" keyword (requires authentication)'
1834 _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
1835 _LOGIN_REQUIRED = True
1837 def _real_extract(self, url):
1838 webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
1839 playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, u'favourites playlist id')
1840 return self.url_result(playlist_id, 'YoutubePlaylist')
1843 class YoutubeTruncatedURLIE(InfoExtractor):
1844 IE_NAME = 'youtube:truncated_url'
1845 IE_DESC = False # Do not list
1846 _VALID_URL = r'(?:https?://)?[^/]+/watch\?feature=[a-z_]+$'
1848 def _real_extract(self, url):
1849 raise ExtractorError(
1850 u'Did you forget to quote the URL? Remember that & is a meta '
1851 u'character in most shells, so you want to put the URL in quotes, '
1853 u'\'http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc\''
1854 u' (or simply youtube-dl BaW_jenozKc ).',