X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Fyoutube.py;h=6beda8f3b6a983b3ce44e25321d6ac70eea2482e;hb=695dc094ab16a0df40712bf504bc2880c62e598f;hp=45b593a124ab7fe784c62e4a95a382d7078c00ef;hpb=95dbd2f9907416e86424e4372dbd2593c1699e7d;p=youtube-dl diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 45b593a12..6beda8f3b 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1,20 +1,23 @@ # coding: utf-8 import collections -import itertools +import errno import io +import itertools import json -import netrc +import os.path import re import socket import string import struct import traceback +import xml.etree.ElementTree import zlib from .common import InfoExtractor, SearchInfoExtractor from .subtitles import SubtitlesInfoExtractor from ..utils import ( + compat_chr, compat_http_client, compat_parse_qs, compat_urllib_error, @@ -28,6 +31,7 @@ from ..utils import ( unescapeHTML, unified_strdate, orderedSet, + write_json_file, ) class YoutubeBaseInfoExtractor(InfoExtractor): @@ -357,7 +361,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): u"info_dict": { u"upload_date": u"20120506", u"title": u"Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]", - u"description": u"md5:3e2666e0a55044490499ea45fe9037b7", + u"description": u"md5:bdac09887d209a4ed54b8f76b2bdaa8b", u"uploader": u"Icona Pop", u"uploader_id": u"IconaPop" } @@ -400,7 +404,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): def __init__(self, *args, **kwargs): super(YoutubeIE, self).__init__(*args, **kwargs) - self._jsplayer_cache = {} + self._player_cache = {} def report_video_webpage_download(self, video_id): """Report attempt to download video webpage.""" @@ -422,27 +426,100 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): """Indicate the download will use the RTMP protocol.""" self.to_screen(u'RTMP download detected') - def _extract_signature_function(self, video_id, player_url): - id_m = re.match(r'.*-(?P[^.]+)\.(?P[^.]+)$', player_url) + def _extract_signature_function(self, video_id, player_url, slen): + id_m = re.match(r'.*-(?P[a-zA-Z0-9_-]+)\.(?P[a-z]+)$', + player_url) player_type = id_m.group('ext') player_id = id_m.group('id') + # Read from filesystem cache + func_id = '%s_%s_%d' % (player_type, player_id, slen) + assert os.path.basename(func_id) == func_id + cache_dir = self._downloader.params.get('cachedir', + u'~/.youtube-dl/cache') + + cache_enabled = cache_dir is not None + if cache_enabled: + cache_fn = os.path.join(os.path.expanduser(cache_dir), + u'youtube-sigfuncs', + func_id + '.json') + try: + with io.open(cache_fn, 'r', encoding='utf-8') as cachef: + cache_spec = json.load(cachef) + return lambda s: u''.join(s[i] for i in cache_spec) + except IOError: + pass # No cache available + if player_type == 'js': code = self._download_webpage( player_url, video_id, - note=u'Downloading %s player %s' % (player_type, jsplayer_id), + note=u'Downloading %s player %s' % (player_type, player_id), errnote=u'Download of %s failed' % player_url) - return self._parse_sig_js(code) - elif player_tpye == 'swf': + res = self._parse_sig_js(code) + elif player_type == 'swf': urlh = self._request_webpage( player_url, video_id, - note=u'Downloading %s player %s' % (player_type, jsplayer_id), + note=u'Downloading %s player %s' % (player_type, player_id), errnote=u'Download of %s failed' % player_url) code = urlh.read() - return self._parse_sig_swf(code) + res = self._parse_sig_swf(code) else: assert False, 'Invalid player type %r' % player_type + if cache_enabled: + try: + test_string = u''.join(map(compat_chr, range(slen))) + cache_res = res(test_string) + cache_spec = [ord(c) for c in cache_res] + try: + os.makedirs(os.path.dirname(cache_fn)) + except OSError as ose: + if ose.errno != errno.EEXIST: + raise + write_json_file(cache_spec, cache_fn) + except Exception: + tb = traceback.format_exc() + self._downloader.report_warning( + u'Writing cache to %r failed: %s' % (cache_fn, tb)) + + return res + + def _print_sig_code(self, func, slen): + def gen_sig_code(idxs): + def _genslice(start, end, step): + starts = u'' if start == 0 else str(start) + ends = (u':%d' % (end+step)) if end + step >= 0 else u':' + steps = u'' if step == 1 else (u':%d' % step) + return u's[%s%s%s]' % (starts, ends, steps) + + step = None + start = '(Never used)' # Quelch pyflakes warnings - start will be + # set as soon as step is set + for i, prev in zip(idxs[1:], idxs[:-1]): + if step is not None: + if i - prev == step: + continue + yield _genslice(start, prev, step) + step = None + continue + if i - prev in [-1, 1]: + step = i - prev + start = prev + continue + else: + yield u's[%d]' % prev + if step is None: + yield u's[%d]' % i + else: + yield _genslice(start, i, step) + + test_string = u''.join(map(compat_chr, range(slen))) + cache_res = func(test_string) + cache_spec = [ord(c) for c in cache_res] + expr_code = u' + '.join(gen_sig_code(cache_spec)) + code = u'if len(s) == %d:\n return %s\n' % (slen, expr_code) + self.to_screen(u'Extracted signature function:\n' + code) + def _parse_sig_js(self, jscode): funcname = self._search_regex( r'signature=([a-zA-Z]+)', jscode, @@ -455,7 +532,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): def interpret_statement(stmt, local_vars, allow_recursion=20): if allow_recursion < 0: - raise ExctractorError(u'Recursion limit reached') + raise ExtractorError(u'Recursion limit reached') if stmt.startswith(u'var '): stmt = stmt[len(u'var '):] @@ -583,99 +660,83 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): for tag_code, tag in extract_tags(content) if tag_code == 82) p = code_tag.index(b'\0', 4) + 1 + code_reader = io.BytesIO(code_tag[p:]) # Parse ABC (AVM2 ByteCode) - def read_int(data=None, pos=None): - if hasattr(data, 'read'): - assert pos is None - - res = 0 - shift = 0 - for _ in range(5): - buf = data.read(1) - assert len(buf) == 1 - b = struct.unpack('> 4 methods = {} if kind in [0x00, 0x06]: # Slot or Const - _, pos = u30(pos=pos) # Slot id - type_name_idx, pos = u30(pos=pos) - vindex, pos = u30(pos=pos) + u30() # Slot id + u30() # type_name_idx + vindex = u30() if vindex != 0: - _, pos = read_byte(pos=pos) # vkind + read_byte() # vkind elif kind in [0x01, 0x02, 0x03]: # Method / Getter / Setter - _, pos = u30(pos=pos) # disp_id - method_idx, pos = u30(pos=pos) + u30() # disp_id + method_idx = u30() methods[multinames[trait_name_idx]] = method_idx elif kind == 0x04: # Class - _, pos = u30(pos=pos) # slot_id - _, pos = u30(pos=pos) # classi + u30() # slot_id + u30() # classi elif kind == 0x05: # Function - _, pos = u30(pos=pos) # slot_id - function_idx, pos = u30(pos=pos) + u30() # slot_id + function_idx = u30() methods[function_idx] = multinames[trait_name_idx] else: raise ExtractorError(u'Unsupported trait kind %d' % kind) if attrs & 0x4 != 0: # Metadata present - metadata_count, pos = u30(pos=pos) + metadata_count = u30() for _c3 in range(metadata_count): - _, pos = u30(pos=pos) + u30() # metadata index - return (methods, pos) + return methods # Classes TARGET_CLASSNAME = u'SignatureDecipher' searched_idx = multinames.index(TARGET_CLASSNAME) searched_class_id = None - class_count, p = u30() + class_count = u30() for class_id in range(class_count): - name_idx, p = u30() + name_idx = u30() if name_idx == searched_idx: # We found the class we're looking for! searched_class_id = class_id - _, p = u30() # super_name idx - flags, p = read_byte() + u30() # super_name idx + flags = read_byte() if flags & 0x08 != 0: # Protected namespace is present - protected_ns_idx, p = u30() - intrf_count, p = u30() + u30() # protected_ns_idx + intrf_count = u30() for _c2 in range(intrf_count): - _, p = u30() - _, p = u30() # iinit - trait_count, p = u30() + u30() + u30() # iinit + trait_count = u30() for _c2 in range(trait_count): - _, p = parse_traits_info() + parse_traits_info() if searched_class_id is None: raise ExtractorError(u'Target class %r not found' % @@ -800,10 +859,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): method_names = {} method_idxs = {} for class_id in range(class_count): - _, p = u30() # cinit - trait_count, p = u30() + u30() # cinit + trait_count = u30() for _c2 in range(trait_count): - trait_methods, p = parse_traits_info() + trait_methods = parse_traits_info() if class_id == searched_class_id: method_names.update(trait_methods.items()) method_idxs.update(dict( @@ -811,40 +870,40 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): for name, idx in trait_methods.items())) # Scripts - script_count, p = u30() + script_count = u30() for _c in range(script_count): - _, p = u30() # init - trait_count, p = u30() + u30() # init + trait_count = u30() for _c2 in range(trait_count): - _, p = parse_traits_info() + parse_traits_info() # Method bodies - method_body_count, p = u30() + method_body_count = u30() Method = collections.namedtuple('Method', ['code', 'local_count']) methods = {} for _c in range(method_body_count): - method_idx, p = u30() - max_stack, p = u30() - local_count, p = u30() - init_scope_depth, p = u30() - max_scope_depth, p = u30() - code_length, p = u30() + method_idx = u30() + u30() # max_stack + local_count = u30() + u30() # init_scope_depth + u30() # max_scope_depth + code_length = u30() + code = read_bytes(code_length) if method_idx in method_idxs: - m = Method(code_tag[p:p+code_length], local_count) + m = Method(code, local_count) methods[method_idxs[method_idx]] = m - p += code_length - exception_count, p = u30() + exception_count = u30() for _c2 in range(exception_count): - _, p = u30() # from - _, p = u30() # to - _, p = u30() # target - _, p = u30() # exc_type - _, p = u30() # var_name - trait_count, p = u30() + u30() # from + u30() # to + u30() # target + u30() # exc_type + u30() # var_name + trait_count = u30() for _c2 in range(trait_count): - _, p = parse_traits_info() + parse_traits_info() - assert p == len(code_tag) + assert p + code_reader.tell() == len(code_tag) assert len(methods) == len(method_idxs) method_pyfunctions = {} @@ -957,7 +1016,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): assert isinstance(obj, list) stack.append(obj[idx]) elif opcode == 128: # coerce - _ = u30(coder) + u30(coder) elif opcode == 133: # coerce_s assert isinstance(stack[-1], (type(None), compat_str)) elif opcode == 164: # modulo @@ -987,30 +1046,43 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): initial_function = extract_function(u'decipher') return lambda s: initial_function([s]) - def _decrypt_signature(self, s, video_id, jsplayer_url, age_gate=False): + def _decrypt_signature(self, s, video_id, player_url, age_gate=False): """Turn the encrypted s field into a working signature""" - if jsplayer_url is not None: + if player_url is not None: try: - if jsplayer_url not in self._jsplayer_cache: - self._jsplayer_cache[jsplayer_url] = self._extract_signature_function( - video_id, jsplayer_url + if player_url not in self._player_cache: + func = self._extract_signature_function( + video_id, player_url, len(s) ) - return self._jsplayer_cache[jsplayer_url]([s]) - except Exception as e: + self._player_cache[player_url] = func + func = self._player_cache[player_url] + if self._downloader.params.get('youtube_print_sig_code'): + self._print_sig_code(func, len(s)) + return func(s) + except Exception: tb = traceback.format_exc() - self._downloader.report_warning(u'Automatic signature extraction failed: ' + tb) + self._downloader.report_warning( + u'Automatic signature extraction failed: ' + tb) - self._downloader.report_warning(u'Warning: Falling back to static signature algorithm') + self._downloader.report_warning( + u'Warning: Falling back to static signature algorithm') + return self._static_decrypt_signature( + s, video_id, player_url, age_gate) + def _static_decrypt_signature(self, s, video_id, player_url, age_gate): if age_gate: # The videos with age protection use another player, so the # algorithms can be different. if len(s) == 86: return s[2:63] + s[82] + s[64:82] + s[63] - if len(s) == 92: + if len(s) == 93: + return s[86:29:-1] + s[88] + s[28:5:-1] + elif len(s) == 92: return s[25] + s[3:25] + s[0] + s[26:42] + s[79] + s[43:79] + s[91] + s[80:83] + elif len(s) == 91: + return s[84:27:-1] + s[86] + s[26:5:-1] elif len(s) == 90: return s[25] + s[3:25] + s[2] + s[26:40] + s[77] + s[41:77] + s[89] + s[78:81] elif len(s) == 89: @@ -1020,11 +1092,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): elif len(s) == 87: return s[6:27] + s[4] + s[28:39] + s[27] + s[40:59] + s[2] + s[60:] elif len(s) == 86: - return s[5:34] + s[0] + s[35:38] + s[3] + s[39:45] + s[38] + s[46:53] + s[73] + s[54:73] + s[85] + s[74:85] + s[53] + return s[80:72:-1] + s[16] + s[71:39:-1] + s[72] + s[38:16:-1] + s[82] + s[15::-1] elif len(s) == 85: return s[3:11] + s[0] + s[12:55] + s[84] + s[56:84] elif len(s) == 84: - return s[81:36:-1] + s[0] + s[35:2:-1] + return s[78:70:-1] + s[14] + s[69:37:-1] + s[70] + s[36:14:-1] + s[80] + s[:14][::-1] elif len(s) == 83: return s[81:64:-1] + s[82] + s[63:52:-1] + s[45] + s[51:45:-1] + s[1] + s[44:1:-1] + s[0] elif len(s) == 82: @@ -1039,15 +1111,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): else: raise ExtractorError(u'Unable to decrypt signature, key length %d not supported; retrying might work' % (len(s))) - def _decrypt_signature_age_gate(self, s): - # The videos with age protection use another player, so the algorithms - # can be different. - if len(s) == 86: - return s[2:63] + s[82] + s[64:82] + s[63] - else: - # Fallback to the other algortihms - return self._decrypt_signature(s) - def _get_available_subtitles(self, video_id): try: sub_list = self._download_webpage( @@ -1376,12 +1439,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): encrypted_sig = url_data['s'][0] if self._downloader.params.get('verbose'): if age_gate: - player_version = self._search_regex(r'-(.+)\.swf$', - player_url if player_url else 'NOT FOUND', - 'flash player', fatal=False) + if player_url is None: + player_version = 'unknown' + else: + player_version = self._search_regex( + r'-(.+)\.swf$', player_url, + u'flash player', fatal=False) player_desc = 'flash player %s' % player_version else: - player_version = self._search_regex(r'html5player-(.+?)\.js', video_webpage, + player_version = self._search_regex( + r'html5player-(.+?)\.js', video_webpage, 'html5 player', fatal=False) player_desc = u'html5 player %s' % player_version @@ -1389,15 +1456,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): self.to_screen(u'encrypted signature length %d (%s), itag %s, %s' % (len(encrypted_sig), parts_sizes, url_data['itag'][0], player_desc)) - if age_gate: - jsplayer_url = None - else: + if not age_gate: jsplayer_url_json = self._search_regex( r'"assets":.+?"js":\s*("[^"]+")', video_webpage, u'JS player URL') - jsplayer_url = json.loads(jsplayer_url_json) + player_url = json.loads(jsplayer_url_json) - signature = self._decrypt_signature(encrypted_sig, video_id, jsplayer_url, age_gate) + signature = self._decrypt_signature( + encrypted_sig, video_id, player_url, age_gate) url += '&signature=' + signature if 'ratebypass' not in url: url += '&ratebypass=yes'