X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fcompat.py;h=ace5bd716aac0161b356625260f1138687568314;hb=d5d7bdaeb517f389fff5a6557f072f3586e3c440;hp=554e3d5db899d1b5d7aff78e08f49286e279ccfd;hpb=851229a01f34129286a57d46f8a27b9bb5fd9a6b;p=youtube-dl diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 554e3d5db..ace5bd716 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -42,6 +42,11 @@ try: except ImportError: # Python 2 import cookielib as compat_cookiejar +try: + import http.cookies as compat_cookies +except ImportError: # Python 2 + import Cookie as compat_cookies + try: import html.entities as compat_html_entities except ImportError: # Python 2 @@ -75,8 +80,17 @@ except ImportError: import BaseHTTPServer as compat_http_server try: + from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes from urllib.parse import unquote as compat_urllib_parse_unquote -except ImportError: + from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus +except ImportError: # Python 2 + _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire') + else re.compile('([\x00-\x7f]+)')) + + # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus + # implementations from cpython 3.4.3's stdlib. Python 2's version + # is apparently broken (see https://github.com/rg3/youtube-dl/pull/6244) + def compat_urllib_parse_unquote_to_bytes(string): """unquote_to_bytes('abc%20def') -> b'abc def'.""" # Note: strings are encoded as UTF-8. This is only an issue if it contains @@ -85,30 +99,23 @@ except ImportError: # Is it a string-like object? string.split return b'' - if isinstance(string, str): + if isinstance(string, unicode): string = string.encode('utf-8') - # string = encode('utf-8') - - # python3 -> 2: must implicitly convert to bits - bits = bytes(string).split(b'%') - + bits = string.split(b'%') if len(bits) == 1: return string res = [bits[0]] append = res.append - for item in bits[1:]: try: - append(item[:2].decode('hex')) + append(compat_urllib_parse._hextochr[item[:2]]) append(item[2:]) - except: + except KeyError: append(b'%') append(item) return b''.join(res) - compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)') - - def new_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): + def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): """Replace %xx escapes by their single-character equivalent. The optional encoding and errors parameters specify how to decode percent-encoded sequences into Unicode characters, as accepted by the bytes.decode() @@ -118,7 +125,6 @@ except ImportError: unquote('abc%20def') -> 'abc def'. """ - if '%' not in string: string.split return string @@ -126,60 +132,22 @@ except ImportError: encoding = 'utf-8' if errors is None: errors = 'replace' - - bits = compat_urllib_parse_asciire.split(string) + bits = _asciire.split(string) res = [bits[0]] append = res.append for i in range(1, len(bits), 2): - foo = compat_urllib_parse_unquote_to_bytes(bits[i]) - foo = foo.decode(encoding, errors) - append(foo) - - if bits[i + 1]: - bar = bits[i + 1] - if not isinstance(bar, unicode): - bar = bar.decode('utf-8') - append(bar) + append(compat_urllib_parse_unquote_to_bytes(bits[i]).decode(encoding, errors)) + append(bits[i + 1]) return ''.join(res) - def old_compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): - if string == '': - return string - res = string.split('%') - if len(res) == 1: - return string - if encoding is None: - encoding = 'utf-8' - if errors is None: - errors = 'replace' - # pct_sequence: contiguous sequence of percent-encoded bytes, decoded - pct_sequence = b'' - string = res[0] - for item in res[1:]: - try: - if not item: - raise ValueError - if not re.match('[0-9a-fA-F][0-9a-fA-F]',item[:2]): - raise ValueError - pct_sequence += item[:2].decode('hex') - rest = item[2:] - if not rest: - # This segment was just a single percent-encoded character. - # May be part of a sequence of code units, so delay decoding. - # (Stored in pct_sequence). - continue - except ValueError: - rest = '%' + item - # Encountered non-percent-encoded characters. Flush the current - # pct_sequence. - string += pct_sequence.decode(encoding, errors) + rest - pct_sequence = b'' - if pct_sequence: - # Flush the final pct_sequence - string += pct_sequence.decode(encoding, errors) - return string - - compat_urllib_parse_unquote = new_compat_urllib_parse_unquote + def compat_urllib_parse_unquote_plus(string, encoding='utf-8', errors='replace'): + """Like unquote(), but also replace plus signs by spaces, as required for + unquoting HTML form values. + + unquote_plus('%7e/abc+def') -> '~/abc def' + """ + string = string.replace('+', ' ') + return compat_urllib_parse_unquote(string, encoding, errors) try: compat_str = unicode # Python 2 @@ -468,11 +436,17 @@ except TypeError: # Python 2.6 yield n n += step +if sys.version_info >= (3, 0): + from tokenize import tokenize as compat_tokenize_tokenize +else: + from tokenize import generate_tokens as compat_tokenize_tokenize + __all__ = [ 'compat_HTTPError', 'compat_basestring', 'compat_chr', 'compat_cookiejar', + 'compat_cookies', 'compat_expanduser', 'compat_get_terminal_size', 'compat_getenv', @@ -488,9 +462,12 @@ __all__ = [ 'compat_socket_create_connection', 'compat_str', 'compat_subprocess_get_DEVNULL', + 'compat_tokenize_tokenize', 'compat_urllib_error', 'compat_urllib_parse', 'compat_urllib_parse_unquote', + 'compat_urllib_parse_unquote_plus', + 'compat_urllib_parse_unquote_to_bytes', 'compat_urllib_parse_urlparse', 'compat_urllib_request', 'compat_urlparse',