X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Futils.py;h=42f0b07ce89bc882fe38319ded77c077238250d2;hb=4340decad25e264863e27538847e17918b4c1ae2;hp=4b0567c938fa8fe8d78175114ee94db69099af15;hpb=e2f89ec7aada2a58ca658a77401762b764adc112;p=youtube-dl diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 4b0567c93..42f0b07ce 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -10,6 +10,7 @@ import ctypes import datetime import email.utils import errno +import functools import gzip import itertools import io @@ -34,7 +35,9 @@ from .compat import ( compat_chr, compat_getenv, compat_html_entities, + compat_http_client, compat_parse_qs, + compat_socket_create_connection, compat_str, compat_urllib_error, compat_urllib_parse, @@ -205,6 +208,10 @@ def get_element_by_attribute(attribute, value, html): def clean_html(html): """Clean an HTML snippet into a readable string""" + + if html is None: # Convenience for sanitizing descriptions etc. + return html + # Newline vs
html = html.replace('\n', ' ') html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html) @@ -363,7 +370,7 @@ def encodeArgument(s): if not isinstance(s, compat_str): # Legacy code that uses byte strings # Uncomment the following line after fixing all post processors - #assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s)) + # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s)) s = s.decode('ascii') return encodeFilename(s, True) @@ -387,7 +394,19 @@ def formatSeconds(secs): return '%d' % secs -def make_HTTPS_handler(opts_no_check_certificate, **kwargs): +def make_HTTPS_handler(params, **kwargs): + opts_no_check_certificate = params.get('nocheckcertificate', False) + if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9 + context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + if opts_no_check_certificate: + context.verify_mode = ssl.CERT_NONE + try: + return YoutubeDLHTTPSHandler(params, context=context, **kwargs) + except TypeError: + # Python 2.7.8 + # (create_default_context present but HTTPSHandler has no context=) + pass + if sys.version_info < (3, 2): import httplib @@ -405,27 +424,14 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs): except ssl.SSLError: self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23) - class HTTPSHandlerV3(compat_urllib_request.HTTPSHandler): - def https_open(self, req): - return self.do_open(HTTPSConnectionV3, req) - return HTTPSHandlerV3(**kwargs) - elif hasattr(ssl, 'create_default_context'): # Python >= 3.4 - context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - context.options &= ~ssl.OP_NO_SSLv3 # Allow older, not-as-secure SSLv3 - if opts_no_check_certificate: - context.verify_mode = ssl.CERT_NONE - return compat_urllib_request.HTTPSHandler(context=context, **kwargs) + return YoutubeDLHTTPSHandler(params, https_conn_class=HTTPSConnectionV3, **kwargs) else: # Python < 3.4 context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.verify_mode = (ssl.CERT_NONE if opts_no_check_certificate else ssl.CERT_REQUIRED) context.set_default_verify_paths() - try: - context.load_default_certs() - except AttributeError: - pass # Python < 3.4 - return compat_urllib_request.HTTPSHandler(context=context, **kwargs) + return YoutubeDLHTTPSHandler(params, context=context, **kwargs) class ExtractorError(Exception): @@ -463,6 +469,13 @@ class ExtractorError(Exception): return ''.join(traceback.format_tb(self.traceback)) +class UnsupportedError(ExtractorError): + def __init__(self, url): + super(UnsupportedError, self).__init__( + 'Unsupported URL: %s' % url, expected=True) + self.url = url + + class RegexNotFoundError(ExtractorError): """Error when a regex didn't match""" pass @@ -532,6 +545,26 @@ class ContentTooShortError(Exception): self.expected = expected +def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs): + hc = http_class(*args, **kwargs) + source_address = ydl_handler._params.get('source_address') + if source_address is not None: + sa = (source_address, 0) + if hasattr(hc, 'source_address'): # Python 2.7+ + hc.source_address = sa + else: # Python 2.6 + def _hc_connect(self, *args, **kwargs): + sock = compat_socket_create_connection( + (self.host, self.port), self.timeout, sa) + if is_https: + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) + else: + self.sock = sock + hc.connect = functools.partial(_hc_connect, hc) + + return hc + + class YoutubeDLHandler(compat_urllib_request.HTTPHandler): """Handler for HTTP requests and responses. @@ -550,6 +583,15 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler): public domain. """ + def __init__(self, params, *args, **kwargs): + compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs) + self._params = params + + def http_open(self, req): + return self.do_open(functools.partial( + _create_http_connection, self, compat_http_client.HTTPConnection, False), + req) + @staticmethod def deflate(data): try: @@ -619,6 +661,18 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler): https_response = http_response +class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler): + def __init__(self, params, https_conn_class=None, *args, **kwargs): + compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs) + self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection + self._params = params + + def https_open(self, req): + return self.do_open(functools.partial( + _create_http_connection, self, self._https_conn_class, True), + req) + + def parse_iso8601(date_str, delimiter='T'): """ Return a UNIX timestamp from the given date """ @@ -667,9 +721,6 @@ def unified_strdate(date_str, day_first=True): '%b %dth %Y %I:%M%p', '%Y-%m-%d', '%Y/%m/%d', - '%d.%m.%Y', - '%d/%m/%Y', - '%d/%m/%y', '%Y/%m/%d %H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f', @@ -684,10 +735,16 @@ def unified_strdate(date_str, day_first=True): ] if day_first: format_expressions.extend([ + '%d.%m.%Y', + '%d/%m/%Y', + '%d/%m/%y', '%d/%m/%Y %H:%M:%S', ]) else: format_expressions.extend([ + '%m.%d.%Y', + '%m/%d/%Y', + '%m/%d/%y', '%m/%d/%Y %H:%M:%S', ]) for expression in format_expressions: @@ -819,22 +876,22 @@ def _windows_write_string(s, out): GetStdHandle = ctypes.WINFUNCTYPE( ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)( - ("GetStdHandle", ctypes.windll.kernel32)) + (b"GetStdHandle", ctypes.windll.kernel32)) h = GetStdHandle(WIN_OUTPUT_IDS[fileno]) WriteConsoleW = ctypes.WINFUNCTYPE( ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR, ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD), - ctypes.wintypes.LPVOID)(("WriteConsoleW", ctypes.windll.kernel32)) + ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32)) written = ctypes.wintypes.DWORD(0) - GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(("GetFileType", ctypes.windll.kernel32)) + GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32)) FILE_TYPE_CHAR = 0x0002 FILE_TYPE_REMOTE = 0x8000 GetConsoleMode = ctypes.WINFUNCTYPE( ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.POINTER(ctypes.wintypes.DWORD))( - ("GetConsoleMode", ctypes.windll.kernel32)) + (b"GetConsoleMode", ctypes.windll.kernel32)) INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value def not_a_console(handle): @@ -1261,18 +1318,25 @@ def check_executable(exe, args=[]): def get_exe_version(exe, args=['--version'], - version_re=r'version\s+([0-9._-a-zA-Z]+)', - unrecognized='present'): + version_re=None, unrecognized='present'): """ Returns the version of the specified executable, or False if the executable is not present """ try: - out, err = subprocess.Popen( + out, _ = subprocess.Popen( [exe] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate() except OSError: return False - firstline = out.partition(b'\n')[0].decode('ascii', 'ignore') - m = re.search(version_re, firstline) + if isinstance(out, bytes): # Python 2.x + out = out.decode('ascii', 'ignore') + return detect_exe_version(out, version_re, unrecognized) + + +def detect_exe_version(output, version_re=None, unrecognized='present'): + assert isinstance(output, compat_str) + if version_re is None: + version_re = r'version\s+([-0-9._a-zA-Z]+)' + m = re.search(version_re, output) if m: return m.group(1) else: @@ -1535,3 +1599,23 @@ def ytdl_is_updateable(): def args_to_str(args): # Get a short string representation for a subprocess command return ' '.join(shlex_quote(a) for a in args) + + +def urlhandle_detect_ext(url_handle): + try: + url_handle.headers + getheader = lambda h: url_handle.headers[h] + except AttributeError: # Python < 3 + getheader = url_handle.info().getheader + + return getheader('Content-Type').split("/")[1] + + +def age_restricted(content_limit, age_limit): + """ Returns True iff the content should be blocked """ + + if age_limit is None: # No limit set + return False + if content_limit is None: + return False # Content available for everyone + return age_limit < content_limit