X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Futils.py;h=b8c52af74768053d27b4642173022af3b2c6723d;hb=d4f64cabf4ede444b390bb71b90ad4103ce572c0;hp=cc5f510f40fff66ce374f2586aae43686a240b7d;hpb=be4a824d74add1a3b78b8244dff12f4f078f168a;p=youtube-dl diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index cc5f510f4..b8c52af74 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -287,6 +287,8 @@ def sanitize_filename(s, restricted=False, is_id=False): return '_' return char + # Handle timestamps + s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s) result = ''.join(map(replace_insane, s)) if not is_id: while '__' in result: @@ -397,8 +399,9 @@ def formatSeconds(secs): def make_HTTPS_handler(params, **kwargs): opts_no_check_certificate = params.get('nocheckcertificate', False) if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9 - context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) if opts_no_check_certificate: + context.check_hostname = False context.verify_mode = ssl.CERT_NONE try: return YoutubeDLHTTPSHandler(params, context=context, **kwargs) @@ -408,25 +411,9 @@ def make_HTTPS_handler(params, **kwargs): pass if sys.version_info < (3, 2): - import httplib - - class HTTPSConnectionV3(httplib.HTTPSConnection): - def __init__(self, *args, **kwargs): - httplib.HTTPSConnection.__init__(self, *args, **kwargs) - - def connect(self): - sock = socket.create_connection((self.host, self.port), self.timeout) - if getattr(self, '_tunnel_host', False): - self.sock = sock - self._tunnel() - try: - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_TLSv1) - except ssl.SSLError: - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23) - - return YoutubeDLHTTPSHandler(params, https_conn_class=HTTPSConnectionV3, **kwargs) + return YoutubeDLHTTPSHandler(params, **kwargs) else: # Python < 3.4 - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) context.verify_mode = (ssl.CERT_NONE if opts_no_check_certificate else ssl.CERT_REQUIRED) @@ -545,7 +532,7 @@ class ContentTooShortError(Exception): self.expected = expected -def _create_http_connection(ydl_handler, http_class, is_https=False, *args, **kwargs): +def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs): hc = http_class(*args, **kwargs) source_address = ydl_handler._params.get('source_address') if source_address is not None: @@ -557,7 +544,9 @@ def _create_http_connection(ydl_handler, http_class, is_https=False, *args, **kw sock = compat_socket_create_connection( (self.host, self.port), self.timeout, sa) if is_https: - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) + self.sock = ssl.wrap_socket( + sock, self.key_file, self.cert_file, + ssl_version=ssl.PROTOCOL_TLSv1) else: self.sock = sock hc.connect = functools.partial(_hc_connect, hc) @@ -589,7 +578,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler): def http_open(self, req): return self.do_open(functools.partial( - _create_http_connection, self, compat_http_client.HTTPConnection), + _create_http_connection, self, compat_http_client.HTTPConnection, False), req) @staticmethod @@ -609,17 +598,14 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler): def http_request(self, req): for h, v in std_headers.items(): - if h not in req.headers: + # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275 + # The dict keys are capitalized because of this bug by urllib + if h.capitalize() not in req.headers: req.add_header(h, v) if 'Youtubedl-no-compression' in req.headers: if 'Accept-encoding' in req.headers: del req.headers['Accept-encoding'] del req.headers['Youtubedl-no-compression'] - if 'Youtubedl-user-agent' in req.headers: - if 'User-agent' in req.headers: - del req.headers['User-agent'] - req.headers['User-agent'] = req.headers['Youtubedl-user-agent'] - del req.headers['Youtubedl-user-agent'] if sys.version_info < (2, 7) and '#' in req.get_full_url(): # Python 2.6 is brain-dead when it comes to fragments @@ -719,6 +705,7 @@ def unified_strdate(date_str, day_first=True): '%b %dst %Y %I:%M%p', '%b %dnd %Y %I:%M%p', '%b %dth %Y %I:%M%p', + '%Y %m %d', '%Y-%m-%d', '%Y/%m/%d', '%Y/%m/%d %H:%M:%S', @@ -871,6 +858,9 @@ def _windows_write_string(s, out): except AttributeError: # If the output stream doesn't have a fileno, it's virtual return False + except io.UnsupportedOperation: + # Some strange Windows pseudo files? + return False if fileno not in WIN_OUTPUT_IDS: return False @@ -1267,13 +1257,13 @@ def float_or_none(v, scale=1, invscale=1, default=None): def parse_duration(s): - if s is None: + if not isinstance(s, basestring if sys.version_info < (3, 0) else compat_str): return None s = s.strip() m = re.match( - r'''(?ix)T? + r'''(?ix)(?:P?T)? (?: (?P[0-9.]+)\s*(?:mins?|minutes?)\s*| (?P[0-9.]+)\s*(?:hours?)| @@ -1608,6 +1598,14 @@ def urlhandle_detect_ext(url_handle): except AttributeError: # Python < 3 getheader = url_handle.info().getheader + cd = getheader('Content-Disposition') + if cd: + m = re.match(r'attachment;\s*filename="(?P[^"]+)"', cd) + if m: + e = determine_ext(m.group('filename'), default_ext=None) + if e: + return e + return getheader('Content-Type').split("/")[1] @@ -1619,3 +1617,53 @@ def age_restricted(content_limit, age_limit): if content_limit is None: return False # Content available for everyone return age_limit < content_limit + + +def is_html(first_bytes): + """ Detect whether a file contains HTML by examining its first bytes. """ + + BOMS = [ + (b'\xef\xbb\xbf', 'utf-8'), + (b'\x00\x00\xfe\xff', 'utf-32-be'), + (b'\xff\xfe\x00\x00', 'utf-32-le'), + (b'\xff\xfe', 'utf-16-le'), + (b'\xfe\xff', 'utf-16-be'), + ] + for bom, enc in BOMS: + if first_bytes.startswith(bom): + s = first_bytes[len(bom):].decode(enc, 'replace') + break + else: + s = first_bytes.decode('utf-8', 'replace') + + return re.match(r'^\s*<', s) + + +def determine_protocol(info_dict): + protocol = info_dict.get('protocol') + if protocol is not None: + return protocol + + url = info_dict['url'] + if url.startswith('rtmp'): + return 'rtmp' + elif url.startswith('mms'): + return 'mms' + elif url.startswith('rtsp'): + return 'rtsp' + + ext = determine_ext(url) + if ext == 'm3u8': + return 'm3u8' + elif ext == 'f4m': + return 'f4m' + + return compat_urllib_parse_urlparse(url).scheme + + +def render_table(header_row, data): + """ Render a list of rows, each as a list of values """ + table = [header_row] + data + max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)] + format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s' + return '\n'.join(format_str % tuple(row) for row in table)