2 # -*- coding: utf-8 -*-
4 from __future__ import unicode_literals
33 import xml.etree.ElementTree
40 compat_etree_fromstring,
42 compat_html_entities_html5,
47 compat_socket_create_connection,
52 compat_urllib_parse_urlencode,
53 compat_urllib_parse_urlparse,
54 compat_urllib_parse_unquote_plus,
55 compat_urllib_request,
66 def register_socks_protocols():
67 # "Register" SOCKS protocols
68 # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
69 # URLs with protocols not in urlparse.uses_netloc are not handled correctly
70 for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
71 if scheme not in compat_urlparse.uses_netloc:
72 compat_urlparse.uses_netloc.append(scheme)
75 # This is not clearly defined otherwise
76 compiled_regex_type = type(re.compile(''))
79 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/47.0 (Chrome)',
80 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
81 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
82 'Accept-Encoding': 'gzip, deflate',
83 'Accept-Language': 'en-us,en;q=0.5',
89 ENGLISH_MONTH_NAMES = [
90 'January', 'February', 'March', 'April', 'May', 'June',
91 'July', 'August', 'September', 'October', 'November', 'December']
94 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
95 'flv', 'f4v', 'f4a', 'f4b',
96 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
106 'f4f', 'f4m', 'm3u8', 'smil')
108 # needed for sanitizing filenames in restricted mode
109 ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
110 itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUYP', ['ss'],
111 'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuypy')))
126 '%Y-%m-%d %H:%M:%S.%f',
129 '%Y-%m-%dT%H:%M:%SZ',
130 '%Y-%m-%dT%H:%M:%S.%fZ',
131 '%Y-%m-%dT%H:%M:%S.%f0Z',
133 '%Y-%m-%dT%H:%M:%S.%f',
137 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
138 DATE_FORMATS_DAY_FIRST.extend([
147 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
148 DATE_FORMATS_MONTH_FIRST.extend([
157 def preferredencoding():
158 """Get preferred encoding.
160 Returns the best encoding scheme for the system, based on
161 locale.getpreferredencoding() and some further tweaks.
164 pref = locale.getpreferredencoding()
172 def write_json_file(obj, fn):
173 """ Encode obj as JSON and write it to fn, atomically if possible """
175 fn = encodeFilename(fn)
176 if sys.version_info < (3, 0) and sys.platform != 'win32':
177 encoding = get_filesystem_encoding()
178 # os.path.basename returns a bytes object, but NamedTemporaryFile
179 # will fail if the filename contains non ascii characters unless we
180 # use a unicode object
181 path_basename = lambda f: os.path.basename(fn).decode(encoding)
182 # the same for os.path.dirname
183 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
185 path_basename = os.path.basename
186 path_dirname = os.path.dirname
190 'prefix': path_basename(fn) + '.',
191 'dir': path_dirname(fn),
195 # In Python 2.x, json.dump expects a bytestream.
196 # In Python 3.x, it writes to a character stream
197 if sys.version_info < (3, 0):
205 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
210 if sys.platform == 'win32':
211 # Need to remove existing file on Windows, else os.rename raises
212 # WindowsError or FileExistsError.
217 os.rename(tf.name, fn)
226 if sys.version_info >= (2, 7):
227 def find_xpath_attr(node, xpath, key, val=None):
228 """ Find the xpath xpath[@key=val] """
229 assert re.match(r'^[a-zA-Z_-]+$', key)
230 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
231 return node.find(expr)
233 def find_xpath_attr(node, xpath, key, val=None):
234 for f in node.findall(compat_xpath(xpath)):
235 if key not in f.attrib:
237 if val is None or f.attrib.get(key) == val:
241 # On python2.6 the xml.etree.ElementTree.Element methods don't support
242 # the namespace parameter
245 def xpath_with_ns(path, ns_map):
246 components = [c.split(':') for c in path.split('/')]
250 replaced.append(c[0])
253 replaced.append('{%s}%s' % (ns_map[ns], tag))
254 return '/'.join(replaced)
257 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
258 def _find_xpath(xpath):
259 return node.find(compat_xpath(xpath))
261 if isinstance(xpath, (str, compat_str)):
262 n = _find_xpath(xpath)
270 if default is not NO_DEFAULT:
273 name = xpath if name is None else name
274 raise ExtractorError('Could not find XML element %s' % name)
280 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
281 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
282 if n is None or n == default:
285 if default is not NO_DEFAULT:
288 name = xpath if name is None else name
289 raise ExtractorError('Could not find XML element\'s text %s' % name)
295 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
296 n = find_xpath_attr(node, xpath, key)
298 if default is not NO_DEFAULT:
301 name = '%s[@%s]' % (xpath, key) if name is None else name
302 raise ExtractorError('Could not find XML attribute %s' % name)
308 def get_element_by_id(id, html):
309 """Return the content of the tag with the specified ID in the passed HTML document"""
310 return get_element_by_attribute('id', id, html)
313 def get_element_by_attribute(attribute, value, html):
314 """Return the content of the tag with the specified attribute in the passed HTML document"""
316 m = re.search(r'''(?xs)
318 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
320 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
324 ''' % (re.escape(attribute), re.escape(value)), html)
328 res = m.group('content')
330 if res.startswith('"') or res.startswith("'"):
333 return unescapeHTML(res)
336 class HTMLAttributeParser(compat_HTMLParser):
337 """Trivial HTML parser to gather the attributes for a single element"""
340 compat_HTMLParser.__init__(self)
342 def handle_starttag(self, tag, attrs):
343 self.attrs = dict(attrs)
346 def extract_attributes(html_element):
347 """Given a string for an HTML element such as
349 a="foo" B="bar" c="&98;az" d=boz
350 empty= noval entity="&"
353 Decode and return a dictionary of attributes.
355 'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
356 'empty': '', 'noval': None, 'entity': '&',
357 'sq': '"', 'dq': '\''
359 NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
360 but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
362 parser = HTMLAttributeParser()
363 parser.feed(html_element)
368 def clean_html(html):
369 """Clean an HTML snippet into a readable string"""
371 if html is None: # Convenience for sanitizing descriptions etc.
375 html = html.replace('\n', ' ')
376 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
377 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
379 html = re.sub('<.*?>', '', html)
380 # Replace html entities
381 html = unescapeHTML(html)
385 def sanitize_open(filename, open_mode):
386 """Try to open the given filename, and slightly tweak it if this fails.
388 Attempts to open the given filename. If this fails, it tries to change
389 the filename slightly, step by step, until it's either able to open it
390 or it fails and raises a final exception, like the standard open()
393 It returns the tuple (stream, definitive_file_name).
397 if sys.platform == 'win32':
399 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
400 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
401 stream = open(encodeFilename(filename), open_mode)
402 return (stream, filename)
403 except (IOError, OSError) as err:
404 if err.errno in (errno.EACCES,):
407 # In case of error, try to remove win32 forbidden chars
408 alt_filename = sanitize_path(filename)
409 if alt_filename == filename:
412 # An exception here should be caught in the caller
413 stream = open(encodeFilename(alt_filename), open_mode)
414 return (stream, alt_filename)
417 def timeconvert(timestr):
418 """Convert RFC 2822 defined time string into system timestamp"""
420 timetuple = email.utils.parsedate_tz(timestr)
421 if timetuple is not None:
422 timestamp = email.utils.mktime_tz(timetuple)
426 def sanitize_filename(s, restricted=False, is_id=False):
427 """Sanitizes a string so it could be used as part of a filename.
428 If restricted is set, use a stricter subset of allowed characters.
429 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
431 def replace_insane(char):
432 if restricted and char in ACCENT_CHARS:
433 return ACCENT_CHARS[char]
434 if char == '?' or ord(char) < 32 or ord(char) == 127:
437 return '' if restricted else '\''
439 return '_-' if restricted else ' -'
440 elif char in '\\/|*<>':
442 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
444 if restricted and ord(char) > 127:
449 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
450 result = ''.join(map(replace_insane, s))
452 while '__' in result:
453 result = result.replace('__', '_')
454 result = result.strip('_')
455 # Common case of "Foreign band name - English song title"
456 if restricted and result.startswith('-_'):
458 if result.startswith('-'):
459 result = '_' + result[len('-'):]
460 result = result.lstrip('.')
466 def sanitize_path(s):
467 """Sanitizes and normalizes path on Windows"""
468 if sys.platform != 'win32':
470 drive_or_unc, _ = os.path.splitdrive(s)
471 if sys.version_info < (2, 7) and not drive_or_unc:
472 drive_or_unc, _ = os.path.splitunc(s)
473 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
477 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part)
478 for path_part in norm_path]
480 sanitized_path.insert(0, drive_or_unc + os.path.sep)
481 return os.path.join(*sanitized_path)
484 # Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
485 # unwanted failures due to missing protocol
486 def sanitize_url(url):
487 return 'http:%s' % url if url.startswith('//') else url
490 def sanitized_Request(url, *args, **kwargs):
491 return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)
494 def orderedSet(iterable):
495 """ Remove all duplicates from the input iterable """
503 def _htmlentity_transform(entity_with_semicolon):
504 """Transforms an HTML entity to a character."""
505 entity = entity_with_semicolon[:-1]
507 # Known non-numeric HTML entity
508 if entity in compat_html_entities.name2codepoint:
509 return compat_chr(compat_html_entities.name2codepoint[entity])
511 # TODO: HTML5 allows entities without a semicolon. For example,
512 # 'Éric' should be decoded as 'Éric'.
513 if entity_with_semicolon in compat_html_entities_html5:
514 return compat_html_entities_html5[entity_with_semicolon]
516 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
518 numstr = mobj.group(1)
519 if numstr.startswith('x'):
521 numstr = '0%s' % numstr
524 # See https://github.com/rg3/youtube-dl/issues/7518
526 return compat_chr(int(numstr, base))
530 # Unknown entity in name, return its literal representation
531 return '&%s;' % entity
537 assert type(s) == compat_str
540 r'&([^;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
543 def get_subprocess_encoding():
544 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
545 # For subprocess calls, encode with locale encoding
546 # Refer to http://stackoverflow.com/a/9951851/35070
547 encoding = preferredencoding()
549 encoding = sys.getfilesystemencoding()
555 def encodeFilename(s, for_subprocess=False):
557 @param s The name of the file
560 assert type(s) == compat_str
562 # Python 3 has a Unicode API
563 if sys.version_info >= (3, 0):
566 # Pass '' directly to use Unicode APIs on Windows 2000 and up
567 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
568 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
569 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
572 # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
573 if sys.platform.startswith('java'):
576 return s.encode(get_subprocess_encoding(), 'ignore')
579 def decodeFilename(b, for_subprocess=False):
581 if sys.version_info >= (3, 0):
584 if not isinstance(b, bytes):
587 return b.decode(get_subprocess_encoding(), 'ignore')
590 def encodeArgument(s):
591 if not isinstance(s, compat_str):
592 # Legacy code that uses byte strings
593 # Uncomment the following line after fixing all post processors
594 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
595 s = s.decode('ascii')
596 return encodeFilename(s, True)
599 def decodeArgument(b):
600 return decodeFilename(b, True)
603 def decodeOption(optval):
606 if isinstance(optval, bytes):
607 optval = optval.decode(preferredencoding())
609 assert isinstance(optval, compat_str)
613 def formatSeconds(secs):
615 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
617 return '%d:%02d' % (secs // 60, secs % 60)
622 def make_HTTPS_handler(params, **kwargs):
623 opts_no_check_certificate = params.get('nocheckcertificate', False)
624 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
625 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
626 if opts_no_check_certificate:
627 context.check_hostname = False
628 context.verify_mode = ssl.CERT_NONE
630 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
633 # (create_default_context present but HTTPSHandler has no context=)
636 if sys.version_info < (3, 2):
637 return YoutubeDLHTTPSHandler(params, **kwargs)
639 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
640 context.verify_mode = (ssl.CERT_NONE
641 if opts_no_check_certificate
642 else ssl.CERT_REQUIRED)
643 context.set_default_verify_paths()
644 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
647 def bug_reports_message():
648 if ytdl_is_updateable():
649 update_cmd = 'type youtube-dl -U to update'
651 update_cmd = 'see https://yt-dl.org/update on how to update'
652 msg = '; please report this issue on https://yt-dl.org/bug .'
653 msg += ' Make sure you are using the latest version; %s.' % update_cmd
654 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
658 class ExtractorError(Exception):
659 """Error during info extraction."""
661 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
662 """ tb, if given, is the original traceback (so that it can be printed out).
663 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
666 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
668 if video_id is not None:
669 msg = video_id + ': ' + msg
671 msg += ' (caused by %r)' % cause
673 msg += bug_reports_message()
674 super(ExtractorError, self).__init__(msg)
677 self.exc_info = sys.exc_info() # preserve original exception
679 self.video_id = video_id
681 def format_traceback(self):
682 if self.traceback is None:
684 return ''.join(traceback.format_tb(self.traceback))
687 class UnsupportedError(ExtractorError):
688 def __init__(self, url):
689 super(UnsupportedError, self).__init__(
690 'Unsupported URL: %s' % url, expected=True)
694 class RegexNotFoundError(ExtractorError):
695 """Error when a regex didn't match"""
699 class DownloadError(Exception):
700 """Download Error exception.
702 This exception may be thrown by FileDownloader objects if they are not
703 configured to continue on errors. They will contain the appropriate
707 def __init__(self, msg, exc_info=None):
708 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
709 super(DownloadError, self).__init__(msg)
710 self.exc_info = exc_info
713 class SameFileError(Exception):
714 """Same File exception.
716 This exception will be thrown by FileDownloader objects if they detect
717 multiple files would have to be downloaded to the same file on disk.
722 class PostProcessingError(Exception):
723 """Post Processing exception.
725 This exception may be raised by PostProcessor's .run() method to
726 indicate an error in the postprocessing task.
729 def __init__(self, msg):
733 class MaxDownloadsReached(Exception):
734 """ --max-downloads limit has been reached. """
738 class UnavailableVideoError(Exception):
739 """Unavailable Format exception.
741 This exception will be thrown when a video is requested
742 in a format that is not available for that video.
747 class ContentTooShortError(Exception):
748 """Content Too Short exception.
750 This exception may be raised by FileDownloader objects when a file they
751 download is too small for what the server announced first, indicating
752 the connection was probably interrupted.
755 def __init__(self, downloaded, expected):
757 self.downloaded = downloaded
758 self.expected = expected
761 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
762 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
763 # expected HTTP responses to meet HTTP/1.0 or later (see also
764 # https://github.com/rg3/youtube-dl/issues/6727)
765 if sys.version_info < (3, 0):
766 kwargs[b'strict'] = True
767 hc = http_class(*args, **kwargs)
768 source_address = ydl_handler._params.get('source_address')
769 if source_address is not None:
770 sa = (source_address, 0)
771 if hasattr(hc, 'source_address'): # Python 2.7+
772 hc.source_address = sa
774 def _hc_connect(self, *args, **kwargs):
775 sock = compat_socket_create_connection(
776 (self.host, self.port), self.timeout, sa)
778 self.sock = ssl.wrap_socket(
779 sock, self.key_file, self.cert_file,
780 ssl_version=ssl.PROTOCOL_TLSv1)
783 hc.connect = functools.partial(_hc_connect, hc)
788 def handle_youtubedl_headers(headers):
789 filtered_headers = headers
791 if 'Youtubedl-no-compression' in filtered_headers:
792 filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
793 del filtered_headers['Youtubedl-no-compression']
795 return filtered_headers
798 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
799 """Handler for HTTP requests and responses.
801 This class, when installed with an OpenerDirector, automatically adds
802 the standard headers to every HTTP request and handles gzipped and
803 deflated responses from web servers. If compression is to be avoided in
804 a particular request, the original request in the program code only has
805 to include the HTTP header "Youtubedl-no-compression", which will be
806 removed before making the real request.
808 Part of this code was copied from:
810 http://techknack.net/python-urllib2-handlers/
812 Andrew Rowls, the author of that code, agreed to release it to the
816 def __init__(self, params, *args, **kwargs):
817 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
818 self._params = params
820 def http_open(self, req):
821 conn_class = compat_http_client.HTTPConnection
823 socks_proxy = req.headers.get('Ytdl-socks-proxy')
825 conn_class = make_socks_conn_class(conn_class, socks_proxy)
826 del req.headers['Ytdl-socks-proxy']
828 return self.do_open(functools.partial(
829 _create_http_connection, self, conn_class, False),
835 return zlib.decompress(data, -zlib.MAX_WBITS)
837 return zlib.decompress(data)
840 def addinfourl_wrapper(stream, headers, url, code):
841 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
842 return compat_urllib_request.addinfourl(stream, headers, url, code)
843 ret = compat_urllib_request.addinfourl(stream, headers, url)
847 def http_request(self, req):
848 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
849 # always respected by websites, some tend to give out URLs with non percent-encoded
850 # non-ASCII characters (see telemb.py, ard.py [#3412])
851 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
852 # To work around aforementioned issue we will replace request's original URL with
853 # percent-encoded one
854 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
855 # the code of this workaround has been moved here from YoutubeDL.urlopen()
856 url = req.get_full_url()
857 url_escaped = escape_url(url)
859 # Substitute URL if any change after escaping
860 if url != url_escaped:
861 req = update_Request(req, url=url_escaped)
863 for h, v in std_headers.items():
864 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
865 # The dict keys are capitalized because of this bug by urllib
866 if h.capitalize() not in req.headers:
869 req.headers = handle_youtubedl_headers(req.headers)
871 if sys.version_info < (2, 7) and '#' in req.get_full_url():
872 # Python 2.6 is brain-dead when it comes to fragments
873 req._Request__original = req._Request__original.partition('#')[0]
874 req._Request__r_type = req._Request__r_type.partition('#')[0]
878 def http_response(self, req, resp):
881 if resp.headers.get('Content-encoding', '') == 'gzip':
882 content = resp.read()
883 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
885 uncompressed = io.BytesIO(gz.read())
886 except IOError as original_ioerror:
887 # There may be junk add the end of the file
888 # See http://stackoverflow.com/q/4928560/35070 for details
889 for i in range(1, 1024):
891 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
892 uncompressed = io.BytesIO(gz.read())
897 raise original_ioerror
898 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
899 resp.msg = old_resp.msg
900 del resp.headers['Content-encoding']
902 if resp.headers.get('Content-encoding', '') == 'deflate':
903 gz = io.BytesIO(self.deflate(resp.read()))
904 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
905 resp.msg = old_resp.msg
906 del resp.headers['Content-encoding']
907 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
908 # https://github.com/rg3/youtube-dl/issues/6457).
909 if 300 <= resp.code < 400:
910 location = resp.headers.get('Location')
912 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
913 if sys.version_info >= (3, 0):
914 location = location.encode('iso-8859-1').decode('utf-8')
916 location = location.decode('utf-8')
917 location_escaped = escape_url(location)
918 if location != location_escaped:
919 del resp.headers['Location']
920 if sys.version_info < (3, 0):
921 location_escaped = location_escaped.encode('utf-8')
922 resp.headers['Location'] = location_escaped
925 https_request = http_request
926 https_response = http_response
929 def make_socks_conn_class(base_class, socks_proxy):
930 assert issubclass(base_class, (
931 compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
933 url_components = compat_urlparse.urlparse(socks_proxy)
934 if url_components.scheme.lower() == 'socks5':
935 socks_type = ProxyType.SOCKS5
936 elif url_components.scheme.lower() in ('socks', 'socks4'):
937 socks_type = ProxyType.SOCKS4
938 elif url_components.scheme.lower() == 'socks4a':
939 socks_type = ProxyType.SOCKS4A
941 def unquote_if_non_empty(s):
944 return compat_urllib_parse_unquote_plus(s)
948 url_components.hostname, url_components.port or 1080,
950 unquote_if_non_empty(url_components.username),
951 unquote_if_non_empty(url_components.password),
954 class SocksConnection(base_class):
956 self.sock = sockssocket()
957 self.sock.setproxy(*proxy_args)
958 if type(self.timeout) in (int, float):
959 self.sock.settimeout(self.timeout)
960 self.sock.connect((self.host, self.port))
962 if isinstance(self, compat_http_client.HTTPSConnection):
963 if hasattr(self, '_context'): # Python > 2.6
964 self.sock = self._context.wrap_socket(
965 self.sock, server_hostname=self.host)
967 self.sock = ssl.wrap_socket(self.sock)
969 return SocksConnection
972 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
973 def __init__(self, params, https_conn_class=None, *args, **kwargs):
974 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
975 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
976 self._params = params
978 def https_open(self, req):
980 conn_class = self._https_conn_class
982 if hasattr(self, '_context'): # python > 2.6
983 kwargs['context'] = self._context
984 if hasattr(self, '_check_hostname'): # python 3.x
985 kwargs['check_hostname'] = self._check_hostname
987 socks_proxy = req.headers.get('Ytdl-socks-proxy')
989 conn_class = make_socks_conn_class(conn_class, socks_proxy)
990 del req.headers['Ytdl-socks-proxy']
992 return self.do_open(functools.partial(
993 _create_http_connection, self, conn_class, True),
997 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
998 def __init__(self, cookiejar=None):
999 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
1001 def http_response(self, request, response):
1002 # Python 2 will choke on next HTTP request in row if there are non-ASCII
1003 # characters in Set-Cookie HTTP header of last response (see
1004 # https://github.com/rg3/youtube-dl/issues/6769).
1005 # In order to at least prevent crashing we will percent encode Set-Cookie
1006 # header before HTTPCookieProcessor starts processing it.
1007 # if sys.version_info < (3, 0) and response.headers:
1008 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
1009 # set_cookie = response.headers.get(set_cookie_header)
1011 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
1012 # if set_cookie != set_cookie_escaped:
1013 # del response.headers[set_cookie_header]
1014 # response.headers[set_cookie_header] = set_cookie_escaped
1015 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
1017 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
1018 https_response = http_response
1021 def extract_timezone(date_str):
1023 r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
1026 timezone = datetime.timedelta()
1028 date_str = date_str[:-len(m.group('tz'))]
1029 if not m.group('sign'):
1030 timezone = datetime.timedelta()
1032 sign = 1 if m.group('sign') == '+' else -1
1033 timezone = datetime.timedelta(
1034 hours=sign * int(m.group('hours')),
1035 minutes=sign * int(m.group('minutes')))
1036 return timezone, date_str
1039 def parse_iso8601(date_str, delimiter='T', timezone=None):
1040 """ Return a UNIX timestamp from the given date """
1042 if date_str is None:
1045 date_str = re.sub(r'\.[0-9]+', '', date_str)
1047 if timezone is None:
1048 timezone, date_str = extract_timezone(date_str)
1051 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
1052 dt = datetime.datetime.strptime(date_str, date_format) - timezone
1053 return calendar.timegm(dt.timetuple())
1058 def date_formats(day_first=True):
1059 return DATE_FORMATS_DAY_FIRST if day_first else DATE_FORMATS_MONTH_FIRST
1062 def unified_strdate(date_str, day_first=True):
1063 """Return a string with the date in the format YYYYMMDD"""
1065 if date_str is None:
1069 date_str = date_str.replace(',', ' ')
1070 # Remove AM/PM + timezone
1071 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
1072 _, date_str = extract_timezone(date_str)
1074 for expression in date_formats(day_first):
1076 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
1079 if upload_date is None:
1080 timetuple = email.utils.parsedate_tz(date_str)
1083 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
1086 if upload_date is not None:
1087 return compat_str(upload_date)
1090 def unified_timestamp(date_str, day_first=True):
1091 if date_str is None:
1094 date_str = date_str.replace(',', ' ')
1096 pm_delta = datetime.timedelta(hours=12 if re.search(r'(?i)PM', date_str) else 0)
1097 timezone, date_str = extract_timezone(date_str)
1099 # Remove AM/PM + timezone
1100 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
1102 for expression in date_formats(day_first):
1104 dt = datetime.datetime.strptime(date_str, expression) - timezone + pm_delta
1105 return calendar.timegm(dt.timetuple())
1108 timetuple = email.utils.parsedate_tz(date_str)
1110 return calendar.timegm(timetuple.timetuple())
1113 def determine_ext(url, default_ext='unknown_video'):
1116 guess = url.partition('?')[0].rpartition('.')[2]
1117 if re.match(r'^[A-Za-z0-9]+$', guess):
1119 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
1120 elif guess.rstrip('/') in KNOWN_EXTENSIONS:
1121 return guess.rstrip('/')
1126 def subtitles_filename(filename, sub_lang, sub_format):
1127 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
1130 def date_from_str(date_str):
1132 Return a datetime object from a string in the format YYYYMMDD or
1133 (now|today)[+-][0-9](day|week|month|year)(s)?"""
1134 today = datetime.date.today()
1135 if date_str in ('now', 'today'):
1137 if date_str == 'yesterday':
1138 return today - datetime.timedelta(days=1)
1139 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
1140 if match is not None:
1141 sign = match.group('sign')
1142 time = int(match.group('time'))
1145 unit = match.group('unit')
1146 # A bad approximation?
1150 elif unit == 'year':
1154 delta = datetime.timedelta(**{unit: time})
1155 return today + delta
1156 return datetime.datetime.strptime(date_str, '%Y%m%d').date()
1159 def hyphenate_date(date_str):
1161 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
1162 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
1163 if match is not None:
1164 return '-'.join(match.groups())
1169 class DateRange(object):
1170 """Represents a time interval between two dates"""
1172 def __init__(self, start=None, end=None):
1173 """start and end must be strings in the format accepted by date"""
1174 if start is not None:
1175 self.start = date_from_str(start)
1177 self.start = datetime.datetime.min.date()
1179 self.end = date_from_str(end)
1181 self.end = datetime.datetime.max.date()
1182 if self.start > self.end:
1183 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
1187 """Returns a range that only contains the given day"""
1188 return cls(day, day)
1190 def __contains__(self, date):
1191 """Check if the date is in the range"""
1192 if not isinstance(date, datetime.date):
1193 date = date_from_str(date)
1194 return self.start <= date <= self.end
1197 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
1200 def platform_name():
1201 """ Returns the platform name as a compat_str """
1202 res = platform.platform()
1203 if isinstance(res, bytes):
1204 res = res.decode(preferredencoding())
1206 assert isinstance(res, compat_str)
1210 def _windows_write_string(s, out):
1211 """ Returns True if the string was written using special methods,
1212 False if it has yet to be written out."""
1213 # Adapted from http://stackoverflow.com/a/3259271/35070
1216 import ctypes.wintypes
1224 fileno = out.fileno()
1225 except AttributeError:
1226 # If the output stream doesn't have a fileno, it's virtual
1228 except io.UnsupportedOperation:
1229 # Some strange Windows pseudo files?
1231 if fileno not in WIN_OUTPUT_IDS:
1234 GetStdHandle = ctypes.WINFUNCTYPE(
1235 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1236 (b'GetStdHandle', ctypes.windll.kernel32))
1237 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1239 WriteConsoleW = ctypes.WINFUNCTYPE(
1240 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1241 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1242 ctypes.wintypes.LPVOID)((b'WriteConsoleW', ctypes.windll.kernel32))
1243 written = ctypes.wintypes.DWORD(0)
1245 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b'GetFileType', ctypes.windll.kernel32))
1246 FILE_TYPE_CHAR = 0x0002
1247 FILE_TYPE_REMOTE = 0x8000
1248 GetConsoleMode = ctypes.WINFUNCTYPE(
1249 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1250 ctypes.POINTER(ctypes.wintypes.DWORD))(
1251 (b'GetConsoleMode', ctypes.windll.kernel32))
1252 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1254 def not_a_console(handle):
1255 if handle == INVALID_HANDLE_VALUE or handle is None:
1257 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1258 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1260 if not_a_console(h):
1263 def next_nonbmp_pos(s):
1265 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1266 except StopIteration:
1270 count = min(next_nonbmp_pos(s), 1024)
1272 ret = WriteConsoleW(
1273 h, s, count if count else 2, ctypes.byref(written), None)
1275 raise OSError('Failed to write string')
1276 if not count: # We just wrote a non-BMP character
1277 assert written.value == 2
1280 assert written.value > 0
1281 s = s[written.value:]
1285 def write_string(s, out=None, encoding=None):
1288 assert type(s) == compat_str
1290 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1291 if _windows_write_string(s, out):
1294 if ('b' in getattr(out, 'mode', '') or
1295 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1296 byt = s.encode(encoding or preferredencoding(), 'ignore')
1298 elif hasattr(out, 'buffer'):
1299 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1300 byt = s.encode(enc, 'ignore')
1301 out.buffer.write(byt)
1307 def bytes_to_intlist(bs):
1310 if isinstance(bs[0], int): # Python 3
1313 return [ord(c) for c in bs]
1316 def intlist_to_bytes(xs):
1319 return compat_struct_pack('%dB' % len(xs), *xs)
1322 # Cross-platform file locking
1323 if sys.platform == 'win32':
1324 import ctypes.wintypes
1327 class OVERLAPPED(ctypes.Structure):
1329 ('Internal', ctypes.wintypes.LPVOID),
1330 ('InternalHigh', ctypes.wintypes.LPVOID),
1331 ('Offset', ctypes.wintypes.DWORD),
1332 ('OffsetHigh', ctypes.wintypes.DWORD),
1333 ('hEvent', ctypes.wintypes.HANDLE),
1336 kernel32 = ctypes.windll.kernel32
1337 LockFileEx = kernel32.LockFileEx
1338 LockFileEx.argtypes = [
1339 ctypes.wintypes.HANDLE, # hFile
1340 ctypes.wintypes.DWORD, # dwFlags
1341 ctypes.wintypes.DWORD, # dwReserved
1342 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1343 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1344 ctypes.POINTER(OVERLAPPED) # Overlapped
1346 LockFileEx.restype = ctypes.wintypes.BOOL
1347 UnlockFileEx = kernel32.UnlockFileEx
1348 UnlockFileEx.argtypes = [
1349 ctypes.wintypes.HANDLE, # hFile
1350 ctypes.wintypes.DWORD, # dwReserved
1351 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1352 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1353 ctypes.POINTER(OVERLAPPED) # Overlapped
1355 UnlockFileEx.restype = ctypes.wintypes.BOOL
1356 whole_low = 0xffffffff
1357 whole_high = 0x7fffffff
1359 def _lock_file(f, exclusive):
1360 overlapped = OVERLAPPED()
1361 overlapped.Offset = 0
1362 overlapped.OffsetHigh = 0
1363 overlapped.hEvent = 0
1364 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1365 handle = msvcrt.get_osfhandle(f.fileno())
1366 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1367 whole_low, whole_high, f._lock_file_overlapped_p):
1368 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1370 def _unlock_file(f):
1371 assert f._lock_file_overlapped_p
1372 handle = msvcrt.get_osfhandle(f.fileno())
1373 if not UnlockFileEx(handle, 0,
1374 whole_low, whole_high, f._lock_file_overlapped_p):
1375 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1378 # Some platforms, such as Jython, is missing fcntl
1382 def _lock_file(f, exclusive):
1383 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1385 def _unlock_file(f):
1386 fcntl.flock(f, fcntl.LOCK_UN)
1388 UNSUPPORTED_MSG = 'file locking is not supported on this platform'
1390 def _lock_file(f, exclusive):
1391 raise IOError(UNSUPPORTED_MSG)
1393 def _unlock_file(f):
1394 raise IOError(UNSUPPORTED_MSG)
1397 class locked_file(object):
1398 def __init__(self, filename, mode, encoding=None):
1399 assert mode in ['r', 'a', 'w']
1400 self.f = io.open(filename, mode, encoding=encoding)
1403 def __enter__(self):
1404 exclusive = self.mode != 'r'
1406 _lock_file(self.f, exclusive)
1412 def __exit__(self, etype, value, traceback):
1414 _unlock_file(self.f)
1421 def write(self, *args):
1422 return self.f.write(*args)
1424 def read(self, *args):
1425 return self.f.read(*args)
1428 def get_filesystem_encoding():
1429 encoding = sys.getfilesystemencoding()
1430 return encoding if encoding is not None else 'utf-8'
1433 def shell_quote(args):
1435 encoding = get_filesystem_encoding()
1437 if isinstance(a, bytes):
1438 # We may get a filename encoded with 'encodeFilename'
1439 a = a.decode(encoding)
1440 quoted_args.append(pipes.quote(a))
1441 return ' '.join(quoted_args)
1444 def smuggle_url(url, data):
1445 """ Pass additional data in a URL for internal use. """
1447 url, idata = unsmuggle_url(url, {})
1449 sdata = compat_urllib_parse_urlencode(
1450 {'__youtubedl_smuggle': json.dumps(data)})
1451 return url + '#' + sdata
1454 def unsmuggle_url(smug_url, default=None):
1455 if '#__youtubedl_smuggle' not in smug_url:
1456 return smug_url, default
1457 url, _, sdata = smug_url.rpartition('#')
1458 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1459 data = json.loads(jsond)
1463 def format_bytes(bytes):
1466 if type(bytes) is str:
1467 bytes = float(bytes)
1471 exponent = int(math.log(bytes, 1024.0))
1472 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1473 converted = float(bytes) / float(1024 ** exponent)
1474 return '%.2f%s' % (converted, suffix)
1477 def lookup_unit_table(unit_table, s):
1478 units_re = '|'.join(re.escape(u) for u in unit_table)
1480 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
1483 num_str = m.group('num').replace(',', '.')
1484 mult = unit_table[m.group('unit')]
1485 return int(float(num_str) * mult)
1488 def parse_filesize(s):
1492 # The lower-case forms are of course incorrect and unofficial,
1493 # but we support those too
1531 return lookup_unit_table(_UNIT_TABLE, s)
1540 if re.match(r'^[\d,.]+$', s):
1541 return str_to_int(s)
1552 return lookup_unit_table(_UNIT_TABLE, s)
1555 def month_by_name(name):
1556 """ Return the number of a month by (locale-independently) English name """
1559 return ENGLISH_MONTH_NAMES.index(name) + 1
1564 def month_by_abbreviation(abbrev):
1565 """ Return the number of a month by (locale-independently) English
1569 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1574 def fix_xml_ampersands(xml_str):
1575 """Replace all the '&' by '&' in XML"""
1577 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1582 def setproctitle(title):
1583 assert isinstance(title, compat_str)
1585 # ctypes in Jython is not complete
1586 # http://bugs.jython.org/issue2148
1587 if sys.platform.startswith('java'):
1591 libc = ctypes.cdll.LoadLibrary('libc.so.6')
1594 title_bytes = title.encode('utf-8')
1595 buf = ctypes.create_string_buffer(len(title_bytes))
1596 buf.value = title_bytes
1598 libc.prctl(15, buf, 0, 0, 0)
1599 except AttributeError:
1600 return # Strange libc, just skip this
1603 def remove_start(s, start):
1604 return s[len(start):] if s is not None and s.startswith(start) else s
1607 def remove_end(s, end):
1608 return s[:-len(end)] if s is not None and s.endswith(end) else s
1611 def remove_quotes(s):
1612 if s is None or len(s) < 2:
1614 for quote in ('"', "'", ):
1615 if s[0] == quote and s[-1] == quote:
1620 def url_basename(url):
1621 path = compat_urlparse.urlparse(url).path
1622 return path.strip('/').split('/')[-1]
1625 class HEADRequest(compat_urllib_request.Request):
1626 def get_method(self):
1630 class PUTRequest(compat_urllib_request.Request):
1631 def get_method(self):
1635 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1638 v = getattr(v, get_attr, None)
1644 return int(v) * invscale // scale
1649 def str_or_none(v, default=None):
1650 return default if v is None else compat_str(v)
1653 def str_to_int(int_str):
1654 """ A more relaxed version of int_or_none """
1657 int_str = re.sub(r'[,\.\+]', '', int_str)
1661 def float_or_none(v, scale=1, invscale=1, default=None):
1665 return float(v) * invscale / scale
1670 def strip_or_none(v):
1671 return None if v is None else v.strip()
1674 def parse_duration(s):
1675 if not isinstance(s, compat_basestring):
1680 days, hours, mins, secs, ms = [None] * 5
1681 m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?$', s)
1683 days, hours, mins, secs, ms = m.groups()
1688 (?P<days>[0-9]+)\s*d(?:ays?)?\s*
1691 (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
1694 (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
1697 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
1700 days, hours, mins, secs, ms = m.groups()
1702 m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)$', s)
1704 hours, mins = m.groups()
1710 duration += float(secs)
1712 duration += float(mins) * 60
1714 duration += float(hours) * 60 * 60
1716 duration += float(days) * 24 * 60 * 60
1718 duration += float(ms)
1722 def prepend_extension(filename, ext, expected_real_ext=None):
1723 name, real_ext = os.path.splitext(filename)
1725 '{0}.{1}{2}'.format(name, ext, real_ext)
1726 if not expected_real_ext or real_ext[1:] == expected_real_ext
1727 else '{0}.{1}'.format(filename, ext))
1730 def replace_extension(filename, ext, expected_real_ext=None):
1731 name, real_ext = os.path.splitext(filename)
1732 return '{0}.{1}'.format(
1733 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1737 def check_executable(exe, args=[]):
1738 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1739 args can be a list of arguments for a short output (like -version) """
1741 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1747 def get_exe_version(exe, args=['--version'],
1748 version_re=None, unrecognized='present'):
1749 """ Returns the version of the specified executable,
1750 or False if the executable is not present """
1752 out, _ = subprocess.Popen(
1753 [encodeArgument(exe)] + args,
1754 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1757 if isinstance(out, bytes): # Python 2.x
1758 out = out.decode('ascii', 'ignore')
1759 return detect_exe_version(out, version_re, unrecognized)
1762 def detect_exe_version(output, version_re=None, unrecognized='present'):
1763 assert isinstance(output, compat_str)
1764 if version_re is None:
1765 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1766 m = re.search(version_re, output)
1773 class PagedList(object):
1775 # This is only useful for tests
1776 return len(self.getslice())
1779 class OnDemandPagedList(PagedList):
1780 def __init__(self, pagefunc, pagesize, use_cache=False):
1781 self._pagefunc = pagefunc
1782 self._pagesize = pagesize
1783 self._use_cache = use_cache
1787 def getslice(self, start=0, end=None):
1789 for pagenum in itertools.count(start // self._pagesize):
1790 firstid = pagenum * self._pagesize
1791 nextfirstid = pagenum * self._pagesize + self._pagesize
1792 if start >= nextfirstid:
1797 page_results = self._cache.get(pagenum)
1798 if page_results is None:
1799 page_results = list(self._pagefunc(pagenum))
1801 self._cache[pagenum] = page_results
1804 start % self._pagesize
1805 if firstid <= start < nextfirstid
1809 ((end - 1) % self._pagesize) + 1
1810 if (end is not None and firstid <= end <= nextfirstid)
1813 if startv != 0 or endv is not None:
1814 page_results = page_results[startv:endv]
1815 res.extend(page_results)
1817 # A little optimization - if current page is not "full", ie. does
1818 # not contain page_size videos then we can assume that this page
1819 # is the last one - there are no more ids on further pages -
1820 # i.e. no need to query again.
1821 if len(page_results) + startv < self._pagesize:
1824 # If we got the whole page, but the next page is not interesting,
1825 # break out early as well
1826 if end == nextfirstid:
1831 class InAdvancePagedList(PagedList):
1832 def __init__(self, pagefunc, pagecount, pagesize):
1833 self._pagefunc = pagefunc
1834 self._pagecount = pagecount
1835 self._pagesize = pagesize
1837 def getslice(self, start=0, end=None):
1839 start_page = start // self._pagesize
1841 self._pagecount if end is None else (end // self._pagesize + 1))
1842 skip_elems = start - start_page * self._pagesize
1843 only_more = None if end is None else end - start
1844 for pagenum in range(start_page, end_page):
1845 page = list(self._pagefunc(pagenum))
1847 page = page[skip_elems:]
1849 if only_more is not None:
1850 if len(page) < only_more:
1851 only_more -= len(page)
1853 page = page[:only_more]
1860 def uppercase_escape(s):
1861 unicode_escape = codecs.getdecoder('unicode_escape')
1863 r'\\U[0-9a-fA-F]{8}',
1864 lambda m: unicode_escape(m.group(0))[0],
1868 def lowercase_escape(s):
1869 unicode_escape = codecs.getdecoder('unicode_escape')
1871 r'\\u[0-9a-fA-F]{4}',
1872 lambda m: unicode_escape(m.group(0))[0],
1876 def escape_rfc3986(s):
1877 """Escape non-ASCII characters as suggested by RFC 3986"""
1878 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1879 s = s.encode('utf-8')
1880 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1883 def escape_url(url):
1884 """Escape URL as suggested by RFC 3986"""
1885 url_parsed = compat_urllib_parse_urlparse(url)
1886 return url_parsed._replace(
1887 netloc=url_parsed.netloc.encode('idna').decode('ascii'),
1888 path=escape_rfc3986(url_parsed.path),
1889 params=escape_rfc3986(url_parsed.params),
1890 query=escape_rfc3986(url_parsed.query),
1891 fragment=escape_rfc3986(url_parsed.fragment)
1895 def read_batch_urls(batch_fd):
1897 if not isinstance(url, compat_str):
1898 url = url.decode('utf-8', 'replace')
1899 BOM_UTF8 = '\xef\xbb\xbf'
1900 if url.startswith(BOM_UTF8):
1901 url = url[len(BOM_UTF8):]
1903 if url.startswith(('#', ';', ']')):
1907 with contextlib.closing(batch_fd) as fd:
1908 return [url for url in map(fixup, fd) if url]
1911 def urlencode_postdata(*args, **kargs):
1912 return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')
1915 def update_url_query(url, query):
1918 parsed_url = compat_urlparse.urlparse(url)
1919 qs = compat_parse_qs(parsed_url.query)
1921 return compat_urlparse.urlunparse(parsed_url._replace(
1922 query=compat_urllib_parse_urlencode(qs, True)))
1925 def update_Request(req, url=None, data=None, headers={}, query={}):
1926 req_headers = req.headers.copy()
1927 req_headers.update(headers)
1928 req_data = data or req.data
1929 req_url = update_url_query(url or req.get_full_url(), query)
1930 req_get_method = req.get_method()
1931 if req_get_method == 'HEAD':
1932 req_type = HEADRequest
1933 elif req_get_method == 'PUT':
1934 req_type = PUTRequest
1936 req_type = compat_urllib_request.Request
1938 req_url, data=req_data, headers=req_headers,
1939 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1940 if hasattr(req, 'timeout'):
1941 new_req.timeout = req.timeout
1945 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
1946 if isinstance(key_or_keys, (list, tuple)):
1947 for key in key_or_keys:
1948 if key not in d or d[key] is None or skip_false_values and not d[key]:
1952 return d.get(key_or_keys, default)
1955 def try_get(src, getter, expected_type=None):
1958 except (AttributeError, KeyError, TypeError, IndexError):
1961 if expected_type is None or isinstance(v, expected_type):
1965 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
1966 return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
1978 def parse_age_limit(s):
1981 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1982 return int(m.group('age')) if m else US_RATINGS.get(s)
1985 def strip_jsonp(code):
1987 r'(?s)^[a-zA-Z0-9_.$]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1990 def js_to_json(code):
1993 if v in ('true', 'false', 'null'):
1995 elif v.startswith('/*') or v == ',':
1998 if v[0] in ("'", '"'):
1999 v = re.sub(r'(?s)\\.|"', lambda m: {
2004 }.get(m.group(0), m.group(0)), v[1:-1])
2007 (r'^0[xX][0-9a-fA-F]+', 16),
2011 for regex, base in INTEGER_TABLE:
2012 im = re.match(regex, v)
2014 i = int(im.group(0), base)
2015 return '"%d":' % i if v.endswith(':') else '%d' % i
2019 return re.sub(r'''(?sx)
2020 "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
2021 '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
2022 /\*.*?\*/|,(?=\s*[\]}])|
2023 [a-zA-Z_][.a-zA-Z_0-9]*|
2024 \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:\s*:)?|
2029 def qualities(quality_ids):
2030 """ Get a numeric quality value out of a list of possible values """
2033 return quality_ids.index(qid)
2039 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
2042 def limit_length(s, length):
2043 """ Add ellipses to overly long strings """
2048 return s[:length - len(ELLIPSES)] + ELLIPSES
2052 def version_tuple(v):
2053 return tuple(int(e) for e in re.split(r'[-.]', v))
2056 def is_outdated_version(version, limit, assume_new=True):
2058 return not assume_new
2060 return version_tuple(version) < version_tuple(limit)
2062 return not assume_new
2065 def ytdl_is_updateable():
2066 """ Returns if youtube-dl can be updated with -U """
2067 from zipimport import zipimporter
2069 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
2072 def args_to_str(args):
2073 # Get a short string representation for a subprocess command
2074 return ' '.join(compat_shlex_quote(a) for a in args)
2077 def error_to_compat_str(err):
2079 # On python 2 error byte string must be decoded with proper
2080 # encoding rather than ascii
2081 if sys.version_info[0] < 3:
2082 err_str = err_str.decode(preferredencoding())
2086 def mimetype2ext(mt):
2092 # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
2093 # it's the most popular one
2094 'audio/mpeg': 'mp3',
2099 _, _, res = mt.rpartition('/')
2103 'smptett+xml': 'tt',
2109 'x-mp4-fragmented': 'mp4',
2114 def urlhandle_detect_ext(url_handle):
2115 getheader = url_handle.headers.get
2117 cd = getheader('Content-Disposition')
2119 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
2121 e = determine_ext(m.group('filename'), default_ext=None)
2125 return mimetype2ext(getheader('Content-Type'))
2128 def encode_data_uri(data, mime_type):
2129 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
2132 def age_restricted(content_limit, age_limit):
2133 """ Returns True iff the content should be blocked """
2135 if age_limit is None: # No limit set
2137 if content_limit is None:
2138 return False # Content available for everyone
2139 return age_limit < content_limit
2142 def is_html(first_bytes):
2143 """ Detect whether a file contains HTML by examining its first bytes. """
2146 (b'\xef\xbb\xbf', 'utf-8'),
2147 (b'\x00\x00\xfe\xff', 'utf-32-be'),
2148 (b'\xff\xfe\x00\x00', 'utf-32-le'),
2149 (b'\xff\xfe', 'utf-16-le'),
2150 (b'\xfe\xff', 'utf-16-be'),
2152 for bom, enc in BOMS:
2153 if first_bytes.startswith(bom):
2154 s = first_bytes[len(bom):].decode(enc, 'replace')
2157 s = first_bytes.decode('utf-8', 'replace')
2159 return re.match(r'^\s*<', s)
2162 def determine_protocol(info_dict):
2163 protocol = info_dict.get('protocol')
2164 if protocol is not None:
2167 url = info_dict['url']
2168 if url.startswith('rtmp'):
2170 elif url.startswith('mms'):
2172 elif url.startswith('rtsp'):
2175 ext = determine_ext(url)
2181 return compat_urllib_parse_urlparse(url).scheme
2184 def render_table(header_row, data):
2185 """ Render a list of rows, each as a list of values """
2186 table = [header_row] + data
2187 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
2188 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
2189 return '\n'.join(format_str % tuple(row) for row in table)
2192 def _match_one(filter_part, dct):
2193 COMPARISON_OPERATORS = {
2201 operator_rex = re.compile(r'''(?x)\s*
2203 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
2205 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
2206 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
2209 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
2210 m = operator_rex.search(filter_part)
2212 op = COMPARISON_OPERATORS[m.group('op')]
2213 if m.group('strval') is not None:
2214 if m.group('op') not in ('=', '!='):
2216 'Operator %s does not support string values!' % m.group('op'))
2217 comparison_value = m.group('strval')
2220 comparison_value = int(m.group('intval'))
2222 comparison_value = parse_filesize(m.group('intval'))
2223 if comparison_value is None:
2224 comparison_value = parse_filesize(m.group('intval') + 'B')
2225 if comparison_value is None:
2227 'Invalid integer value %r in filter part %r' % (
2228 m.group('intval'), filter_part))
2229 actual_value = dct.get(m.group('key'))
2230 if actual_value is None:
2231 return m.group('none_inclusive')
2232 return op(actual_value, comparison_value)
2235 '': lambda v: v is not None,
2236 '!': lambda v: v is None,
2238 operator_rex = re.compile(r'''(?x)\s*
2239 (?P<op>%s)\s*(?P<key>[a-z_]+)
2241 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
2242 m = operator_rex.search(filter_part)
2244 op = UNARY_OPERATORS[m.group('op')]
2245 actual_value = dct.get(m.group('key'))
2246 return op(actual_value)
2248 raise ValueError('Invalid filter part %r' % filter_part)
2251 def match_str(filter_str, dct):
2252 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
2255 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
2258 def match_filter_func(filter_str):
2259 def _match_func(info_dict):
2260 if match_str(filter_str, info_dict):
2263 video_title = info_dict.get('title', info_dict.get('id', 'video'))
2264 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
2268 def parse_dfxp_time_expr(time_expr):
2272 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
2274 return float(mobj.group('time_offset'))
2276 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
2278 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
2281 def srt_subtitles_timecode(seconds):
2282 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
2285 def dfxp2srt(dfxp_data):
2286 _x = functools.partial(xpath_with_ns, ns_map={
2287 'ttml': 'http://www.w3.org/ns/ttml',
2288 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
2289 'ttaf1_0604': 'http://www.w3.org/2006/04/ttaf1',
2292 class TTMLPElementParser(object):
2295 def start(self, tag, attrib):
2296 if tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
2302 def data(self, data):
2306 return self.out.strip()
2308 def parse_node(node):
2309 target = TTMLPElementParser()
2310 parser = xml.etree.ElementTree.XMLParser(target=target)
2311 parser.feed(xml.etree.ElementTree.tostring(node))
2312 return parser.close()
2314 dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
2316 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall(_x('.//ttaf1_0604:p')) or dfxp.findall('.//p')
2319 raise ValueError('Invalid dfxp/TTML subtitle')
2321 for para, index in zip(paras, itertools.count(1)):
2322 begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
2323 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
2324 dur = parse_dfxp_time_expr(para.attrib.get('dur'))
2325 if begin_time is None:
2330 end_time = begin_time + dur
2331 out.append('%d\n%s --> %s\n%s\n\n' % (
2333 srt_subtitles_timecode(begin_time),
2334 srt_subtitles_timecode(end_time),
2340 def cli_option(params, command_option, param):
2341 param = params.get(param)
2342 return [command_option, param] if param is not None else []
2345 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
2346 param = params.get(param)
2347 assert isinstance(param, bool)
2349 return [command_option + separator + (true_value if param else false_value)]
2350 return [command_option, true_value if param else false_value]
2353 def cli_valueless_option(params, command_option, param, expected_value=True):
2354 param = params.get(param)
2355 return [command_option] if param == expected_value else []
2358 def cli_configuration_args(params, param, default=[]):
2359 ex_args = params.get(param)
2362 assert isinstance(ex_args, list)
2366 class ISO639Utils(object):
2367 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2556 def short2long(cls, code):
2557 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2558 return cls._lang_map.get(code[:2])
2561 def long2short(cls, code):
2562 """Convert language code from ISO 639-2/T to ISO 639-1"""
2563 for short_name, long_name in cls._lang_map.items():
2564 if long_name == code:
2568 class ISO3166Utils(object):
2569 # From http://data.okfn.org/data/core/country-list
2571 'AF': 'Afghanistan',
2572 'AX': 'Åland Islands',
2575 'AS': 'American Samoa',
2580 'AG': 'Antigua and Barbuda',
2597 'BO': 'Bolivia, Plurinational State of',
2598 'BQ': 'Bonaire, Sint Eustatius and Saba',
2599 'BA': 'Bosnia and Herzegovina',
2601 'BV': 'Bouvet Island',
2603 'IO': 'British Indian Ocean Territory',
2604 'BN': 'Brunei Darussalam',
2606 'BF': 'Burkina Faso',
2612 'KY': 'Cayman Islands',
2613 'CF': 'Central African Republic',
2617 'CX': 'Christmas Island',
2618 'CC': 'Cocos (Keeling) Islands',
2622 'CD': 'Congo, the Democratic Republic of the',
2623 'CK': 'Cook Islands',
2625 'CI': 'Côte d\'Ivoire',
2630 'CZ': 'Czech Republic',
2634 'DO': 'Dominican Republic',
2637 'SV': 'El Salvador',
2638 'GQ': 'Equatorial Guinea',
2642 'FK': 'Falkland Islands (Malvinas)',
2643 'FO': 'Faroe Islands',
2647 'GF': 'French Guiana',
2648 'PF': 'French Polynesia',
2649 'TF': 'French Southern Territories',
2664 'GW': 'Guinea-Bissau',
2667 'HM': 'Heard Island and McDonald Islands',
2668 'VA': 'Holy See (Vatican City State)',
2675 'IR': 'Iran, Islamic Republic of',
2678 'IM': 'Isle of Man',
2688 'KP': 'Korea, Democratic People\'s Republic of',
2689 'KR': 'Korea, Republic of',
2692 'LA': 'Lao People\'s Democratic Republic',
2698 'LI': 'Liechtenstein',
2702 'MK': 'Macedonia, the Former Yugoslav Republic of',
2709 'MH': 'Marshall Islands',
2715 'FM': 'Micronesia, Federated States of',
2716 'MD': 'Moldova, Republic of',
2727 'NL': 'Netherlands',
2728 'NC': 'New Caledonia',
2729 'NZ': 'New Zealand',
2734 'NF': 'Norfolk Island',
2735 'MP': 'Northern Mariana Islands',
2740 'PS': 'Palestine, State of',
2742 'PG': 'Papua New Guinea',
2745 'PH': 'Philippines',
2749 'PR': 'Puerto Rico',
2753 'RU': 'Russian Federation',
2755 'BL': 'Saint Barthélemy',
2756 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2757 'KN': 'Saint Kitts and Nevis',
2758 'LC': 'Saint Lucia',
2759 'MF': 'Saint Martin (French part)',
2760 'PM': 'Saint Pierre and Miquelon',
2761 'VC': 'Saint Vincent and the Grenadines',
2764 'ST': 'Sao Tome and Principe',
2765 'SA': 'Saudi Arabia',
2769 'SL': 'Sierra Leone',
2771 'SX': 'Sint Maarten (Dutch part)',
2774 'SB': 'Solomon Islands',
2776 'ZA': 'South Africa',
2777 'GS': 'South Georgia and the South Sandwich Islands',
2778 'SS': 'South Sudan',
2783 'SJ': 'Svalbard and Jan Mayen',
2786 'CH': 'Switzerland',
2787 'SY': 'Syrian Arab Republic',
2788 'TW': 'Taiwan, Province of China',
2790 'TZ': 'Tanzania, United Republic of',
2792 'TL': 'Timor-Leste',
2796 'TT': 'Trinidad and Tobago',
2799 'TM': 'Turkmenistan',
2800 'TC': 'Turks and Caicos Islands',
2804 'AE': 'United Arab Emirates',
2805 'GB': 'United Kingdom',
2806 'US': 'United States',
2807 'UM': 'United States Minor Outlying Islands',
2811 'VE': 'Venezuela, Bolivarian Republic of',
2813 'VG': 'Virgin Islands, British',
2814 'VI': 'Virgin Islands, U.S.',
2815 'WF': 'Wallis and Futuna',
2816 'EH': 'Western Sahara',
2823 def short2full(cls, code):
2824 """Convert an ISO 3166-2 country code to the corresponding full name"""
2825 return cls._country_map.get(code.upper())
2828 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2829 def __init__(self, proxies=None):
2830 # Set default handlers
2831 for type in ('http', 'https'):
2832 setattr(self, '%s_open' % type,
2833 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2834 meth(r, proxy, type))
2835 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2837 def proxy_open(self, req, proxy, type):
2838 req_proxy = req.headers.get('Ytdl-request-proxy')
2839 if req_proxy is not None:
2841 del req.headers['Ytdl-request-proxy']
2843 if proxy == '__noproxy__':
2844 return None # No Proxy
2845 if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
2846 req.add_header('Ytdl-socks-proxy', proxy)
2847 # youtube-dl's http/https handlers do wrapping the socket with socks
2849 return compat_urllib_request.ProxyHandler.proxy_open(
2850 self, req, proxy, type)
2853 def ohdave_rsa_encrypt(data, exponent, modulus):
2855 Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
2858 data: data to encrypt, bytes-like object
2859 exponent, modulus: parameter e and N of RSA algorithm, both integer
2860 Output: hex string of encrypted data
2862 Limitation: supports one block encryption only
2865 payload = int(binascii.hexlify(data[::-1]), 16)
2866 encrypted = pow(payload, exponent, modulus)
2867 return '%x' % encrypted
2870 def encode_base_n(num, n, table=None):
2871 FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
2873 table = FULL_TABLE[:n]
2876 raise ValueError('base %d exceeds table length %d' % (n, len(table)))
2883 ret = table[num % n] + ret
2888 def decode_packed_codes(code):
2890 r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)",
2892 obfucasted_code, base, count, symbols = mobj.groups()
2895 symbols = symbols.split('|')
2900 base_n_count = encode_base_n(count, base)
2901 symbol_table[base_n_count] = symbols[count] or base_n_count
2904 r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
2908 def parse_m3u8_attributes(attrib):
2910 for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
2911 if val.startswith('"'):
2917 def urshift(val, n):
2918 return val >> n if val >= 0 else (val + 0x100000000) >> n