4 from __future__ import unicode_literals
34 import xml.etree.ElementTree
38 compat_HTMLParseError,
43 compat_ctypes_WINFUNCTYPE,
44 compat_etree_fromstring,
47 compat_html_entities_html5,
59 compat_urllib_parse_urlencode,
60 compat_urllib_parse_urlparse,
61 compat_urllib_parse_unquote_plus,
62 compat_urllib_request,
73 def register_socks_protocols():
74 # "Register" SOCKS protocols
75 # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
76 # URLs with protocols not in urlparse.uses_netloc are not handled correctly
77 for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
78 if scheme not in compat_urlparse.uses_netloc:
79 compat_urlparse.uses_netloc.append(scheme)
82 # This is not clearly defined otherwise
83 compiled_regex_type = type(re.compile(''))
86 def random_user_agent():
87 _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36'
1666 return _USER_AGENT_TPL % random.choice(_CHROME_VERSIONS)
1670 'User-Agent': random_user_agent(),
1671 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
1672 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1673 'Accept-Encoding': 'gzip, deflate',
1674 'Accept-Language': 'en-us,en;q=0.5',
1679 'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
1683 NO_DEFAULT = object()
1685 ENGLISH_MONTH_NAMES = [
1686 'January', 'February', 'March', 'April', 'May', 'June',
1687 'July', 'August', 'September', 'October', 'November', 'December']
1690 'en': ENGLISH_MONTH_NAMES,
1692 'janvier', 'février', 'mars', 'avril', 'mai', 'juin',
1693 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'],
1696 KNOWN_EXTENSIONS = (
1697 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
1698 'flv', 'f4v', 'f4a', 'f4b',
1699 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
1700 'mkv', 'mka', 'mk3d',
1703 'asf', 'wmv', 'wma',
1709 'f4f', 'f4m', 'm3u8', 'smil')
1711 # needed for sanitizing filenames in restricted mode
1712 ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
1713 itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUY', ['TH', 'ss'],
1714 'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuy', ['th'], 'y')))
1737 '%Y/%m/%d %H:%M:%S',
1739 '%Y-%m-%d %H:%M:%S',
1740 '%Y-%m-%d %H:%M:%S.%f',
1743 '%Y-%m-%dT%H:%M:%SZ',
1744 '%Y-%m-%dT%H:%M:%S.%fZ',
1745 '%Y-%m-%dT%H:%M:%S.%f0Z',
1746 '%Y-%m-%dT%H:%M:%S',
1747 '%Y-%m-%dT%H:%M:%S.%f',
1749 '%b %d %Y at %H:%M',
1750 '%b %d %Y at %H:%M:%S',
1751 '%B %d %Y at %H:%M',
1752 '%B %d %Y at %H:%M:%S',
1755 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
1756 DATE_FORMATS_DAY_FIRST.extend([
1762 '%d/%m/%Y %H:%M:%S',
1765 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
1766 DATE_FORMATS_MONTH_FIRST.extend([
1771 '%m/%d/%Y %H:%M:%S',
1774 PACKED_CODES_RE = r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
1775 JSON_LD_RE = r'(?is)<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>'
1778 def preferredencoding():
1779 """Get preferred encoding.
1781 Returns the best encoding scheme for the system, based on
1782 locale.getpreferredencoding() and some further tweaks.
1785 pref = locale.getpreferredencoding()
1793 def write_json_file(obj, fn):
1794 """ Encode obj as JSON and write it to fn, atomically if possible """
1796 fn = encodeFilename(fn)
1797 if sys.version_info < (3, 0) and sys.platform != 'win32':
1798 encoding = get_filesystem_encoding()
1799 # os.path.basename returns a bytes object, but NamedTemporaryFile
1800 # will fail if the filename contains non ascii characters unless we
1801 # use a unicode object
1802 path_basename = lambda f: os.path.basename(fn).decode(encoding)
1803 # the same for os.path.dirname
1804 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
1806 path_basename = os.path.basename
1807 path_dirname = os.path.dirname
1811 'prefix': path_basename(fn) + '.',
1812 'dir': path_dirname(fn),
1816 # In Python 2.x, json.dump expects a bytestream.
1817 # In Python 3.x, it writes to a character stream
1818 if sys.version_info < (3, 0):
1823 'encoding': 'utf-8',
1826 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
1831 if sys.platform == 'win32':
1832 # Need to remove existing file on Windows, else os.rename raises
1833 # WindowsError or FileExistsError.
1838 os.rename(tf.name, fn)
1847 if sys.version_info >= (2, 7):
1848 def find_xpath_attr(node, xpath, key, val=None):
1849 """ Find the xpath xpath[@key=val] """
1850 assert re.match(r'^[a-zA-Z_-]+$', key)
1851 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
1852 return node.find(expr)
1854 def find_xpath_attr(node, xpath, key, val=None):
1855 for f in node.findall(compat_xpath(xpath)):
1856 if key not in f.attrib:
1858 if val is None or f.attrib.get(key) == val:
1862 # On python2.6 the xml.etree.ElementTree.Element methods don't support
1863 # the namespace parameter
1866 def xpath_with_ns(path, ns_map):
1867 components = [c.split(':') for c in path.split('/')]
1869 for c in components:
1871 replaced.append(c[0])
1874 replaced.append('{%s}%s' % (ns_map[ns], tag))
1875 return '/'.join(replaced)
1878 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
1879 def _find_xpath(xpath):
1880 return node.find(compat_xpath(xpath))
1882 if isinstance(xpath, (str, compat_str)):
1883 n = _find_xpath(xpath)
1891 if default is not NO_DEFAULT:
1894 name = xpath if name is None else name
1895 raise ExtractorError('Could not find XML element %s' % name)
1901 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
1902 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
1903 if n is None or n == default:
1906 if default is not NO_DEFAULT:
1909 name = xpath if name is None else name
1910 raise ExtractorError('Could not find XML element\'s text %s' % name)
1916 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
1917 n = find_xpath_attr(node, xpath, key)
1919 if default is not NO_DEFAULT:
1922 name = '%s[@%s]' % (xpath, key) if name is None else name
1923 raise ExtractorError('Could not find XML attribute %s' % name)
1926 return n.attrib[key]
1929 def get_element_by_id(id, html):
1930 """Return the content of the tag with the specified ID in the passed HTML document"""
1931 return get_element_by_attribute('id', id, html)
1934 def get_element_by_class(class_name, html):
1935 """Return the content of the first tag with the specified class in the passed HTML document"""
1936 retval = get_elements_by_class(class_name, html)
1937 return retval[0] if retval else None
1940 def get_element_by_attribute(attribute, value, html, escape_value=True):
1941 retval = get_elements_by_attribute(attribute, value, html, escape_value)
1942 return retval[0] if retval else None
1945 def get_elements_by_class(class_name, html):
1946 """Return the content of all tags with the specified class in the passed HTML document as a list"""
1947 return get_elements_by_attribute(
1948 'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
1949 html, escape_value=False)
1952 def get_elements_by_attribute(attribute, value, html, escape_value=True):
1953 """Return the content of the tag with the specified attribute in the passed HTML document"""
1955 value = re.escape(value) if escape_value else value
1958 for m in re.finditer(r'''(?xs)
1960 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
1962 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
1966 ''' % (re.escape(attribute), value), html):
1967 res = m.group('content')
1969 if res.startswith('"') or res.startswith("'"):
1972 retlist.append(unescapeHTML(res))
1977 class HTMLAttributeParser(compat_HTMLParser):
1978 """Trivial HTML parser to gather the attributes for a single element"""
1981 compat_HTMLParser.__init__(self)
1983 def handle_starttag(self, tag, attrs):
1984 self.attrs = dict(attrs)
1987 def extract_attributes(html_element):
1988 """Given a string for an HTML element such as
1990 a="foo" B="bar" c="&98;az" d=boz
1991 empty= noval entity="&"
1994 Decode and return a dictionary of attributes.
1996 'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
1997 'empty': '', 'noval': None, 'entity': '&',
1998 'sq': '"', 'dq': '\''
2000 NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
2001 but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
2003 parser = HTMLAttributeParser()
2005 parser.feed(html_element)
2007 # Older Python may throw HTMLParseError in case of malformed HTML
2008 except compat_HTMLParseError:
2013 def clean_html(html):
2014 """Clean an HTML snippet into a readable string"""
2016 if html is None: # Convenience for sanitizing descriptions etc.
2020 html = html.replace('\n', ' ')
2021 html = re.sub(r'(?u)\s*<\s*br\s*/?\s*>\s*', '\n', html)
2022 html = re.sub(r'(?u)<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
2024 html = re.sub('<.*?>', '', html)
2025 # Replace html entities
2026 html = unescapeHTML(html)
2030 def sanitize_open(filename, open_mode):
2031 """Try to open the given filename, and slightly tweak it if this fails.
2033 Attempts to open the given filename. If this fails, it tries to change
2034 the filename slightly, step by step, until it's either able to open it
2035 or it fails and raises a final exception, like the standard open()
2038 It returns the tuple (stream, definitive_file_name).
2042 if sys.platform == 'win32':
2044 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
2045 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
2046 stream = open(encodeFilename(filename), open_mode)
2047 return (stream, filename)
2048 except (IOError, OSError) as err:
2049 if err.errno in (errno.EACCES,):
2052 # In case of error, try to remove win32 forbidden chars
2053 alt_filename = sanitize_path(filename)
2054 if alt_filename == filename:
2057 # An exception here should be caught in the caller
2058 stream = open(encodeFilename(alt_filename), open_mode)
2059 return (stream, alt_filename)
2062 def timeconvert(timestr):
2063 """Convert RFC 2822 defined time string into system timestamp"""
2065 timetuple = email.utils.parsedate_tz(timestr)
2066 if timetuple is not None:
2067 timestamp = email.utils.mktime_tz(timetuple)
2071 def sanitize_filename(s, restricted=False, is_id=False):
2072 """Sanitizes a string so it could be used as part of a filename.
2073 If restricted is set, use a stricter subset of allowed characters.
2074 Set is_id if this is not an arbitrary string, but an ID that should be kept
2077 def replace_insane(char):
2078 if restricted and char in ACCENT_CHARS:
2079 return ACCENT_CHARS[char]
2080 if char == '?' or ord(char) < 32 or ord(char) == 127:
2083 return '' if restricted else '\''
2085 return '_-' if restricted else ' -'
2086 elif char in '\\/|*<>':
2088 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
2090 if restricted and ord(char) > 127:
2095 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
2096 result = ''.join(map(replace_insane, s))
2098 while '__' in result:
2099 result = result.replace('__', '_')
2100 result = result.strip('_')
2101 # Common case of "Foreign band name - English song title"
2102 if restricted and result.startswith('-_'):
2104 if result.startswith('-'):
2105 result = '_' + result[len('-'):]
2106 result = result.lstrip('.')
2112 def sanitize_path(s):
2113 """Sanitizes and normalizes path on Windows"""
2114 if sys.platform != 'win32':
2116 drive_or_unc, _ = os.path.splitdrive(s)
2117 if sys.version_info < (2, 7) and not drive_or_unc:
2118 drive_or_unc, _ = os.path.splitunc(s)
2119 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
2123 path_part if path_part in ['.', '..'] else re.sub(r'(?:[/<>:"\|\\?\*]|[\s.]$)', '#', path_part)
2124 for path_part in norm_path]
2126 sanitized_path.insert(0, drive_or_unc + os.path.sep)
2127 return os.path.join(*sanitized_path)
2130 def sanitize_url(url):
2131 # Prepend protocol-less URLs with `http:` scheme in order to mitigate
2132 # the number of unwanted failures due to missing protocol
2133 if url.startswith('//'):
2134 return 'http:%s' % url
2135 # Fix some common typos seen so far
2137 # https://github.com/ytdl-org/youtube-dl/issues/15649
2138 (r'^httpss://', r'https://'),
2139 # https://bx1.be/lives/direct-tv/
2140 (r'^rmtp([es]?)://', r'rtmp\1://'),
2142 for mistake, fixup in COMMON_TYPOS:
2143 if re.match(mistake, url):
2144 return re.sub(mistake, fixup, url)
2148 def sanitized_Request(url, *args, **kwargs):
2149 return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)
2153 """Expand shell variables and ~"""
2154 return os.path.expandvars(compat_expanduser(s))
2157 def orderedSet(iterable):
2158 """ Remove all duplicates from the input iterable """
2166 def _htmlentity_transform(entity_with_semicolon):
2167 """Transforms an HTML entity to a character."""
2168 entity = entity_with_semicolon[:-1]
2170 # Known non-numeric HTML entity
2171 if entity in compat_html_entities.name2codepoint:
2172 return compat_chr(compat_html_entities.name2codepoint[entity])
2174 # TODO: HTML5 allows entities without a semicolon. For example,
2175 # 'Éric' should be decoded as 'Éric'.
2176 if entity_with_semicolon in compat_html_entities_html5:
2177 return compat_html_entities_html5[entity_with_semicolon]
2179 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
2180 if mobj is not None:
2181 numstr = mobj.group(1)
2182 if numstr.startswith('x'):
2184 numstr = '0%s' % numstr
2187 # See https://github.com/ytdl-org/youtube-dl/issues/7518
2189 return compat_chr(int(numstr, base))
2193 # Unknown entity in name, return its literal representation
2194 return '&%s;' % entity
2197 def unescapeHTML(s):
2200 assert type(s) == compat_str
2203 r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
2206 def get_subprocess_encoding():
2207 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
2208 # For subprocess calls, encode with locale encoding
2209 # Refer to http://stackoverflow.com/a/9951851/35070
2210 encoding = preferredencoding()
2212 encoding = sys.getfilesystemencoding()
2213 if encoding is None:
2218 def encodeFilename(s, for_subprocess=False):
2220 @param s The name of the file
2223 assert type(s) == compat_str
2225 # Python 3 has a Unicode API
2226 if sys.version_info >= (3, 0):
2229 # Pass '' directly to use Unicode APIs on Windows 2000 and up
2230 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
2231 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
2232 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
2235 # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
2236 if sys.platform.startswith('java'):
2239 return s.encode(get_subprocess_encoding(), 'ignore')
2242 def decodeFilename(b, for_subprocess=False):
2244 if sys.version_info >= (3, 0):
2247 if not isinstance(b, bytes):
2250 return b.decode(get_subprocess_encoding(), 'ignore')
2253 def encodeArgument(s):
2254 if not isinstance(s, compat_str):
2255 # Legacy code that uses byte strings
2256 # Uncomment the following line after fixing all post processors
2257 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
2258 s = s.decode('ascii')
2259 return encodeFilename(s, True)
2262 def decodeArgument(b):
2263 return decodeFilename(b, True)
2266 def decodeOption(optval):
2269 if isinstance(optval, bytes):
2270 optval = optval.decode(preferredencoding())
2272 assert isinstance(optval, compat_str)
2276 def formatSeconds(secs):
2278 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
2280 return '%d:%02d' % (secs // 60, secs % 60)
2285 def make_HTTPS_handler(params, **kwargs):
2286 opts_no_check_certificate = params.get('nocheckcertificate', False)
2287 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
2288 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
2289 if opts_no_check_certificate:
2290 context.check_hostname = False
2291 context.verify_mode = ssl.CERT_NONE
2293 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
2296 # (create_default_context present but HTTPSHandler has no context=)
2299 if sys.version_info < (3, 2):
2300 return YoutubeDLHTTPSHandler(params, **kwargs)
2301 else: # Python < 3.4
2302 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
2303 context.verify_mode = (ssl.CERT_NONE
2304 if opts_no_check_certificate
2305 else ssl.CERT_REQUIRED)
2306 context.set_default_verify_paths()
2307 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
2310 def bug_reports_message():
2311 if ytdl_is_updateable():
2312 update_cmd = 'type youtube-dl -U to update'
2314 update_cmd = 'see https://yt-dl.org/update on how to update'
2315 msg = '; please report this issue on https://yt-dl.org/bug .'
2316 msg += ' Make sure you are using the latest version; %s.' % update_cmd
2317 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
2321 class YoutubeDLError(Exception):
2322 """Base exception for YoutubeDL errors."""
2326 class ExtractorError(YoutubeDLError):
2327 """Error during info extraction."""
2329 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
2330 """ tb, if given, is the original traceback (so that it can be printed out).
2331 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
2334 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
2336 if video_id is not None:
2337 msg = video_id + ': ' + msg
2339 msg += ' (caused by %r)' % cause
2341 msg += bug_reports_message()
2342 super(ExtractorError, self).__init__(msg)
2345 self.exc_info = sys.exc_info() # preserve original exception
2347 self.video_id = video_id
2349 def format_traceback(self):
2350 if self.traceback is None:
2352 return ''.join(traceback.format_tb(self.traceback))
2355 class UnsupportedError(ExtractorError):
2356 def __init__(self, url):
2357 super(UnsupportedError, self).__init__(
2358 'Unsupported URL: %s' % url, expected=True)
2362 class RegexNotFoundError(ExtractorError):
2363 """Error when a regex didn't match"""
2367 class GeoRestrictedError(ExtractorError):
2368 """Geographic restriction Error exception.
2370 This exception may be thrown when a video is not available from your
2371 geographic location due to geographic restrictions imposed by a website.
2373 def __init__(self, msg, countries=None):
2374 super(GeoRestrictedError, self).__init__(msg, expected=True)
2376 self.countries = countries
2379 class DownloadError(YoutubeDLError):
2380 """Download Error exception.
2382 This exception may be thrown by FileDownloader objects if they are not
2383 configured to continue on errors. They will contain the appropriate
2387 def __init__(self, msg, exc_info=None):
2388 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
2389 super(DownloadError, self).__init__(msg)
2390 self.exc_info = exc_info
2393 class SameFileError(YoutubeDLError):
2394 """Same File exception.
2396 This exception will be thrown by FileDownloader objects if they detect
2397 multiple files would have to be downloaded to the same file on disk.
2402 class PostProcessingError(YoutubeDLError):
2403 """Post Processing exception.
2405 This exception may be raised by PostProcessor's .run() method to
2406 indicate an error in the postprocessing task.
2409 def __init__(self, msg):
2410 super(PostProcessingError, self).__init__(msg)
2414 class MaxDownloadsReached(YoutubeDLError):
2415 """ --max-downloads limit has been reached. """
2419 class UnavailableVideoError(YoutubeDLError):
2420 """Unavailable Format exception.
2422 This exception will be thrown when a video is requested
2423 in a format that is not available for that video.
2428 class ContentTooShortError(YoutubeDLError):
2429 """Content Too Short exception.
2431 This exception may be raised by FileDownloader objects when a file they
2432 download is too small for what the server announced first, indicating
2433 the connection was probably interrupted.
2436 def __init__(self, downloaded, expected):
2437 super(ContentTooShortError, self).__init__(
2438 'Downloaded {0} bytes, expected {1} bytes'.format(downloaded, expected)
2441 self.downloaded = downloaded
2442 self.expected = expected
2445 class XAttrMetadataError(YoutubeDLError):
2446 def __init__(self, code=None, msg='Unknown error'):
2447 super(XAttrMetadataError, self).__init__(msg)
2451 # Parsing code and msg
2452 if (self.code in (errno.ENOSPC, errno.EDQUOT)
2453 or 'No space left' in self.msg or 'Disk quota excedded' in self.msg):
2454 self.reason = 'NO_SPACE'
2455 elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
2456 self.reason = 'VALUE_TOO_LONG'
2458 self.reason = 'NOT_SUPPORTED'
2461 class XAttrUnavailableError(YoutubeDLError):
2465 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
2466 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
2467 # expected HTTP responses to meet HTTP/1.0 or later (see also
2468 # https://github.com/ytdl-org/youtube-dl/issues/6727)
2469 if sys.version_info < (3, 0):
2470 kwargs['strict'] = True
2471 hc = http_class(*args, **compat_kwargs(kwargs))
2472 source_address = ydl_handler._params.get('source_address')
2474 if source_address is not None:
2475 # This is to workaround _create_connection() from socket where it will try all
2476 # address data from getaddrinfo() including IPv6. This filters the result from
2477 # getaddrinfo() based on the source_address value.
2478 # This is based on the cpython socket.create_connection() function.
2479 # https://github.com/python/cpython/blob/master/Lib/socket.py#L691
2480 def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
2481 host, port = address
2483 addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
2484 af = socket.AF_INET if '.' in source_address[0] else socket.AF_INET6
2485 ip_addrs = [addr for addr in addrs if addr[0] == af]
2486 if addrs and not ip_addrs:
2487 ip_version = 'v4' if af == socket.AF_INET else 'v6'
2489 "No remote IP%s addresses available for connect, can't use '%s' as source address"
2490 % (ip_version, source_address[0]))
2491 for res in ip_addrs:
2492 af, socktype, proto, canonname, sa = res
2495 sock = socket.socket(af, socktype, proto)
2496 if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
2497 sock.settimeout(timeout)
2498 sock.bind(source_address)
2500 err = None # Explicitly break reference cycle
2502 except socket.error as _:
2504 if sock is not None:
2509 raise socket.error('getaddrinfo returns an empty list')
2510 if hasattr(hc, '_create_connection'):
2511 hc._create_connection = _create_connection
2512 sa = (source_address, 0)
2513 if hasattr(hc, 'source_address'): # Python 2.7+
2514 hc.source_address = sa
2516 def _hc_connect(self, *args, **kwargs):
2517 sock = _create_connection(
2518 (self.host, self.port), self.timeout, sa)
2520 self.sock = ssl.wrap_socket(
2521 sock, self.key_file, self.cert_file,
2522 ssl_version=ssl.PROTOCOL_TLSv1)
2525 hc.connect = functools.partial(_hc_connect, hc)
2530 def handle_youtubedl_headers(headers):
2531 filtered_headers = headers
2533 if 'Youtubedl-no-compression' in filtered_headers:
2534 filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
2535 del filtered_headers['Youtubedl-no-compression']
2537 return filtered_headers
2540 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
2541 """Handler for HTTP requests and responses.
2543 This class, when installed with an OpenerDirector, automatically adds
2544 the standard headers to every HTTP request and handles gzipped and
2545 deflated responses from web servers. If compression is to be avoided in
2546 a particular request, the original request in the program code only has
2547 to include the HTTP header "Youtubedl-no-compression", which will be
2548 removed before making the real request.
2550 Part of this code was copied from:
2552 http://techknack.net/python-urllib2-handlers/
2554 Andrew Rowls, the author of that code, agreed to release it to the
2558 def __init__(self, params, *args, **kwargs):
2559 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
2560 self._params = params
2562 def http_open(self, req):
2563 conn_class = compat_http_client.HTTPConnection
2565 socks_proxy = req.headers.get('Ytdl-socks-proxy')
2567 conn_class = make_socks_conn_class(conn_class, socks_proxy)
2568 del req.headers['Ytdl-socks-proxy']
2570 return self.do_open(functools.partial(
2571 _create_http_connection, self, conn_class, False),
2577 return zlib.decompress(data, -zlib.MAX_WBITS)
2579 return zlib.decompress(data)
2581 def http_request(self, req):
2582 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
2583 # always respected by websites, some tend to give out URLs with non percent-encoded
2584 # non-ASCII characters (see telemb.py, ard.py [#3412])
2585 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
2586 # To work around aforementioned issue we will replace request's original URL with
2587 # percent-encoded one
2588 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
2589 # the code of this workaround has been moved here from YoutubeDL.urlopen()
2590 url = req.get_full_url()
2591 url_escaped = escape_url(url)
2593 # Substitute URL if any change after escaping
2594 if url != url_escaped:
2595 req = update_Request(req, url=url_escaped)
2597 for h, v in std_headers.items():
2598 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
2599 # The dict keys are capitalized because of this bug by urllib
2600 if h.capitalize() not in req.headers:
2601 req.add_header(h, v)
2603 req.headers = handle_youtubedl_headers(req.headers)
2605 if sys.version_info < (2, 7) and '#' in req.get_full_url():
2606 # Python 2.6 is brain-dead when it comes to fragments
2607 req._Request__original = req._Request__original.partition('#')[0]
2608 req._Request__r_type = req._Request__r_type.partition('#')[0]
2612 def http_response(self, req, resp):
2615 if resp.headers.get('Content-encoding', '') == 'gzip':
2616 content = resp.read()
2617 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
2619 uncompressed = io.BytesIO(gz.read())
2620 except IOError as original_ioerror:
2621 # There may be junk add the end of the file
2622 # See http://stackoverflow.com/q/4928560/35070 for details
2623 for i in range(1, 1024):
2625 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
2626 uncompressed = io.BytesIO(gz.read())
2631 raise original_ioerror
2632 resp = compat_urllib_request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
2633 resp.msg = old_resp.msg
2634 del resp.headers['Content-encoding']
2636 if resp.headers.get('Content-encoding', '') == 'deflate':
2637 gz = io.BytesIO(self.deflate(resp.read()))
2638 resp = compat_urllib_request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
2639 resp.msg = old_resp.msg
2640 del resp.headers['Content-encoding']
2641 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
2642 # https://github.com/ytdl-org/youtube-dl/issues/6457).
2643 if 300 <= resp.code < 400:
2644 location = resp.headers.get('Location')
2646 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
2647 if sys.version_info >= (3, 0):
2648 location = location.encode('iso-8859-1').decode('utf-8')
2650 location = location.decode('utf-8')
2651 location_escaped = escape_url(location)
2652 if location != location_escaped:
2653 del resp.headers['Location']
2654 if sys.version_info < (3, 0):
2655 location_escaped = location_escaped.encode('utf-8')
2656 resp.headers['Location'] = location_escaped
2659 https_request = http_request
2660 https_response = http_response
2663 def make_socks_conn_class(base_class, socks_proxy):
2664 assert issubclass(base_class, (
2665 compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
2667 url_components = compat_urlparse.urlparse(socks_proxy)
2668 if url_components.scheme.lower() == 'socks5':
2669 socks_type = ProxyType.SOCKS5
2670 elif url_components.scheme.lower() in ('socks', 'socks4'):
2671 socks_type = ProxyType.SOCKS4
2672 elif url_components.scheme.lower() == 'socks4a':
2673 socks_type = ProxyType.SOCKS4A
2675 def unquote_if_non_empty(s):
2678 return compat_urllib_parse_unquote_plus(s)
2682 url_components.hostname, url_components.port or 1080,
2684 unquote_if_non_empty(url_components.username),
2685 unquote_if_non_empty(url_components.password),
2688 class SocksConnection(base_class):
2690 self.sock = sockssocket()
2691 self.sock.setproxy(*proxy_args)
2692 if type(self.timeout) in (int, float):
2693 self.sock.settimeout(self.timeout)
2694 self.sock.connect((self.host, self.port))
2696 if isinstance(self, compat_http_client.HTTPSConnection):
2697 if hasattr(self, '_context'): # Python > 2.6
2698 self.sock = self._context.wrap_socket(
2699 self.sock, server_hostname=self.host)
2701 self.sock = ssl.wrap_socket(self.sock)
2703 return SocksConnection
2706 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
2707 def __init__(self, params, https_conn_class=None, *args, **kwargs):
2708 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
2709 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
2710 self._params = params
2712 def https_open(self, req):
2714 conn_class = self._https_conn_class
2716 if hasattr(self, '_context'): # python > 2.6
2717 kwargs['context'] = self._context
2718 if hasattr(self, '_check_hostname'): # python 3.x
2719 kwargs['check_hostname'] = self._check_hostname
2721 socks_proxy = req.headers.get('Ytdl-socks-proxy')
2723 conn_class = make_socks_conn_class(conn_class, socks_proxy)
2724 del req.headers['Ytdl-socks-proxy']
2726 return self.do_open(functools.partial(
2727 _create_http_connection, self, conn_class, True),
2731 class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
2732 _HTTPONLY_PREFIX = '#HttpOnly_'
2734 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
2735 # Store session cookies with `expires` set to 0 instead of an empty
2738 if cookie.expires is None:
2740 compat_cookiejar.MozillaCookieJar.save(self, filename, ignore_discard, ignore_expires)
2742 def load(self, filename=None, ignore_discard=False, ignore_expires=False):
2743 """Load cookies from a file."""
2744 if filename is None:
2745 if self.filename is not None:
2746 filename = self.filename
2748 raise ValueError(compat_cookiejar.MISSING_FILENAME_TEXT)
2751 with open(filename) as f:
2753 if line.startswith(self._HTTPONLY_PREFIX):
2754 line = line[len(self._HTTPONLY_PREFIX):]
2755 # Cookie file may contain spaces instead of tabs.
2756 # Replace all spaces with tabs to make such cookie files work
2757 # with MozillaCookieJar.
2758 if not line.startswith('#'):
2759 line = re.sub(r' +', r'\t', line)
2760 cf.write(compat_str(line))
2762 self._really_load(cf, filename, ignore_discard, ignore_expires)
2763 # Session cookies are denoted by either `expires` field set to
2764 # an empty string or 0. MozillaCookieJar only recognizes the former
2765 # (see [1]). So we need force the latter to be recognized as session
2766 # cookies on our own.
2767 # Session cookies may be important for cookies-based authentication,
2768 # e.g. usually, when user does not check 'Remember me' check box while
2769 # logging in on a site, some important cookies are stored as session
2770 # cookies so that not recognizing them will result in failed login.
2771 # 1. https://bugs.python.org/issue17164
2773 # Treat `expires=0` cookies as session cookies
2774 if cookie.expires == 0:
2775 cookie.expires = None
2776 cookie.discard = True
2779 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
2780 def __init__(self, cookiejar=None):
2781 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
2783 def http_response(self, request, response):
2784 # Python 2 will choke on next HTTP request in row if there are non-ASCII
2785 # characters in Set-Cookie HTTP header of last response (see
2786 # https://github.com/ytdl-org/youtube-dl/issues/6769).
2787 # In order to at least prevent crashing we will percent encode Set-Cookie
2788 # header before HTTPCookieProcessor starts processing it.
2789 # if sys.version_info < (3, 0) and response.headers:
2790 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
2791 # set_cookie = response.headers.get(set_cookie_header)
2793 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
2794 # if set_cookie != set_cookie_escaped:
2795 # del response.headers[set_cookie_header]
2796 # response.headers[set_cookie_header] = set_cookie_escaped
2797 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
2799 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
2800 https_response = http_response
2803 class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
2804 if sys.version_info[0] < 3:
2805 def redirect_request(self, req, fp, code, msg, headers, newurl):
2806 # On python 2 urlh.geturl() may sometimes return redirect URL
2807 # as byte string instead of unicode. This workaround allows
2808 # to force it always return unicode.
2809 return compat_urllib_request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, compat_str(newurl))
2812 def extract_timezone(date_str):
2814 r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
2817 timezone = datetime.timedelta()
2819 date_str = date_str[:-len(m.group('tz'))]
2820 if not m.group('sign'):
2821 timezone = datetime.timedelta()
2823 sign = 1 if m.group('sign') == '+' else -1
2824 timezone = datetime.timedelta(
2825 hours=sign * int(m.group('hours')),
2826 minutes=sign * int(m.group('minutes')))
2827 return timezone, date_str
2830 def parse_iso8601(date_str, delimiter='T', timezone=None):
2831 """ Return a UNIX timestamp from the given date """
2833 if date_str is None:
2836 date_str = re.sub(r'\.[0-9]+', '', date_str)
2838 if timezone is None:
2839 timezone, date_str = extract_timezone(date_str)
2842 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
2843 dt = datetime.datetime.strptime(date_str, date_format) - timezone
2844 return calendar.timegm(dt.timetuple())
2849 def date_formats(day_first=True):
2850 return DATE_FORMATS_DAY_FIRST if day_first else DATE_FORMATS_MONTH_FIRST
2853 def unified_strdate(date_str, day_first=True):
2854 """Return a string with the date in the format YYYYMMDD"""
2856 if date_str is None:
2860 date_str = date_str.replace(',', ' ')
2861 # Remove AM/PM + timezone
2862 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
2863 _, date_str = extract_timezone(date_str)
2865 for expression in date_formats(day_first):
2867 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
2870 if upload_date is None:
2871 timetuple = email.utils.parsedate_tz(date_str)
2874 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
2877 if upload_date is not None:
2878 return compat_str(upload_date)
2881 def unified_timestamp(date_str, day_first=True):
2882 if date_str is None:
2885 date_str = re.sub(r'[,|]', '', date_str)
2887 pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
2888 timezone, date_str = extract_timezone(date_str)
2890 # Remove AM/PM + timezone
2891 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
2893 # Remove unrecognized timezones from ISO 8601 alike timestamps
2894 m = re.search(r'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str)
2896 date_str = date_str[:-len(m.group('tz'))]
2898 # Python only supports microseconds, so remove nanoseconds
2899 m = re.search(r'^([0-9]{4,}-[0-9]{1,2}-[0-9]{1,2}T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]{6})[0-9]+$', date_str)
2901 date_str = m.group(1)
2903 for expression in date_formats(day_first):
2905 dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
2906 return calendar.timegm(dt.timetuple())
2909 timetuple = email.utils.parsedate_tz(date_str)
2911 return calendar.timegm(timetuple) + pm_delta * 3600
2914 def determine_ext(url, default_ext='unknown_video'):
2915 if url is None or '.' not in url:
2917 guess = url.partition('?')[0].rpartition('.')[2]
2918 if re.match(r'^[A-Za-z0-9]+$', guess):
2920 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
2921 elif guess.rstrip('/') in KNOWN_EXTENSIONS:
2922 return guess.rstrip('/')
2927 def subtitles_filename(filename, sub_lang, sub_format, expected_real_ext=None):
2928 return replace_extension(filename, sub_lang + '.' + sub_format, expected_real_ext)
2931 def date_from_str(date_str):
2933 Return a datetime object from a string in the format YYYYMMDD or
2934 (now|today)[+-][0-9](day|week|month|year)(s)?"""
2935 today = datetime.date.today()
2936 if date_str in ('now', 'today'):
2938 if date_str == 'yesterday':
2939 return today - datetime.timedelta(days=1)
2940 match = re.match(r'(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
2941 if match is not None:
2942 sign = match.group('sign')
2943 time = int(match.group('time'))
2946 unit = match.group('unit')
2947 # A bad approximation?
2951 elif unit == 'year':
2955 delta = datetime.timedelta(**{unit: time})
2956 return today + delta
2957 return datetime.datetime.strptime(date_str, '%Y%m%d').date()
2960 def hyphenate_date(date_str):
2962 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
2963 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
2964 if match is not None:
2965 return '-'.join(match.groups())
2970 class DateRange(object):
2971 """Represents a time interval between two dates"""
2973 def __init__(self, start=None, end=None):
2974 """start and end must be strings in the format accepted by date"""
2975 if start is not None:
2976 self.start = date_from_str(start)
2978 self.start = datetime.datetime.min.date()
2980 self.end = date_from_str(end)
2982 self.end = datetime.datetime.max.date()
2983 if self.start > self.end:
2984 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
2988 """Returns a range that only contains the given day"""
2989 return cls(day, day)
2991 def __contains__(self, date):
2992 """Check if the date is in the range"""
2993 if not isinstance(date, datetime.date):
2994 date = date_from_str(date)
2995 return self.start <= date <= self.end
2998 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
3001 def platform_name():
3002 """ Returns the platform name as a compat_str """
3003 res = platform.platform()
3004 if isinstance(res, bytes):
3005 res = res.decode(preferredencoding())
3007 assert isinstance(res, compat_str)
3011 def _windows_write_string(s, out):
3012 """ Returns True if the string was written using special methods,
3013 False if it has yet to be written out."""
3014 # Adapted from http://stackoverflow.com/a/3259271/35070
3017 import ctypes.wintypes
3025 fileno = out.fileno()
3026 except AttributeError:
3027 # If the output stream doesn't have a fileno, it's virtual
3029 except io.UnsupportedOperation:
3030 # Some strange Windows pseudo files?
3032 if fileno not in WIN_OUTPUT_IDS:
3035 GetStdHandle = compat_ctypes_WINFUNCTYPE(
3036 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
3037 ('GetStdHandle', ctypes.windll.kernel32))
3038 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
3040 WriteConsoleW = compat_ctypes_WINFUNCTYPE(
3041 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
3042 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
3043 ctypes.wintypes.LPVOID)(('WriteConsoleW', ctypes.windll.kernel32))
3044 written = ctypes.wintypes.DWORD(0)
3046 GetFileType = compat_ctypes_WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(('GetFileType', ctypes.windll.kernel32))
3047 FILE_TYPE_CHAR = 0x0002
3048 FILE_TYPE_REMOTE = 0x8000
3049 GetConsoleMode = compat_ctypes_WINFUNCTYPE(
3050 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
3051 ctypes.POINTER(ctypes.wintypes.DWORD))(
3052 ('GetConsoleMode', ctypes.windll.kernel32))
3053 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
3055 def not_a_console(handle):
3056 if handle == INVALID_HANDLE_VALUE or handle is None:
3058 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
3059 or GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
3061 if not_a_console(h):
3064 def next_nonbmp_pos(s):
3066 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
3067 except StopIteration:
3071 count = min(next_nonbmp_pos(s), 1024)
3073 ret = WriteConsoleW(
3074 h, s, count if count else 2, ctypes.byref(written), None)
3076 raise OSError('Failed to write string')
3077 if not count: # We just wrote a non-BMP character
3078 assert written.value == 2
3081 assert written.value > 0
3082 s = s[written.value:]
3086 def write_string(s, out=None, encoding=None):
3089 assert type(s) == compat_str
3091 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
3092 if _windows_write_string(s, out):
3095 if ('b' in getattr(out, 'mode', '')
3096 or sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
3097 byt = s.encode(encoding or preferredencoding(), 'ignore')
3099 elif hasattr(out, 'buffer'):
3100 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
3101 byt = s.encode(enc, 'ignore')
3102 out.buffer.write(byt)
3108 def bytes_to_intlist(bs):
3111 if isinstance(bs[0], int): # Python 3
3114 return [ord(c) for c in bs]
3117 def intlist_to_bytes(xs):
3120 return compat_struct_pack('%dB' % len(xs), *xs)
3123 # Cross-platform file locking
3124 if sys.platform == 'win32':
3125 import ctypes.wintypes
3128 class OVERLAPPED(ctypes.Structure):
3130 ('Internal', ctypes.wintypes.LPVOID),
3131 ('InternalHigh', ctypes.wintypes.LPVOID),
3132 ('Offset', ctypes.wintypes.DWORD),
3133 ('OffsetHigh', ctypes.wintypes.DWORD),
3134 ('hEvent', ctypes.wintypes.HANDLE),
3137 kernel32 = ctypes.windll.kernel32
3138 LockFileEx = kernel32.LockFileEx
3139 LockFileEx.argtypes = [
3140 ctypes.wintypes.HANDLE, # hFile
3141 ctypes.wintypes.DWORD, # dwFlags
3142 ctypes.wintypes.DWORD, # dwReserved
3143 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
3144 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
3145 ctypes.POINTER(OVERLAPPED) # Overlapped
3147 LockFileEx.restype = ctypes.wintypes.BOOL
3148 UnlockFileEx = kernel32.UnlockFileEx
3149 UnlockFileEx.argtypes = [
3150 ctypes.wintypes.HANDLE, # hFile
3151 ctypes.wintypes.DWORD, # dwReserved
3152 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
3153 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
3154 ctypes.POINTER(OVERLAPPED) # Overlapped
3156 UnlockFileEx.restype = ctypes.wintypes.BOOL
3157 whole_low = 0xffffffff
3158 whole_high = 0x7fffffff
3160 def _lock_file(f, exclusive):
3161 overlapped = OVERLAPPED()
3162 overlapped.Offset = 0
3163 overlapped.OffsetHigh = 0
3164 overlapped.hEvent = 0
3165 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
3166 handle = msvcrt.get_osfhandle(f.fileno())
3167 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
3168 whole_low, whole_high, f._lock_file_overlapped_p):
3169 raise OSError('Locking file failed: %r' % ctypes.FormatError())
3171 def _unlock_file(f):
3172 assert f._lock_file_overlapped_p
3173 handle = msvcrt.get_osfhandle(f.fileno())
3174 if not UnlockFileEx(handle, 0,
3175 whole_low, whole_high, f._lock_file_overlapped_p):
3176 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
3179 # Some platforms, such as Jython, is missing fcntl
3183 def _lock_file(f, exclusive):
3184 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
3186 def _unlock_file(f):
3187 fcntl.flock(f, fcntl.LOCK_UN)
3189 UNSUPPORTED_MSG = 'file locking is not supported on this platform'
3191 def _lock_file(f, exclusive):
3192 raise IOError(UNSUPPORTED_MSG)
3194 def _unlock_file(f):
3195 raise IOError(UNSUPPORTED_MSG)
3198 class locked_file(object):
3199 def __init__(self, filename, mode, encoding=None):
3200 assert mode in ['r', 'a', 'w']
3201 self.f = io.open(filename, mode, encoding=encoding)
3204 def __enter__(self):
3205 exclusive = self.mode != 'r'
3207 _lock_file(self.f, exclusive)
3213 def __exit__(self, etype, value, traceback):
3215 _unlock_file(self.f)
3222 def write(self, *args):
3223 return self.f.write(*args)
3225 def read(self, *args):
3226 return self.f.read(*args)
3229 def get_filesystem_encoding():
3230 encoding = sys.getfilesystemencoding()
3231 return encoding if encoding is not None else 'utf-8'
3234 def shell_quote(args):
3236 encoding = get_filesystem_encoding()
3238 if isinstance(a, bytes):
3239 # We may get a filename encoded with 'encodeFilename'
3240 a = a.decode(encoding)
3241 quoted_args.append(compat_shlex_quote(a))
3242 return ' '.join(quoted_args)
3245 def smuggle_url(url, data):
3246 """ Pass additional data in a URL for internal use. """
3248 url, idata = unsmuggle_url(url, {})
3250 sdata = compat_urllib_parse_urlencode(
3251 {'__youtubedl_smuggle': json.dumps(data)})
3252 return url + '#' + sdata
3255 def unsmuggle_url(smug_url, default=None):
3256 if '#__youtubedl_smuggle' not in smug_url:
3257 return smug_url, default
3258 url, _, sdata = smug_url.rpartition('#')
3259 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
3260 data = json.loads(jsond)
3264 def format_bytes(bytes):
3267 if type(bytes) is str:
3268 bytes = float(bytes)
3272 exponent = int(math.log(bytes, 1024.0))
3273 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
3274 converted = float(bytes) / float(1024 ** exponent)
3275 return '%.2f%s' % (converted, suffix)
3278 def lookup_unit_table(unit_table, s):
3279 units_re = '|'.join(re.escape(u) for u in unit_table)
3281 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
3284 num_str = m.group('num').replace(',', '.')
3285 mult = unit_table[m.group('unit')]
3286 return int(float(num_str) * mult)
3289 def parse_filesize(s):
3293 # The lower-case forms are of course incorrect and unofficial,
3294 # but we support those too
3311 'megabytes': 1000 ** 2,
3312 'mebibytes': 1024 ** 2,
3318 'gigabytes': 1000 ** 3,
3319 'gibibytes': 1024 ** 3,
3325 'terabytes': 1000 ** 4,
3326 'tebibytes': 1024 ** 4,
3332 'petabytes': 1000 ** 5,
3333 'pebibytes': 1024 ** 5,
3339 'exabytes': 1000 ** 6,
3340 'exbibytes': 1024 ** 6,
3346 'zettabytes': 1000 ** 7,
3347 'zebibytes': 1024 ** 7,
3353 'yottabytes': 1000 ** 8,
3354 'yobibytes': 1024 ** 8,
3357 return lookup_unit_table(_UNIT_TABLE, s)
3366 if re.match(r'^[\d,.]+$', s):
3367 return str_to_int(s)
3378 return lookup_unit_table(_UNIT_TABLE, s)
3381 def parse_resolution(s):
3385 mobj = re.search(r'\b(?P<w>\d+)\s*[xX×]\s*(?P<h>\d+)\b', s)
3388 'width': int(mobj.group('w')),
3389 'height': int(mobj.group('h')),
3392 mobj = re.search(r'\b(\d+)[pPiI]\b', s)
3394 return {'height': int(mobj.group(1))}
3396 mobj = re.search(r'\b([48])[kK]\b', s)
3398 return {'height': int(mobj.group(1)) * 540}
3403 def parse_bitrate(s):
3404 if not isinstance(s, compat_str):
3406 mobj = re.search(r'\b(\d+)\s*kbps', s)
3408 return int(mobj.group(1))
3411 def month_by_name(name, lang='en'):
3412 """ Return the number of a month by (locale-independently) English name """
3414 month_names = MONTH_NAMES.get(lang, MONTH_NAMES['en'])
3417 return month_names.index(name) + 1
3422 def month_by_abbreviation(abbrev):
3423 """ Return the number of a month by (locale-independently) English
3427 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
3432 def fix_xml_ampersands(xml_str):
3433 """Replace all the '&' by '&' in XML"""
3435 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
3440 def setproctitle(title):
3441 assert isinstance(title, compat_str)
3443 # ctypes in Jython is not complete
3444 # http://bugs.jython.org/issue2148
3445 if sys.platform.startswith('java'):
3449 libc = ctypes.cdll.LoadLibrary('libc.so.6')
3453 # LoadLibrary in Windows Python 2.7.13 only expects
3454 # a bytestring, but since unicode_literals turns
3455 # every string into a unicode string, it fails.
3457 title_bytes = title.encode('utf-8')
3458 buf = ctypes.create_string_buffer(len(title_bytes))
3459 buf.value = title_bytes
3461 libc.prctl(15, buf, 0, 0, 0)
3462 except AttributeError:
3463 return # Strange libc, just skip this
3466 def remove_start(s, start):
3467 return s[len(start):] if s is not None and s.startswith(start) else s
3470 def remove_end(s, end):
3471 return s[:-len(end)] if s is not None and s.endswith(end) else s
3474 def remove_quotes(s):
3475 if s is None or len(s) < 2:
3477 for quote in ('"', "'", ):
3478 if s[0] == quote and s[-1] == quote:
3483 def url_basename(url):
3484 path = compat_urlparse.urlparse(url).path
3485 return path.strip('/').split('/')[-1]
3489 return re.match(r'https?://[^?#&]+/', url).group()
3492 def urljoin(base, path):
3493 if isinstance(path, bytes):
3494 path = path.decode('utf-8')
3495 if not isinstance(path, compat_str) or not path:
3497 if re.match(r'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
3499 if isinstance(base, bytes):
3500 base = base.decode('utf-8')
3501 if not isinstance(base, compat_str) or not re.match(
3502 r'^(?:https?:)?//', base):
3504 return compat_urlparse.urljoin(base, path)
3507 class HEADRequest(compat_urllib_request.Request):
3508 def get_method(self):
3512 class PUTRequest(compat_urllib_request.Request):
3513 def get_method(self):
3517 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
3520 v = getattr(v, get_attr, None)
3526 return int(v) * invscale // scale
3527 except (ValueError, TypeError):
3531 def str_or_none(v, default=None):
3532 return default if v is None else compat_str(v)
3535 def str_to_int(int_str):
3536 """ A more relaxed version of int_or_none """
3537 if isinstance(int_str, compat_integer_types):
3539 elif isinstance(int_str, compat_str):
3540 int_str = re.sub(r'[,\.\+]', '', int_str)
3541 return int_or_none(int_str)
3544 def float_or_none(v, scale=1, invscale=1, default=None):
3548 return float(v) * invscale / scale
3549 except (ValueError, TypeError):
3553 def bool_or_none(v, default=None):
3554 return v if isinstance(v, bool) else default
3557 def strip_or_none(v, default=None):
3558 return v.strip() if isinstance(v, compat_str) else default
3561 def url_or_none(url):
3562 if not url or not isinstance(url, compat_str):
3565 return url if re.match(r'^(?:[a-zA-Z][\da-zA-Z.+-]*:)?//', url) else None
3568 def parse_duration(s):
3569 if not isinstance(s, compat_basestring):
3574 days, hours, mins, secs, ms = [None] * 5
3575 m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?Z?$', s)
3577 days, hours, mins, secs, ms = m.groups()
3582 [0-9]+\s*y(?:ears?)?\s*
3585 [0-9]+\s*m(?:onths?)?\s*
3588 [0-9]+\s*w(?:eeks?)?\s*
3591 (?P<days>[0-9]+)\s*d(?:ays?)?\s*
3595 (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
3598 (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
3601 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
3604 days, hours, mins, secs, ms = m.groups()
3606 m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)Z?$', s)
3608 hours, mins = m.groups()
3614 duration += float(secs)
3616 duration += float(mins) * 60
3618 duration += float(hours) * 60 * 60
3620 duration += float(days) * 24 * 60 * 60
3622 duration += float(ms)
3626 def prepend_extension(filename, ext, expected_real_ext=None):
3627 name, real_ext = os.path.splitext(filename)
3629 '{0}.{1}{2}'.format(name, ext, real_ext)
3630 if not expected_real_ext or real_ext[1:] == expected_real_ext
3631 else '{0}.{1}'.format(filename, ext))
3634 def replace_extension(filename, ext, expected_real_ext=None):
3635 name, real_ext = os.path.splitext(filename)
3636 return '{0}.{1}'.format(
3637 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
3641 def check_executable(exe, args=[]):
3642 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
3643 args can be a list of arguments for a short output (like -version) """
3645 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
3651 def get_exe_version(exe, args=['--version'],
3652 version_re=None, unrecognized='present'):
3653 """ Returns the version of the specified executable,
3654 or False if the executable is not present """
3656 # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
3657 # SIGTTOU if youtube-dl is run in the background.
3658 # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
3659 out, _ = subprocess.Popen(
3660 [encodeArgument(exe)] + args,
3661 stdin=subprocess.PIPE,
3662 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
3665 if isinstance(out, bytes): # Python 2.x
3666 out = out.decode('ascii', 'ignore')
3667 return detect_exe_version(out, version_re, unrecognized)
3670 def detect_exe_version(output, version_re=None, unrecognized='present'):
3671 assert isinstance(output, compat_str)
3672 if version_re is None:
3673 version_re = r'version\s+([-0-9._a-zA-Z]+)'
3674 m = re.search(version_re, output)
3681 class PagedList(object):
3683 # This is only useful for tests
3684 return len(self.getslice())
3687 class OnDemandPagedList(PagedList):
3688 def __init__(self, pagefunc, pagesize, use_cache=True):
3689 self._pagefunc = pagefunc
3690 self._pagesize = pagesize
3691 self._use_cache = use_cache
3695 def getslice(self, start=0, end=None):
3697 for pagenum in itertools.count(start // self._pagesize):
3698 firstid = pagenum * self._pagesize
3699 nextfirstid = pagenum * self._pagesize + self._pagesize
3700 if start >= nextfirstid:
3705 page_results = self._cache.get(pagenum)
3706 if page_results is None:
3707 page_results = list(self._pagefunc(pagenum))
3709 self._cache[pagenum] = page_results
3712 start % self._pagesize
3713 if firstid <= start < nextfirstid
3717 ((end - 1) % self._pagesize) + 1
3718 if (end is not None and firstid <= end <= nextfirstid)
3721 if startv != 0 or endv is not None:
3722 page_results = page_results[startv:endv]
3723 res.extend(page_results)
3725 # A little optimization - if current page is not "full", ie. does
3726 # not contain page_size videos then we can assume that this page
3727 # is the last one - there are no more ids on further pages -
3728 # i.e. no need to query again.
3729 if len(page_results) + startv < self._pagesize:
3732 # If we got the whole page, but the next page is not interesting,
3733 # break out early as well
3734 if end == nextfirstid:
3739 class InAdvancePagedList(PagedList):
3740 def __init__(self, pagefunc, pagecount, pagesize):
3741 self._pagefunc = pagefunc
3742 self._pagecount = pagecount
3743 self._pagesize = pagesize
3745 def getslice(self, start=0, end=None):
3747 start_page = start // self._pagesize
3749 self._pagecount if end is None else (end // self._pagesize + 1))
3750 skip_elems = start - start_page * self._pagesize
3751 only_more = None if end is None else end - start
3752 for pagenum in range(start_page, end_page):
3753 page = list(self._pagefunc(pagenum))
3755 page = page[skip_elems:]
3757 if only_more is not None:
3758 if len(page) < only_more:
3759 only_more -= len(page)
3761 page = page[:only_more]
3768 def uppercase_escape(s):
3769 unicode_escape = codecs.getdecoder('unicode_escape')
3771 r'\\U[0-9a-fA-F]{8}',
3772 lambda m: unicode_escape(m.group(0))[0],
3776 def lowercase_escape(s):
3777 unicode_escape = codecs.getdecoder('unicode_escape')
3779 r'\\u[0-9a-fA-F]{4}',
3780 lambda m: unicode_escape(m.group(0))[0],
3784 def escape_rfc3986(s):
3785 """Escape non-ASCII characters as suggested by RFC 3986"""
3786 if sys.version_info < (3, 0) and isinstance(s, compat_str):
3787 s = s.encode('utf-8')
3788 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
3791 def escape_url(url):
3792 """Escape URL as suggested by RFC 3986"""
3793 url_parsed = compat_urllib_parse_urlparse(url)
3794 return url_parsed._replace(
3795 netloc=url_parsed.netloc.encode('idna').decode('ascii'),
3796 path=escape_rfc3986(url_parsed.path),
3797 params=escape_rfc3986(url_parsed.params),
3798 query=escape_rfc3986(url_parsed.query),
3799 fragment=escape_rfc3986(url_parsed.fragment)
3803 def read_batch_urls(batch_fd):
3805 if not isinstance(url, compat_str):
3806 url = url.decode('utf-8', 'replace')
3807 BOM_UTF8 = '\xef\xbb\xbf'
3808 if url.startswith(BOM_UTF8):
3809 url = url[len(BOM_UTF8):]
3811 if url.startswith(('#', ';', ']')):
3815 with contextlib.closing(batch_fd) as fd:
3816 return [url for url in map(fixup, fd) if url]
3819 def urlencode_postdata(*args, **kargs):
3820 return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')
3823 def update_url_query(url, query):
3826 parsed_url = compat_urlparse.urlparse(url)
3827 qs = compat_parse_qs(parsed_url.query)
3829 return compat_urlparse.urlunparse(parsed_url._replace(
3830 query=compat_urllib_parse_urlencode(qs, True)))
3833 def update_Request(req, url=None, data=None, headers={}, query={}):
3834 req_headers = req.headers.copy()
3835 req_headers.update(headers)
3836 req_data = data or req.data
3837 req_url = update_url_query(url or req.get_full_url(), query)
3838 req_get_method = req.get_method()
3839 if req_get_method == 'HEAD':
3840 req_type = HEADRequest
3841 elif req_get_method == 'PUT':
3842 req_type = PUTRequest
3844 req_type = compat_urllib_request.Request
3846 req_url, data=req_data, headers=req_headers,
3847 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
3848 if hasattr(req, 'timeout'):
3849 new_req.timeout = req.timeout
3853 def _multipart_encode_impl(data, boundary):
3854 content_type = 'multipart/form-data; boundary=%s' % boundary
3857 for k, v in data.items():
3858 out += b'--' + boundary.encode('ascii') + b'\r\n'
3859 if isinstance(k, compat_str):
3860 k = k.encode('utf-8')
3861 if isinstance(v, compat_str):
3862 v = v.encode('utf-8')
3863 # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
3864 # suggests sending UTF-8 directly. Firefox sends UTF-8, too
3865 content = b'Content-Disposition: form-data; name="' + k + b'"\r\n\r\n' + v + b'\r\n'
3866 if boundary.encode('ascii') in content:
3867 raise ValueError('Boundary overlaps with data')
3870 out += b'--' + boundary.encode('ascii') + b'--\r\n'
3872 return out, content_type
3875 def multipart_encode(data, boundary=None):
3877 Encode a dict to RFC 7578-compliant form-data
3880 A dict where keys and values can be either Unicode or bytes-like
3883 If specified a Unicode object, it's used as the boundary. Otherwise
3884 a random boundary is generated.
3886 Reference: https://tools.ietf.org/html/rfc7578
3888 has_specified_boundary = boundary is not None
3891 if boundary is None:
3892 boundary = '---------------' + str(random.randrange(0x0fffffff, 0xffffffff))
3895 out, content_type = _multipart_encode_impl(data, boundary)
3898 if has_specified_boundary:
3902 return out, content_type
3905 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
3906 if isinstance(key_or_keys, (list, tuple)):
3907 for key in key_or_keys:
3908 if key not in d or d[key] is None or skip_false_values and not d[key]:
3912 return d.get(key_or_keys, default)
3915 def try_get(src, getter, expected_type=None):
3916 if not isinstance(getter, (list, tuple)):
3921 except (AttributeError, KeyError, TypeError, IndexError):
3924 if expected_type is None or isinstance(v, expected_type):
3928 def merge_dicts(*dicts):
3930 for a_dict in dicts:
3931 for k, v in a_dict.items():
3935 or (isinstance(v, compat_str) and v
3936 and isinstance(merged[k], compat_str)
3937 and not merged[k])):
3942 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
3943 return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
3955 TV_PARENTAL_GUIDELINES = {
3965 def parse_age_limit(s):
3967 return s if 0 <= s <= 21 else None
3968 if not isinstance(s, compat_basestring):
3970 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
3972 return int(m.group('age'))
3974 return US_RATINGS[s]
3975 m = re.match(r'^TV[_-]?(%s)$' % '|'.join(k[3:] for k in TV_PARENTAL_GUIDELINES), s)
3977 return TV_PARENTAL_GUIDELINES['TV-' + m.group(1)]
3981 def strip_jsonp(code):
3984 (?:window\.)?(?P<func_name>[a-zA-Z0-9_.$]*)
3985 (?:\s*&&\s*(?P=func_name))?
3986 \s*\(\s*(?P<callback_data>.*)\);?
3987 \s*?(?://[^\n]*)*$''',
3988 r'\g<callback_data>', code)
3991 def js_to_json(code):
3992 COMMENT_RE = r'/\*(?:(?!\*/).)*?\*/|//[^\n]*'
3993 SKIP_RE = r'\s*(?:{comment})?\s*'.format(comment=COMMENT_RE)
3995 (r'(?s)^(0[xX][0-9a-fA-F]+){skip}:?$'.format(skip=SKIP_RE), 16),
3996 (r'(?s)^(0+[0-7]+){skip}:?$'.format(skip=SKIP_RE), 8),
4001 if v in ('true', 'false', 'null'):
4003 elif v.startswith('/*') or v.startswith('//') or v == ',':
4006 if v[0] in ("'", '"'):
4007 v = re.sub(r'(?s)\\.|"', lambda m: {
4012 }.get(m.group(0), m.group(0)), v[1:-1])
4014 for regex, base in INTEGER_TABLE:
4015 im = re.match(regex, v)
4017 i = int(im.group(1), base)
4018 return '"%d":' % i if v.endswith(':') else '%d' % i
4022 return re.sub(r'''(?sx)
4023 "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
4024 '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
4025 {comment}|,(?={skip}[\]}}])|
4026 (?:(?<![0-9])[eE]|[a-df-zA-DF-Z_])[.a-zA-Z_0-9]*|
4027 \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:{skip}:)?|
4029 '''.format(comment=COMMENT_RE, skip=SKIP_RE), fix_kv, code)
4032 def qualities(quality_ids):
4033 """ Get a numeric quality value out of a list of possible values """
4036 return quality_ids.index(qid)
4042 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
4045 def limit_length(s, length):
4046 """ Add ellipses to overly long strings """
4051 return s[:length - len(ELLIPSES)] + ELLIPSES
4055 def version_tuple(v):
4056 return tuple(int(e) for e in re.split(r'[-.]', v))
4059 def is_outdated_version(version, limit, assume_new=True):
4061 return not assume_new
4063 return version_tuple(version) < version_tuple(limit)
4065 return not assume_new
4068 def ytdl_is_updateable():
4069 """ Returns if youtube-dl can be updated with -U """
4070 from zipimport import zipimporter
4072 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
4075 def args_to_str(args):
4076 # Get a short string representation for a subprocess command
4077 return ' '.join(compat_shlex_quote(a) for a in args)
4080 def error_to_compat_str(err):
4082 # On python 2 error byte string must be decoded with proper
4083 # encoding rather than ascii
4084 if sys.version_info[0] < 3:
4085 err_str = err_str.decode(preferredencoding())
4089 def mimetype2ext(mt):
4095 # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
4096 # it's the most popular one
4097 'audio/mpeg': 'mp3',
4102 _, _, res = mt.rpartition('/')
4103 res = res.split(';')[0].strip().lower()
4107 'smptett+xml': 'tt',
4111 'x-mp4-fragmented': 'mp4',
4112 'x-ms-sami': 'sami',
4115 'x-mpegurl': 'm3u8',
4116 'vnd.apple.mpegurl': 'm3u8',
4120 'vnd.ms-sstr+xml': 'ism',
4126 def parse_codecs(codecs_str):
4127 # http://tools.ietf.org/html/rfc6381
4130 splited_codecs = list(filter(None, map(
4131 lambda str: str.strip(), codecs_str.strip().strip(',').split(','))))
4132 vcodec, acodec = None, None
4133 for full_codec in splited_codecs:
4134 codec = full_codec.split('.')[0]
4135 if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01', 'theora'):
4138 elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
4142 write_string('WARNING: Unknown codec %s\n' % full_codec, sys.stderr)
4143 if not vcodec and not acodec:
4144 if len(splited_codecs) == 2:
4146 'vcodec': splited_codecs[0],
4147 'acodec': splited_codecs[1],
4151 'vcodec': vcodec or 'none',
4152 'acodec': acodec or 'none',
4157 def urlhandle_detect_ext(url_handle):
4158 getheader = url_handle.headers.get
4160 cd = getheader('Content-Disposition')
4162 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
4164 e = determine_ext(m.group('filename'), default_ext=None)
4168 return mimetype2ext(getheader('Content-Type'))
4171 def encode_data_uri(data, mime_type):
4172 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
4175 def age_restricted(content_limit, age_limit):
4176 """ Returns True iff the content should be blocked """
4178 if age_limit is None: # No limit set
4180 if content_limit is None:
4181 return False # Content available for everyone
4182 return age_limit < content_limit
4185 def is_html(first_bytes):
4186 """ Detect whether a file contains HTML by examining its first bytes. """
4189 (b'\xef\xbb\xbf', 'utf-8'),
4190 (b'\x00\x00\xfe\xff', 'utf-32-be'),
4191 (b'\xff\xfe\x00\x00', 'utf-32-le'),
4192 (b'\xff\xfe', 'utf-16-le'),
4193 (b'\xfe\xff', 'utf-16-be'),
4195 for bom, enc in BOMS:
4196 if first_bytes.startswith(bom):
4197 s = first_bytes[len(bom):].decode(enc, 'replace')
4200 s = first_bytes.decode('utf-8', 'replace')
4202 return re.match(r'^\s*<', s)
4205 def determine_protocol(info_dict):
4206 protocol = info_dict.get('protocol')
4207 if protocol is not None:
4210 url = info_dict['url']
4211 if url.startswith('rtmp'):
4213 elif url.startswith('mms'):
4215 elif url.startswith('rtsp'):
4218 ext = determine_ext(url)
4224 return compat_urllib_parse_urlparse(url).scheme
4227 def render_table(header_row, data):
4228 """ Render a list of rows, each as a list of values """
4229 table = [header_row] + data
4230 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
4231 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
4232 return '\n'.join(format_str % tuple(row) for row in table)
4235 def _match_one(filter_part, dct):
4236 COMPARISON_OPERATORS = {
4244 operator_rex = re.compile(r'''(?x)\s*
4246 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
4248 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
4249 (?P<quote>["\'])(?P<quotedstrval>(?:\\.|(?!(?P=quote)|\\).)+?)(?P=quote)|
4250 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
4253 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
4254 m = operator_rex.search(filter_part)
4256 op = COMPARISON_OPERATORS[m.group('op')]
4257 actual_value = dct.get(m.group('key'))
4258 if (m.group('quotedstrval') is not None
4259 or m.group('strval') is not None
4260 # If the original field is a string and matching comparisonvalue is
4261 # a number we should respect the origin of the original field
4262 # and process comparison value as a string (see
4263 # https://github.com/ytdl-org/youtube-dl/issues/11082).
4264 or actual_value is not None and m.group('intval') is not None
4265 and isinstance(actual_value, compat_str)):
4266 if m.group('op') not in ('=', '!='):
4268 'Operator %s does not support string values!' % m.group('op'))
4269 comparison_value = m.group('quotedstrval') or m.group('strval') or m.group('intval')
4270 quote = m.group('quote')
4271 if quote is not None:
4272 comparison_value = comparison_value.replace(r'\%s' % quote, quote)
4275 comparison_value = int(m.group('intval'))
4277 comparison_value = parse_filesize(m.group('intval'))
4278 if comparison_value is None:
4279 comparison_value = parse_filesize(m.group('intval') + 'B')
4280 if comparison_value is None:
4282 'Invalid integer value %r in filter part %r' % (
4283 m.group('intval'), filter_part))
4284 if actual_value is None:
4285 return m.group('none_inclusive')
4286 return op(actual_value, comparison_value)
4289 '': lambda v: (v is True) if isinstance(v, bool) else (v is not None),
4290 '!': lambda v: (v is False) if isinstance(v, bool) else (v is None),
4292 operator_rex = re.compile(r'''(?x)\s*
4293 (?P<op>%s)\s*(?P<key>[a-z_]+)
4295 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
4296 m = operator_rex.search(filter_part)
4298 op = UNARY_OPERATORS[m.group('op')]
4299 actual_value = dct.get(m.group('key'))
4300 return op(actual_value)
4302 raise ValueError('Invalid filter part %r' % filter_part)
4305 def match_str(filter_str, dct):
4306 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
4309 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
4312 def match_filter_func(filter_str):
4313 def _match_func(info_dict):
4314 if match_str(filter_str, info_dict):
4317 video_title = info_dict.get('title', info_dict.get('id', 'video'))
4318 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
4322 def parse_dfxp_time_expr(time_expr):
4326 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
4328 return float(mobj.group('time_offset'))
4330 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
4332 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
4335 def srt_subtitles_timecode(seconds):
4336 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
4339 def dfxp2srt(dfxp_data):
4341 @param dfxp_data A bytes-like object containing DFXP data
4342 @returns A unicode object containing converted SRT data
4344 LEGACY_NAMESPACES = (
4345 (b'http://www.w3.org/ns/ttml', [
4346 b'http://www.w3.org/2004/11/ttaf1',
4347 b'http://www.w3.org/2006/04/ttaf1',
4348 b'http://www.w3.org/2006/10/ttaf1',
4350 (b'http://www.w3.org/ns/ttml#styling', [
4351 b'http://www.w3.org/ns/ttml#style',
4355 SUPPORTED_STYLING = [
4364 _x = functools.partial(xpath_with_ns, ns_map={
4365 'xml': 'http://www.w3.org/XML/1998/namespace',
4366 'ttml': 'http://www.w3.org/ns/ttml',
4367 'tts': 'http://www.w3.org/ns/ttml#styling',
4373 class TTMLPElementParser(object):
4375 _unclosed_elements = []
4376 _applied_styles = []
4378 def start(self, tag, attrib):
4379 if tag in (_x('ttml:br'), 'br'):
4382 unclosed_elements = []
4384 element_style_id = attrib.get('style')
4386 style.update(default_style)
4387 if element_style_id:
4388 style.update(styles.get(element_style_id, {}))
4389 for prop in SUPPORTED_STYLING:
4390 prop_val = attrib.get(_x('tts:' + prop))
4392 style[prop] = prop_val
4395 for k, v in sorted(style.items()):
4396 if self._applied_styles and self._applied_styles[-1].get(k) == v:
4399 font += ' color="%s"' % v
4400 elif k == 'fontSize':
4401 font += ' size="%s"' % v
4402 elif k == 'fontFamily':
4403 font += ' face="%s"' % v
4404 elif k == 'fontWeight' and v == 'bold':
4406 unclosed_elements.append('b')
4407 elif k == 'fontStyle' and v == 'italic':
4409 unclosed_elements.append('i')
4410 elif k == 'textDecoration' and v == 'underline':
4412 unclosed_elements.append('u')
4414 self._out += '<font' + font + '>'
4415 unclosed_elements.append('font')
4417 if self._applied_styles:
4418 applied_style.update(self._applied_styles[-1])
4419 applied_style.update(style)
4420 self._applied_styles.append(applied_style)
4421 self._unclosed_elements.append(unclosed_elements)
4424 if tag not in (_x('ttml:br'), 'br'):
4425 unclosed_elements = self._unclosed_elements.pop()
4426 for element in reversed(unclosed_elements):
4427 self._out += '</%s>' % element
4428 if unclosed_elements and self._applied_styles:
4429 self._applied_styles.pop()
4431 def data(self, data):
4435 return self._out.strip()
4437 def parse_node(node):
4438 target = TTMLPElementParser()
4439 parser = xml.etree.ElementTree.XMLParser(target=target)
4440 parser.feed(xml.etree.ElementTree.tostring(node))
4441 return parser.close()
4443 for k, v in LEGACY_NAMESPACES:
4445 dfxp_data = dfxp_data.replace(ns, k)
4447 dfxp = compat_etree_fromstring(dfxp_data)
4449 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
4452 raise ValueError('Invalid dfxp/TTML subtitle')
4456 for style in dfxp.findall(_x('.//ttml:style')):
4457 style_id = style.get('id') or style.get(_x('xml:id'))
4460 parent_style_id = style.get('style')
4462 if parent_style_id not in styles:
4465 styles[style_id] = styles[parent_style_id].copy()
4466 for prop in SUPPORTED_STYLING:
4467 prop_val = style.get(_x('tts:' + prop))
4469 styles.setdefault(style_id, {})[prop] = prop_val
4475 for p in ('body', 'div'):
4476 ele = xpath_element(dfxp, [_x('.//ttml:' + p), './/' + p])
4479 style = styles.get(ele.get('style'))
4482 default_style.update(style)
4484 for para, index in zip(paras, itertools.count(1)):
4485 begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
4486 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
4487 dur = parse_dfxp_time_expr(para.attrib.get('dur'))
4488 if begin_time is None:
4493 end_time = begin_time + dur
4494 out.append('%d\n%s --> %s\n%s\n\n' % (
4496 srt_subtitles_timecode(begin_time),
4497 srt_subtitles_timecode(end_time),
4503 def cli_option(params, command_option, param):
4504 param = params.get(param)
4506 param = compat_str(param)
4507 return [command_option, param] if param is not None else []
4510 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
4511 param = params.get(param)
4514 assert isinstance(param, bool)
4516 return [command_option + separator + (true_value if param else false_value)]
4517 return [command_option, true_value if param else false_value]
4520 def cli_valueless_option(params, command_option, param, expected_value=True):
4521 param = params.get(param)
4522 return [command_option] if param == expected_value else []
4525 def cli_configuration_args(params, param, default=[]):
4526 ex_args = params.get(param)
4529 assert isinstance(ex_args, list)
4533 class ISO639Utils(object):
4534 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
4593 'iw': 'heb', # Replaced by he in 1989 revision
4603 'in': 'ind', # Replaced by id in 1989 revision
4718 'ji': 'yid', # Replaced by yi in 1989 revision
4726 def short2long(cls, code):
4727 """Convert language code from ISO 639-1 to ISO 639-2/T"""
4728 return cls._lang_map.get(code[:2])
4731 def long2short(cls, code):
4732 """Convert language code from ISO 639-2/T to ISO 639-1"""
4733 for short_name, long_name in cls._lang_map.items():
4734 if long_name == code:
4738 class ISO3166Utils(object):
4739 # From http://data.okfn.org/data/core/country-list
4741 'AF': 'Afghanistan',
4742 'AX': 'Åland Islands',
4745 'AS': 'American Samoa',
4750 'AG': 'Antigua and Barbuda',
4767 'BO': 'Bolivia, Plurinational State of',
4768 'BQ': 'Bonaire, Sint Eustatius and Saba',
4769 'BA': 'Bosnia and Herzegovina',
4771 'BV': 'Bouvet Island',
4773 'IO': 'British Indian Ocean Territory',
4774 'BN': 'Brunei Darussalam',
4776 'BF': 'Burkina Faso',
4782 'KY': 'Cayman Islands',
4783 'CF': 'Central African Republic',
4787 'CX': 'Christmas Island',
4788 'CC': 'Cocos (Keeling) Islands',
4792 'CD': 'Congo, the Democratic Republic of the',
4793 'CK': 'Cook Islands',
4795 'CI': 'Côte d\'Ivoire',
4800 'CZ': 'Czech Republic',
4804 'DO': 'Dominican Republic',
4807 'SV': 'El Salvador',
4808 'GQ': 'Equatorial Guinea',
4812 'FK': 'Falkland Islands (Malvinas)',
4813 'FO': 'Faroe Islands',
4817 'GF': 'French Guiana',
4818 'PF': 'French Polynesia',
4819 'TF': 'French Southern Territories',
4834 'GW': 'Guinea-Bissau',
4837 'HM': 'Heard Island and McDonald Islands',
4838 'VA': 'Holy See (Vatican City State)',
4845 'IR': 'Iran, Islamic Republic of',
4848 'IM': 'Isle of Man',
4858 'KP': 'Korea, Democratic People\'s Republic of',
4859 'KR': 'Korea, Republic of',
4862 'LA': 'Lao People\'s Democratic Republic',
4868 'LI': 'Liechtenstein',
4872 'MK': 'Macedonia, the Former Yugoslav Republic of',
4879 'MH': 'Marshall Islands',
4885 'FM': 'Micronesia, Federated States of',
4886 'MD': 'Moldova, Republic of',
4897 'NL': 'Netherlands',
4898 'NC': 'New Caledonia',
4899 'NZ': 'New Zealand',
4904 'NF': 'Norfolk Island',
4905 'MP': 'Northern Mariana Islands',
4910 'PS': 'Palestine, State of',
4912 'PG': 'Papua New Guinea',
4915 'PH': 'Philippines',
4919 'PR': 'Puerto Rico',
4923 'RU': 'Russian Federation',
4925 'BL': 'Saint Barthélemy',
4926 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
4927 'KN': 'Saint Kitts and Nevis',
4928 'LC': 'Saint Lucia',
4929 'MF': 'Saint Martin (French part)',
4930 'PM': 'Saint Pierre and Miquelon',
4931 'VC': 'Saint Vincent and the Grenadines',
4934 'ST': 'Sao Tome and Principe',
4935 'SA': 'Saudi Arabia',
4939 'SL': 'Sierra Leone',
4941 'SX': 'Sint Maarten (Dutch part)',
4944 'SB': 'Solomon Islands',
4946 'ZA': 'South Africa',
4947 'GS': 'South Georgia and the South Sandwich Islands',
4948 'SS': 'South Sudan',
4953 'SJ': 'Svalbard and Jan Mayen',
4956 'CH': 'Switzerland',
4957 'SY': 'Syrian Arab Republic',
4958 'TW': 'Taiwan, Province of China',
4960 'TZ': 'Tanzania, United Republic of',
4962 'TL': 'Timor-Leste',
4966 'TT': 'Trinidad and Tobago',
4969 'TM': 'Turkmenistan',
4970 'TC': 'Turks and Caicos Islands',
4974 'AE': 'United Arab Emirates',
4975 'GB': 'United Kingdom',
4976 'US': 'United States',
4977 'UM': 'United States Minor Outlying Islands',
4981 'VE': 'Venezuela, Bolivarian Republic of',
4983 'VG': 'Virgin Islands, British',
4984 'VI': 'Virgin Islands, U.S.',
4985 'WF': 'Wallis and Futuna',
4986 'EH': 'Western Sahara',
4993 def short2full(cls, code):
4994 """Convert an ISO 3166-2 country code to the corresponding full name"""
4995 return cls._country_map.get(code.upper())
4998 class GeoUtils(object):
4999 # Major IPv4 address blocks per country
5001 'AD': '46.172.224.0/19',
5002 'AE': '94.200.0.0/13',
5003 'AF': '149.54.0.0/17',
5004 'AG': '209.59.64.0/18',
5005 'AI': '204.14.248.0/21',
5006 'AL': '46.99.0.0/16',
5007 'AM': '46.70.0.0/15',
5008 'AO': '105.168.0.0/13',
5009 'AP': '182.50.184.0/21',
5010 'AQ': '23.154.160.0/24',
5011 'AR': '181.0.0.0/12',
5012 'AS': '202.70.112.0/20',
5013 'AT': '77.116.0.0/14',
5014 'AU': '1.128.0.0/11',
5015 'AW': '181.41.0.0/18',
5016 'AX': '185.217.4.0/22',
5017 'AZ': '5.197.0.0/16',
5018 'BA': '31.176.128.0/17',
5019 'BB': '65.48.128.0/17',
5020 'BD': '114.130.0.0/16',
5022 'BF': '102.178.0.0/15',
5023 'BG': '95.42.0.0/15',
5024 'BH': '37.131.0.0/17',
5025 'BI': '154.117.192.0/18',
5026 'BJ': '137.255.0.0/16',
5027 'BL': '185.212.72.0/23',
5028 'BM': '196.12.64.0/18',
5029 'BN': '156.31.0.0/16',
5030 'BO': '161.56.0.0/16',
5031 'BQ': '161.0.80.0/20',
5032 'BR': '191.128.0.0/12',
5033 'BS': '24.51.64.0/18',
5034 'BT': '119.2.96.0/19',
5035 'BW': '168.167.0.0/16',
5036 'BY': '178.120.0.0/13',
5037 'BZ': '179.42.192.0/18',
5038 'CA': '99.224.0.0/11',
5039 'CD': '41.243.0.0/16',
5040 'CF': '197.242.176.0/21',
5041 'CG': '160.113.0.0/16',
5042 'CH': '85.0.0.0/13',
5043 'CI': '102.136.0.0/14',
5044 'CK': '202.65.32.0/19',
5045 'CL': '152.172.0.0/14',
5046 'CM': '102.244.0.0/14',
5047 'CN': '36.128.0.0/10',
5048 'CO': '181.240.0.0/12',
5049 'CR': '201.192.0.0/12',
5050 'CU': '152.206.0.0/15',
5051 'CV': '165.90.96.0/19',
5052 'CW': '190.88.128.0/17',
5053 'CY': '31.153.0.0/16',
5054 'CZ': '88.100.0.0/14',
5056 'DJ': '197.241.0.0/17',
5057 'DK': '87.48.0.0/12',
5058 'DM': '192.243.48.0/20',
5059 'DO': '152.166.0.0/15',
5060 'DZ': '41.96.0.0/12',
5061 'EC': '186.68.0.0/15',
5062 'EE': '90.190.0.0/15',
5063 'EG': '156.160.0.0/11',
5064 'ER': '196.200.96.0/20',
5065 'ES': '88.0.0.0/11',
5066 'ET': '196.188.0.0/14',
5067 'EU': '2.16.0.0/13',
5068 'FI': '91.152.0.0/13',
5069 'FJ': '144.120.0.0/16',
5070 'FK': '80.73.208.0/21',
5071 'FM': '119.252.112.0/20',
5072 'FO': '88.85.32.0/19',
5074 'GA': '41.158.0.0/15',
5076 'GD': '74.122.88.0/21',
5077 'GE': '31.146.0.0/16',
5078 'GF': '161.22.64.0/18',
5079 'GG': '62.68.160.0/19',
5080 'GH': '154.160.0.0/12',
5081 'GI': '95.164.0.0/16',
5082 'GL': '88.83.0.0/19',
5083 'GM': '160.182.0.0/15',
5084 'GN': '197.149.192.0/18',
5085 'GP': '104.250.0.0/19',
5086 'GQ': '105.235.224.0/20',
5087 'GR': '94.64.0.0/13',
5088 'GT': '168.234.0.0/16',
5089 'GU': '168.123.0.0/16',
5090 'GW': '197.214.80.0/20',
5091 'GY': '181.41.64.0/18',
5092 'HK': '113.252.0.0/14',
5093 'HN': '181.210.0.0/16',
5094 'HR': '93.136.0.0/13',
5095 'HT': '148.102.128.0/17',
5096 'HU': '84.0.0.0/14',
5097 'ID': '39.192.0.0/10',
5098 'IE': '87.32.0.0/12',
5099 'IL': '79.176.0.0/13',
5100 'IM': '5.62.80.0/20',
5101 'IN': '117.192.0.0/10',
5102 'IO': '203.83.48.0/21',
5103 'IQ': '37.236.0.0/14',
5104 'IR': '2.176.0.0/12',
5105 'IS': '82.221.0.0/16',
5106 'IT': '79.0.0.0/10',
5107 'JE': '87.244.64.0/18',
5108 'JM': '72.27.0.0/17',
5109 'JO': '176.29.0.0/16',
5110 'JP': '133.0.0.0/8',
5111 'KE': '105.48.0.0/12',
5112 'KG': '158.181.128.0/17',
5113 'KH': '36.37.128.0/17',
5114 'KI': '103.25.140.0/22',
5115 'KM': '197.255.224.0/20',
5116 'KN': '198.167.192.0/19',
5117 'KP': '175.45.176.0/22',
5118 'KR': '175.192.0.0/10',
5119 'KW': '37.36.0.0/14',
5120 'KY': '64.96.0.0/15',
5121 'KZ': '2.72.0.0/13',
5122 'LA': '115.84.64.0/18',
5123 'LB': '178.135.0.0/16',
5124 'LC': '24.92.144.0/20',
5125 'LI': '82.117.0.0/19',
5126 'LK': '112.134.0.0/15',
5127 'LR': '102.183.0.0/16',
5128 'LS': '129.232.0.0/17',
5129 'LT': '78.56.0.0/13',
5130 'LU': '188.42.0.0/16',
5131 'LV': '46.109.0.0/16',
5132 'LY': '41.252.0.0/14',
5133 'MA': '105.128.0.0/11',
5134 'MC': '88.209.64.0/18',
5135 'MD': '37.246.0.0/16',
5136 'ME': '178.175.0.0/17',
5137 'MF': '74.112.232.0/21',
5138 'MG': '154.126.0.0/17',
5139 'MH': '117.103.88.0/21',
5140 'MK': '77.28.0.0/15',
5141 'ML': '154.118.128.0/18',
5142 'MM': '37.111.0.0/17',
5143 'MN': '49.0.128.0/17',
5144 'MO': '60.246.0.0/16',
5145 'MP': '202.88.64.0/20',
5146 'MQ': '109.203.224.0/19',
5147 'MR': '41.188.64.0/18',
5148 'MS': '208.90.112.0/22',
5149 'MT': '46.11.0.0/16',
5150 'MU': '105.16.0.0/12',
5151 'MV': '27.114.128.0/18',
5152 'MW': '102.70.0.0/15',
5153 'MX': '187.192.0.0/11',
5154 'MY': '175.136.0.0/13',
5155 'MZ': '197.218.0.0/15',
5156 'NA': '41.182.0.0/16',
5157 'NC': '101.101.0.0/18',
5158 'NE': '197.214.0.0/18',
5159 'NF': '203.17.240.0/22',
5160 'NG': '105.112.0.0/12',
5161 'NI': '186.76.0.0/15',
5162 'NL': '145.96.0.0/11',
5163 'NO': '84.208.0.0/13',
5164 'NP': '36.252.0.0/15',
5165 'NR': '203.98.224.0/19',
5166 'NU': '49.156.48.0/22',
5167 'NZ': '49.224.0.0/14',
5168 'OM': '5.36.0.0/15',
5169 'PA': '186.72.0.0/15',
5170 'PE': '186.160.0.0/14',
5171 'PF': '123.50.64.0/18',
5172 'PG': '124.240.192.0/19',
5173 'PH': '49.144.0.0/13',
5174 'PK': '39.32.0.0/11',
5175 'PL': '83.0.0.0/11',
5176 'PM': '70.36.0.0/20',
5177 'PR': '66.50.0.0/16',
5178 'PS': '188.161.0.0/16',
5179 'PT': '85.240.0.0/13',
5180 'PW': '202.124.224.0/20',
5181 'PY': '181.120.0.0/14',
5182 'QA': '37.210.0.0/15',
5183 'RE': '102.35.0.0/16',
5184 'RO': '79.112.0.0/13',
5185 'RS': '93.86.0.0/15',
5186 'RU': '5.136.0.0/13',
5187 'RW': '41.186.0.0/16',
5188 'SA': '188.48.0.0/13',
5189 'SB': '202.1.160.0/19',
5190 'SC': '154.192.0.0/11',
5191 'SD': '102.120.0.0/13',
5192 'SE': '78.64.0.0/12',
5193 'SG': '8.128.0.0/10',
5194 'SI': '188.196.0.0/14',
5195 'SK': '78.98.0.0/15',
5196 'SL': '102.143.0.0/17',
5197 'SM': '89.186.32.0/19',
5198 'SN': '41.82.0.0/15',
5199 'SO': '154.115.192.0/18',
5200 'SR': '186.179.128.0/17',
5201 'SS': '105.235.208.0/21',
5202 'ST': '197.159.160.0/19',
5203 'SV': '168.243.0.0/16',
5204 'SX': '190.102.0.0/20',
5206 'SZ': '41.84.224.0/19',
5207 'TC': '65.255.48.0/20',
5208 'TD': '154.68.128.0/19',
5209 'TG': '196.168.0.0/14',
5210 'TH': '171.96.0.0/13',
5211 'TJ': '85.9.128.0/18',
5212 'TK': '27.96.24.0/21',
5213 'TL': '180.189.160.0/20',
5214 'TM': '95.85.96.0/19',
5215 'TN': '197.0.0.0/11',
5216 'TO': '175.176.144.0/21',
5217 'TR': '78.160.0.0/11',
5218 'TT': '186.44.0.0/15',
5219 'TV': '202.2.96.0/19',
5220 'TW': '120.96.0.0/11',
5221 'TZ': '156.156.0.0/14',
5222 'UA': '37.52.0.0/14',
5223 'UG': '102.80.0.0/13',
5225 'UY': '167.56.0.0/13',
5226 'UZ': '84.54.64.0/18',
5227 'VA': '212.77.0.0/19',
5228 'VC': '207.191.240.0/21',
5229 'VE': '186.88.0.0/13',
5230 'VG': '66.81.192.0/20',
5231 'VI': '146.226.0.0/16',
5232 'VN': '14.160.0.0/11',
5233 'VU': '202.80.32.0/20',
5234 'WF': '117.20.32.0/21',
5235 'WS': '202.4.32.0/19',
5236 'YE': '134.35.0.0/16',
5237 'YT': '41.242.116.0/22',
5238 'ZA': '41.0.0.0/11',
5239 'ZM': '102.144.0.0/13',
5240 'ZW': '102.177.192.0/18',
5244 def random_ipv4(cls, code_or_block):
5245 if len(code_or_block) == 2:
5246 block = cls._country_ip_map.get(code_or_block.upper())
5250 block = code_or_block
5251 addr, preflen = block.split('/')
5252 addr_min = compat_struct_unpack('!L', socket.inet_aton(addr))[0]
5253 addr_max = addr_min | (0xffffffff >> int(preflen))
5254 return compat_str(socket.inet_ntoa(
5255 compat_struct_pack('!L', random.randint(addr_min, addr_max))))
5258 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
5259 def __init__(self, proxies=None):
5260 # Set default handlers
5261 for type in ('http', 'https'):
5262 setattr(self, '%s_open' % type,
5263 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
5264 meth(r, proxy, type))
5265 compat_urllib_request.ProxyHandler.__init__(self, proxies)
5267 def proxy_open(self, req, proxy, type):
5268 req_proxy = req.headers.get('Ytdl-request-proxy')
5269 if req_proxy is not None:
5271 del req.headers['Ytdl-request-proxy']
5273 if proxy == '__noproxy__':
5274 return None # No Proxy
5275 if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
5276 req.add_header('Ytdl-socks-proxy', proxy)
5277 # youtube-dl's http/https handlers do wrapping the socket with socks
5279 return compat_urllib_request.ProxyHandler.proxy_open(
5280 self, req, proxy, type)
5283 # Both long_to_bytes and bytes_to_long are adapted from PyCrypto, which is
5284 # released into Public Domain
5285 # https://github.com/dlitz/pycrypto/blob/master/lib/Crypto/Util/number.py#L387
5287 def long_to_bytes(n, blocksize=0):
5288 """long_to_bytes(n:long, blocksize:int) : string
5289 Convert a long integer to a byte string.
5291 If optional blocksize is given and greater than zero, pad the front of the
5292 byte string with binary zeros so that the length is a multiple of
5295 # after much testing, this algorithm was deemed to be the fastest
5299 s = compat_struct_pack('>I', n & 0xffffffff) + s
5301 # strip off leading zeros
5302 for i in range(len(s)):
5303 if s[i] != b'\000'[0]:
5306 # only happens when n == 0
5310 # add back some pad bytes. this could be done more efficiently w.r.t. the
5311 # de-padding being done above, but sigh...
5312 if blocksize > 0 and len(s) % blocksize:
5313 s = (blocksize - len(s) % blocksize) * b'\000' + s
5317 def bytes_to_long(s):
5318 """bytes_to_long(string) : long
5319 Convert a byte string to a long integer.
5321 This is (essentially) the inverse of long_to_bytes().
5326 extra = (4 - length % 4)
5327 s = b'\000' * extra + s
5328 length = length + extra
5329 for i in range(0, length, 4):
5330 acc = (acc << 32) + compat_struct_unpack('>I', s[i:i + 4])[0]
5334 def ohdave_rsa_encrypt(data, exponent, modulus):
5336 Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
5339 data: data to encrypt, bytes-like object
5340 exponent, modulus: parameter e and N of RSA algorithm, both integer
5341 Output: hex string of encrypted data
5343 Limitation: supports one block encryption only
5346 payload = int(binascii.hexlify(data[::-1]), 16)
5347 encrypted = pow(payload, exponent, modulus)
5348 return '%x' % encrypted
5351 def pkcs1pad(data, length):
5353 Padding input data with PKCS#1 scheme
5355 @param {int[]} data input data
5356 @param {int} length target length
5357 @returns {int[]} padded data
5359 if len(data) > length - 11:
5360 raise ValueError('Input data too long for PKCS#1 padding')
5362 pseudo_random = [random.randint(0, 254) for _ in range(length - len(data) - 3)]
5363 return [0, 2] + pseudo_random + [0] + data
5366 def encode_base_n(num, n, table=None):
5367 FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
5369 table = FULL_TABLE[:n]
5372 raise ValueError('base %d exceeds table length %d' % (n, len(table)))
5379 ret = table[num % n] + ret
5384 def decode_packed_codes(code):
5385 mobj = re.search(PACKED_CODES_RE, code)
5386 obfucasted_code, base, count, symbols = mobj.groups()
5389 symbols = symbols.split('|')
5394 base_n_count = encode_base_n(count, base)
5395 symbol_table[base_n_count] = symbols[count] or base_n_count
5398 r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
5402 def caesar(s, alphabet, shift):
5407 alphabet[(alphabet.index(c) + shift) % l] if c in alphabet else c
5412 return caesar(s, r'''!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''', 47)
5415 def parse_m3u8_attributes(attrib):
5417 for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
5418 if val.startswith('"'):
5424 def urshift(val, n):
5425 return val >> n if val >= 0 else (val + 0x100000000) >> n
5428 # Based on png2str() written by @gdkchan and improved by @yokrysty
5429 # Originally posted at https://github.com/ytdl-org/youtube-dl/issues/9706
5430 def decode_png(png_data):
5431 # Reference: https://www.w3.org/TR/PNG/
5432 header = png_data[8:]
5434 if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
5435 raise IOError('Not a valid PNG file.')
5437 int_map = {1: '>B', 2: '>H', 4: '>I'}
5438 unpack_integer = lambda x: compat_struct_unpack(int_map[len(x)], x)[0]
5443 length = unpack_integer(header[:4])
5446 chunk_type = header[:4]
5449 chunk_data = header[:length]
5450 header = header[length:]
5452 header = header[4:] # Skip CRC
5460 ihdr = chunks[0]['data']
5462 width = unpack_integer(ihdr[:4])
5463 height = unpack_integer(ihdr[4:8])
5467 for chunk in chunks:
5468 if chunk['type'] == b'IDAT':
5469 idat += chunk['data']
5472 raise IOError('Unable to read PNG data.')
5474 decompressed_data = bytearray(zlib.decompress(idat))
5479 def _get_pixel(idx):
5484 for y in range(height):
5485 basePos = y * (1 + stride)
5486 filter_type = decompressed_data[basePos]
5490 pixels.append(current_row)
5492 for x in range(stride):
5493 color = decompressed_data[1 + basePos + x]
5494 basex = y * stride + x
5499 left = _get_pixel(basex - 3)
5501 up = _get_pixel(basex - stride)
5503 if filter_type == 1: # Sub
5504 color = (color + left) & 0xff
5505 elif filter_type == 2: # Up
5506 color = (color + up) & 0xff
5507 elif filter_type == 3: # Average
5508 color = (color + ((left + up) >> 1)) & 0xff
5509 elif filter_type == 4: # Paeth
5515 c = _get_pixel(basex - stride - 3)
5523 if pa <= pb and pa <= pc:
5524 color = (color + a) & 0xff
5526 color = (color + b) & 0xff
5528 color = (color + c) & 0xff
5530 current_row.append(color)
5532 return width, height, pixels
5535 def write_xattr(path, key, value):
5536 # This mess below finds the best xattr tool for the job
5538 # try the pyxattr module...
5541 if hasattr(xattr, 'set'): # pyxattr
5542 # Unicode arguments are not supported in python-pyxattr until
5544 # See https://github.com/ytdl-org/youtube-dl/issues/5498
5545 pyxattr_required_version = '0.5.0'
5546 if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
5547 # TODO: fallback to CLI tools
5548 raise XAttrUnavailableError(
5549 'python-pyxattr is detected but is too old. '
5550 'youtube-dl requires %s or above while your version is %s. '
5551 'Falling back to other xattr implementations' % (
5552 pyxattr_required_version, xattr.__version__))
5554 setxattr = xattr.set
5556 setxattr = xattr.setxattr
5559 setxattr(path, key, value)
5560 except EnvironmentError as e:
5561 raise XAttrMetadataError(e.errno, e.strerror)
5564 if compat_os_name == 'nt':
5565 # Write xattrs to NTFS Alternate Data Streams:
5566 # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
5567 assert ':' not in key
5568 assert os.path.exists(path)
5570 ads_fn = path + ':' + key
5572 with open(ads_fn, 'wb') as f:
5574 except EnvironmentError as e:
5575 raise XAttrMetadataError(e.errno, e.strerror)
5577 user_has_setfattr = check_executable('setfattr', ['--version'])
5578 user_has_xattr = check_executable('xattr', ['-h'])
5580 if user_has_setfattr or user_has_xattr:
5582 value = value.decode('utf-8')
5583 if user_has_setfattr:
5584 executable = 'setfattr'
5585 opts = ['-n', key, '-v', value]
5586 elif user_has_xattr:
5587 executable = 'xattr'
5588 opts = ['-w', key, value]
5590 cmd = ([encodeFilename(executable, True)]
5591 + [encodeArgument(o) for o in opts]
5592 + [encodeFilename(path, True)])
5595 p = subprocess.Popen(
5596 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
5597 except EnvironmentError as e:
5598 raise XAttrMetadataError(e.errno, e.strerror)
5599 stdout, stderr = p.communicate()
5600 stderr = stderr.decode('utf-8', 'replace')
5601 if p.returncode != 0:
5602 raise XAttrMetadataError(p.returncode, stderr)
5605 # On Unix, and can't find pyxattr, setfattr, or xattr.
5606 if sys.platform.startswith('linux'):
5607 raise XAttrUnavailableError(
5608 "Couldn't find a tool to set the xattrs. "
5609 "Install either the python 'pyxattr' or 'xattr' "
5610 "modules, or the GNU 'attr' package "
5611 "(which contains the 'setfattr' tool).")
5613 raise XAttrUnavailableError(
5614 "Couldn't find a tool to set the xattrs. "
5615 "Install either the python 'xattr' module, "
5616 "or the 'xattr' binary.")
5619 def random_birthday(year_field, month_field, day_field):
5620 start_date = datetime.date(1950, 1, 1)
5621 end_date = datetime.date(1995, 12, 31)
5622 offset = random.randint(0, (end_date - start_date).days)
5623 random_date = start_date + datetime.timedelta(offset)
5625 year_field: str(random_date.year),
5626 month_field: str(random_date.month),
5627 day_field: str(random_date.day),
projects / youtube-dl / blob