4 from __future__ import unicode_literals
34 import xml.etree.ElementTree
38 compat_HTMLParseError,
43 compat_ctypes_WINFUNCTYPE,
44 compat_etree_fromstring,
47 compat_html_entities_html5,
59 compat_urllib_parse_urlencode,
60 compat_urllib_parse_urlparse,
61 compat_urllib_parse_unquote_plus,
62 compat_urllib_request,
73 def register_socks_protocols():
74 # "Register" SOCKS protocols
75 # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
76 # URLs with protocols not in urlparse.uses_netloc are not handled correctly
77 for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
78 if scheme not in compat_urlparse.uses_netloc:
79 compat_urlparse.uses_netloc.append(scheme)
82 # This is not clearly defined otherwise
83 compiled_regex_type = type(re.compile(''))
86 def random_user_agent():
87 _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36'
1666 return _USER_AGENT_TPL % random.choice(_CHROME_VERSIONS)
1670 'User-Agent': random_user_agent(),
1671 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
1672 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
1673 'Accept-Encoding': 'gzip, deflate',
1674 'Accept-Language': 'en-us,en;q=0.5',
1679 'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
1683 NO_DEFAULT = object()
1685 ENGLISH_MONTH_NAMES = [
1686 'January', 'February', 'March', 'April', 'May', 'June',
1687 'July', 'August', 'September', 'October', 'November', 'December']
1690 'en': ENGLISH_MONTH_NAMES,
1692 'janvier', 'février', 'mars', 'avril', 'mai', 'juin',
1693 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'],
1696 KNOWN_EXTENSIONS = (
1697 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
1698 'flv', 'f4v', 'f4a', 'f4b',
1699 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
1700 'mkv', 'mka', 'mk3d',
1703 'asf', 'wmv', 'wma',
1709 'f4f', 'f4m', 'm3u8', 'smil')
1711 # needed for sanitizing filenames in restricted mode
1712 ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
1713 itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUY', ['TH', 'ss'],
1714 'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuy', ['th'], 'y')))
1737 '%Y/%m/%d %H:%M:%S',
1739 '%Y-%m-%d %H:%M:%S',
1740 '%Y-%m-%d %H:%M:%S.%f',
1743 '%Y-%m-%dT%H:%M:%SZ',
1744 '%Y-%m-%dT%H:%M:%S.%fZ',
1745 '%Y-%m-%dT%H:%M:%S.%f0Z',
1746 '%Y-%m-%dT%H:%M:%S',
1747 '%Y-%m-%dT%H:%M:%S.%f',
1749 '%b %d %Y at %H:%M',
1750 '%b %d %Y at %H:%M:%S',
1751 '%B %d %Y at %H:%M',
1752 '%B %d %Y at %H:%M:%S',
1755 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
1756 DATE_FORMATS_DAY_FIRST.extend([
1762 '%d/%m/%Y %H:%M:%S',
1765 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
1766 DATE_FORMATS_MONTH_FIRST.extend([
1771 '%m/%d/%Y %H:%M:%S',
1774 PACKED_CODES_RE = r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
1775 JSON_LD_RE = r'(?is)<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>'
1778 def preferredencoding():
1779 """Get preferred encoding.
1781 Returns the best encoding scheme for the system, based on
1782 locale.getpreferredencoding() and some further tweaks.
1785 pref = locale.getpreferredencoding()
1793 def write_json_file(obj, fn):
1794 """ Encode obj as JSON and write it to fn, atomically if possible """
1796 fn = encodeFilename(fn)
1797 if sys.version_info < (3, 0) and sys.platform != 'win32':
1798 encoding = get_filesystem_encoding()
1799 # os.path.basename returns a bytes object, but NamedTemporaryFile
1800 # will fail if the filename contains non ascii characters unless we
1801 # use a unicode object
1802 path_basename = lambda f: os.path.basename(fn).decode(encoding)
1803 # the same for os.path.dirname
1804 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
1806 path_basename = os.path.basename
1807 path_dirname = os.path.dirname
1811 'prefix': path_basename(fn) + '.',
1812 'dir': path_dirname(fn),
1816 # In Python 2.x, json.dump expects a bytestream.
1817 # In Python 3.x, it writes to a character stream
1818 if sys.version_info < (3, 0):
1823 'encoding': 'utf-8',
1826 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
1831 if sys.platform == 'win32':
1832 # Need to remove existing file on Windows, else os.rename raises
1833 # WindowsError or FileExistsError.
1838 os.rename(tf.name, fn)
1847 if sys.version_info >= (2, 7):
1848 def find_xpath_attr(node, xpath, key, val=None):
1849 """ Find the xpath xpath[@key=val] """
1850 assert re.match(r'^[a-zA-Z_-]+$', key)
1851 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
1852 return node.find(expr)
1854 def find_xpath_attr(node, xpath, key, val=None):
1855 for f in node.findall(compat_xpath(xpath)):
1856 if key not in f.attrib:
1858 if val is None or f.attrib.get(key) == val:
1862 # On python2.6 the xml.etree.ElementTree.Element methods don't support
1863 # the namespace parameter
1866 def xpath_with_ns(path, ns_map):
1867 components = [c.split(':') for c in path.split('/')]
1869 for c in components:
1871 replaced.append(c[0])
1874 replaced.append('{%s}%s' % (ns_map[ns], tag))
1875 return '/'.join(replaced)
1878 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
1879 def _find_xpath(xpath):
1880 return node.find(compat_xpath(xpath))
1882 if isinstance(xpath, (str, compat_str)):
1883 n = _find_xpath(xpath)
1891 if default is not NO_DEFAULT:
1894 name = xpath if name is None else name
1895 raise ExtractorError('Could not find XML element %s' % name)
1901 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
1902 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
1903 if n is None or n == default:
1906 if default is not NO_DEFAULT:
1909 name = xpath if name is None else name
1910 raise ExtractorError('Could not find XML element\'s text %s' % name)
1916 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
1917 n = find_xpath_attr(node, xpath, key)
1919 if default is not NO_DEFAULT:
1922 name = '%s[@%s]' % (xpath, key) if name is None else name
1923 raise ExtractorError('Could not find XML attribute %s' % name)
1926 return n.attrib[key]
1929 def get_element_by_id(id, html):
1930 """Return the content of the tag with the specified ID in the passed HTML document"""
1931 return get_element_by_attribute('id', id, html)
1934 def get_element_by_class(class_name, html):
1935 """Return the content of the first tag with the specified class in the passed HTML document"""
1936 retval = get_elements_by_class(class_name, html)
1937 return retval[0] if retval else None
1940 def get_element_by_attribute(attribute, value, html, escape_value=True):
1941 retval = get_elements_by_attribute(attribute, value, html, escape_value)
1942 return retval[0] if retval else None
1945 def get_elements_by_class(class_name, html):
1946 """Return the content of all tags with the specified class in the passed HTML document as a list"""
1947 return get_elements_by_attribute(
1948 'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
1949 html, escape_value=False)
1952 def get_elements_by_attribute(attribute, value, html, escape_value=True):
1953 """Return the content of the tag with the specified attribute in the passed HTML document"""
1955 value = re.escape(value) if escape_value else value
1958 for m in re.finditer(r'''(?xs)
1960 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
1962 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
1966 ''' % (re.escape(attribute), value), html):
1967 res = m.group('content')
1969 if res.startswith('"') or res.startswith("'"):
1972 retlist.append(unescapeHTML(res))
1977 class HTMLAttributeParser(compat_HTMLParser):
1978 """Trivial HTML parser to gather the attributes for a single element"""
1981 compat_HTMLParser.__init__(self)
1983 def handle_starttag(self, tag, attrs):
1984 self.attrs = dict(attrs)
1987 def extract_attributes(html_element):
1988 """Given a string for an HTML element such as
1990 a="foo" B="bar" c="&98;az" d=boz
1991 empty= noval entity="&"
1994 Decode and return a dictionary of attributes.
1996 'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
1997 'empty': '', 'noval': None, 'entity': '&',
1998 'sq': '"', 'dq': '\''
2000 NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
2001 but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
2003 parser = HTMLAttributeParser()
2005 parser.feed(html_element)
2007 # Older Python may throw HTMLParseError in case of malformed HTML
2008 except compat_HTMLParseError:
2013 def clean_html(html):
2014 """Clean an HTML snippet into a readable string"""
2016 if html is None: # Convenience for sanitizing descriptions etc.
2020 html = html.replace('\n', ' ')
2021 html = re.sub(r'(?u)\s*<\s*br\s*/?\s*>\s*', '\n', html)
2022 html = re.sub(r'(?u)<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
2024 html = re.sub('<.*?>', '', html)
2025 # Replace html entities
2026 html = unescapeHTML(html)
2030 def sanitize_open(filename, open_mode):
2031 """Try to open the given filename, and slightly tweak it if this fails.
2033 Attempts to open the given filename. If this fails, it tries to change
2034 the filename slightly, step by step, until it's either able to open it
2035 or it fails and raises a final exception, like the standard open()
2038 It returns the tuple (stream, definitive_file_name).
2042 if sys.platform == 'win32':
2044 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
2045 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
2046 stream = open(encodeFilename(filename), open_mode)
2047 return (stream, filename)
2048 except (IOError, OSError) as err:
2049 if err.errno in (errno.EACCES,):
2052 # In case of error, try to remove win32 forbidden chars
2053 alt_filename = sanitize_path(filename)
2054 if alt_filename == filename:
2057 # An exception here should be caught in the caller
2058 stream = open(encodeFilename(alt_filename), open_mode)
2059 return (stream, alt_filename)
2062 def timeconvert(timestr):
2063 """Convert RFC 2822 defined time string into system timestamp"""
2065 timetuple = email.utils.parsedate_tz(timestr)
2066 if timetuple is not None:
2067 timestamp = email.utils.mktime_tz(timetuple)
2071 def sanitize_filename(s, restricted=False, is_id=False):
2072 """Sanitizes a string so it could be used as part of a filename.
2073 If restricted is set, use a stricter subset of allowed characters.
2074 Set is_id if this is not an arbitrary string, but an ID that should be kept
2077 def replace_insane(char):
2078 if restricted and char in ACCENT_CHARS:
2079 return ACCENT_CHARS[char]
2080 if char == '?' or ord(char) < 32 or ord(char) == 127:
2083 return '' if restricted else '\''
2085 return '_-' if restricted else ' -'
2086 elif char in '\\/|*<>':
2088 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
2090 if restricted and ord(char) > 127:
2095 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
2096 result = ''.join(map(replace_insane, s))
2098 while '__' in result:
2099 result = result.replace('__', '_')
2100 result = result.strip('_')
2101 # Common case of "Foreign band name - English song title"
2102 if restricted and result.startswith('-_'):
2104 if result.startswith('-'):
2105 result = '_' + result[len('-'):]
2106 result = result.lstrip('.')
2112 def sanitize_path(s):
2113 """Sanitizes and normalizes path on Windows"""
2114 if sys.platform != 'win32':
2116 drive_or_unc, _ = os.path.splitdrive(s)
2117 if sys.version_info < (2, 7) and not drive_or_unc:
2118 drive_or_unc, _ = os.path.splitunc(s)
2119 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
2123 path_part if path_part in ['.', '..'] else re.sub(r'(?:[/<>:"\|\\?\*]|[\s.]$)', '#', path_part)
2124 for path_part in norm_path]
2126 sanitized_path.insert(0, drive_or_unc + os.path.sep)
2127 return os.path.join(*sanitized_path)
2130 def sanitize_url(url):
2131 # Prepend protocol-less URLs with `http:` scheme in order to mitigate
2132 # the number of unwanted failures due to missing protocol
2133 if url.startswith('//'):
2134 return 'http:%s' % url
2135 # Fix some common typos seen so far
2137 # https://github.com/ytdl-org/youtube-dl/issues/15649
2138 (r'^httpss://', r'https://'),
2139 # https://bx1.be/lives/direct-tv/
2140 (r'^rmtp([es]?)://', r'rtmp\1://'),
2142 for mistake, fixup in COMMON_TYPOS:
2143 if re.match(mistake, url):
2144 return re.sub(mistake, fixup, url)
2148 def sanitized_Request(url, *args, **kwargs):
2149 return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)
2153 """Expand shell variables and ~"""
2154 return os.path.expandvars(compat_expanduser(s))
2157 def orderedSet(iterable):
2158 """ Remove all duplicates from the input iterable """
2166 def _htmlentity_transform(entity_with_semicolon):
2167 """Transforms an HTML entity to a character."""
2168 entity = entity_with_semicolon[:-1]
2170 # Known non-numeric HTML entity
2171 if entity in compat_html_entities.name2codepoint:
2172 return compat_chr(compat_html_entities.name2codepoint[entity])
2174 # TODO: HTML5 allows entities without a semicolon. For example,
2175 # 'Éric' should be decoded as 'Éric'.
2176 if entity_with_semicolon in compat_html_entities_html5:
2177 return compat_html_entities_html5[entity_with_semicolon]
2179 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
2180 if mobj is not None:
2181 numstr = mobj.group(1)
2182 if numstr.startswith('x'):
2184 numstr = '0%s' % numstr
2187 # See https://github.com/ytdl-org/youtube-dl/issues/7518
2189 return compat_chr(int(numstr, base))
2193 # Unknown entity in name, return its literal representation
2194 return '&%s;' % entity
2197 def unescapeHTML(s):
2200 assert type(s) == compat_str
2203 r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
2206 def get_subprocess_encoding():
2207 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
2208 # For subprocess calls, encode with locale encoding
2209 # Refer to http://stackoverflow.com/a/9951851/35070
2210 encoding = preferredencoding()
2212 encoding = sys.getfilesystemencoding()
2213 if encoding is None:
2218 def encodeFilename(s, for_subprocess=False):
2220 @param s The name of the file
2223 assert type(s) == compat_str
2225 # Python 3 has a Unicode API
2226 if sys.version_info >= (3, 0):
2229 # Pass '' directly to use Unicode APIs on Windows 2000 and up
2230 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
2231 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
2232 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
2235 # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
2236 if sys.platform.startswith('java'):
2239 return s.encode(get_subprocess_encoding(), 'ignore')
2242 def decodeFilename(b, for_subprocess=False):
2244 if sys.version_info >= (3, 0):
2247 if not isinstance(b, bytes):
2250 return b.decode(get_subprocess_encoding(), 'ignore')
2253 def encodeArgument(s):
2254 if not isinstance(s, compat_str):
2255 # Legacy code that uses byte strings
2256 # Uncomment the following line after fixing all post processors
2257 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
2258 s = s.decode('ascii')
2259 return encodeFilename(s, True)
2262 def decodeArgument(b):
2263 return decodeFilename(b, True)
2266 def decodeOption(optval):
2269 if isinstance(optval, bytes):
2270 optval = optval.decode(preferredencoding())
2272 assert isinstance(optval, compat_str)
2276 def formatSeconds(secs):
2278 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
2280 return '%d:%02d' % (secs // 60, secs % 60)
2285 def make_HTTPS_handler(params, **kwargs):
2286 opts_no_check_certificate = params.get('nocheckcertificate', False)
2287 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
2288 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
2289 if opts_no_check_certificate:
2290 context.check_hostname = False
2291 context.verify_mode = ssl.CERT_NONE
2293 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
2296 # (create_default_context present but HTTPSHandler has no context=)
2299 if sys.version_info < (3, 2):
2300 return YoutubeDLHTTPSHandler(params, **kwargs)
2301 else: # Python < 3.4
2302 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
2303 context.verify_mode = (ssl.CERT_NONE
2304 if opts_no_check_certificate
2305 else ssl.CERT_REQUIRED)
2306 context.set_default_verify_paths()
2307 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
2310 def bug_reports_message():
2311 if ytdl_is_updateable():
2312 update_cmd = 'type youtube-dl -U to update'
2314 update_cmd = 'see https://yt-dl.org/update on how to update'
2315 msg = '; please report this issue on https://yt-dl.org/bug .'
2316 msg += ' Make sure you are using the latest version; %s.' % update_cmd
2317 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
2321 class YoutubeDLError(Exception):
2322 """Base exception for YoutubeDL errors."""
2326 class ExtractorError(YoutubeDLError):
2327 """Error during info extraction."""
2329 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
2330 """ tb, if given, is the original traceback (so that it can be printed out).
2331 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
2334 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
2336 if video_id is not None:
2337 msg = video_id + ': ' + msg
2339 msg += ' (caused by %r)' % cause
2341 msg += bug_reports_message()
2342 super(ExtractorError, self).__init__(msg)
2345 self.exc_info = sys.exc_info() # preserve original exception
2347 self.video_id = video_id
2349 def format_traceback(self):
2350 if self.traceback is None:
2352 return ''.join(traceback.format_tb(self.traceback))
2355 class UnsupportedError(ExtractorError):
2356 def __init__(self, url):
2357 super(UnsupportedError, self).__init__(
2358 'Unsupported URL: %s' % url, expected=True)
2362 class RegexNotFoundError(ExtractorError):
2363 """Error when a regex didn't match"""
2367 class GeoRestrictedError(ExtractorError):
2368 """Geographic restriction Error exception.
2370 This exception may be thrown when a video is not available from your
2371 geographic location due to geographic restrictions imposed by a website.
2373 def __init__(self, msg, countries=None):
2374 super(GeoRestrictedError, self).__init__(msg, expected=True)
2376 self.countries = countries
2379 class DownloadError(YoutubeDLError):
2380 """Download Error exception.
2382 This exception may be thrown by FileDownloader objects if they are not
2383 configured to continue on errors. They will contain the appropriate
2387 def __init__(self, msg, exc_info=None):
2388 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
2389 super(DownloadError, self).__init__(msg)
2390 self.exc_info = exc_info
2393 class SameFileError(YoutubeDLError):
2394 """Same File exception.
2396 This exception will be thrown by FileDownloader objects if they detect
2397 multiple files would have to be downloaded to the same file on disk.
2402 class PostProcessingError(YoutubeDLError):
2403 """Post Processing exception.
2405 This exception may be raised by PostProcessor's .run() method to
2406 indicate an error in the postprocessing task.
2409 def __init__(self, msg):
2410 super(PostProcessingError, self).__init__(msg)
2414 class MaxDownloadsReached(YoutubeDLError):
2415 """ --max-downloads limit has been reached. """
2419 class UnavailableVideoError(YoutubeDLError):
2420 """Unavailable Format exception.
2422 This exception will be thrown when a video is requested
2423 in a format that is not available for that video.
2428 class ContentTooShortError(YoutubeDLError):
2429 """Content Too Short exception.
2431 This exception may be raised by FileDownloader objects when a file they
2432 download is too small for what the server announced first, indicating
2433 the connection was probably interrupted.
2436 def __init__(self, downloaded, expected):
2437 super(ContentTooShortError, self).__init__(
2438 'Downloaded {0} bytes, expected {1} bytes'.format(downloaded, expected)
2441 self.downloaded = downloaded
2442 self.expected = expected
2445 class XAttrMetadataError(YoutubeDLError):
2446 def __init__(self, code=None, msg='Unknown error'):
2447 super(XAttrMetadataError, self).__init__(msg)
2451 # Parsing code and msg
2452 if (self.code in (errno.ENOSPC, errno.EDQUOT)
2453 or 'No space left' in self.msg or 'Disk quota excedded' in self.msg):
2454 self.reason = 'NO_SPACE'
2455 elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
2456 self.reason = 'VALUE_TOO_LONG'
2458 self.reason = 'NOT_SUPPORTED'
2461 class XAttrUnavailableError(YoutubeDLError):
2465 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
2466 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
2467 # expected HTTP responses to meet HTTP/1.0 or later (see also
2468 # https://github.com/ytdl-org/youtube-dl/issues/6727)
2469 if sys.version_info < (3, 0):
2470 kwargs['strict'] = True
2471 hc = http_class(*args, **compat_kwargs(kwargs))
2472 source_address = ydl_handler._params.get('source_address')
2474 if source_address is not None:
2475 # This is to workaround _create_connection() from socket where it will try all
2476 # address data from getaddrinfo() including IPv6. This filters the result from
2477 # getaddrinfo() based on the source_address value.
2478 # This is based on the cpython socket.create_connection() function.
2479 # https://github.com/python/cpython/blob/master/Lib/socket.py#L691
2480 def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
2481 host, port = address
2483 addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
2484 af = socket.AF_INET if '.' in source_address[0] else socket.AF_INET6
2485 ip_addrs = [addr for addr in addrs if addr[0] == af]
2486 if addrs and not ip_addrs:
2487 ip_version = 'v4' if af == socket.AF_INET else 'v6'
2489 "No remote IP%s addresses available for connect, can't use '%s' as source address"
2490 % (ip_version, source_address[0]))
2491 for res in ip_addrs:
2492 af, socktype, proto, canonname, sa = res
2495 sock = socket.socket(af, socktype, proto)
2496 if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
2497 sock.settimeout(timeout)
2498 sock.bind(source_address)
2500 err = None # Explicitly break reference cycle
2502 except socket.error as _:
2504 if sock is not None:
2509 raise socket.error('getaddrinfo returns an empty list')
2510 if hasattr(hc, '_create_connection'):
2511 hc._create_connection = _create_connection
2512 sa = (source_address, 0)
2513 if hasattr(hc, 'source_address'): # Python 2.7+
2514 hc.source_address = sa
2516 def _hc_connect(self, *args, **kwargs):
2517 sock = _create_connection(
2518 (self.host, self.port), self.timeout, sa)
2520 self.sock = ssl.wrap_socket(
2521 sock, self.key_file, self.cert_file,
2522 ssl_version=ssl.PROTOCOL_TLSv1)
2525 hc.connect = functools.partial(_hc_connect, hc)
2530 def handle_youtubedl_headers(headers):
2531 filtered_headers = headers
2533 if 'Youtubedl-no-compression' in filtered_headers:
2534 filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
2535 del filtered_headers['Youtubedl-no-compression']
2537 return filtered_headers
2540 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
2541 """Handler for HTTP requests and responses.
2543 This class, when installed with an OpenerDirector, automatically adds
2544 the standard headers to every HTTP request and handles gzipped and
2545 deflated responses from web servers. If compression is to be avoided in
2546 a particular request, the original request in the program code only has
2547 to include the HTTP header "Youtubedl-no-compression", which will be
2548 removed before making the real request.
2550 Part of this code was copied from:
2552 http://techknack.net/python-urllib2-handlers/
2554 Andrew Rowls, the author of that code, agreed to release it to the
2558 def __init__(self, params, *args, **kwargs):
2559 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
2560 self._params = params
2562 def http_open(self, req):
2563 conn_class = compat_http_client.HTTPConnection
2565 socks_proxy = req.headers.get('Ytdl-socks-proxy')
2567 conn_class = make_socks_conn_class(conn_class, socks_proxy)
2568 del req.headers['Ytdl-socks-proxy']
2570 return self.do_open(functools.partial(
2571 _create_http_connection, self, conn_class, False),
2577 return zlib.decompress(data, -zlib.MAX_WBITS)
2579 return zlib.decompress(data)
2581 def http_request(self, req):
2582 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
2583 # always respected by websites, some tend to give out URLs with non percent-encoded
2584 # non-ASCII characters (see telemb.py, ard.py [#3412])
2585 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
2586 # To work around aforementioned issue we will replace request's original URL with
2587 # percent-encoded one
2588 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
2589 # the code of this workaround has been moved here from YoutubeDL.urlopen()
2590 url = req.get_full_url()
2591 url_escaped = escape_url(url)
2593 # Substitute URL if any change after escaping
2594 if url != url_escaped:
2595 req = update_Request(req, url=url_escaped)
2597 for h, v in std_headers.items():
2598 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
2599 # The dict keys are capitalized because of this bug by urllib
2600 if h.capitalize() not in req.headers:
2601 req.add_header(h, v)
2603 req.headers = handle_youtubedl_headers(req.headers)
2605 if sys.version_info < (2, 7) and '#' in req.get_full_url():
2606 # Python 2.6 is brain-dead when it comes to fragments
2607 req._Request__original = req._Request__original.partition('#')[0]
2608 req._Request__r_type = req._Request__r_type.partition('#')[0]
2612 def http_response(self, req, resp):
2615 if resp.headers.get('Content-encoding', '') == 'gzip':
2616 content = resp.read()
2617 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
2619 uncompressed = io.BytesIO(gz.read())
2620 except IOError as original_ioerror:
2621 # There may be junk add the end of the file
2622 # See http://stackoverflow.com/q/4928560/35070 for details
2623 for i in range(1, 1024):
2625 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
2626 uncompressed = io.BytesIO(gz.read())
2631 raise original_ioerror
2632 resp = compat_urllib_request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
2633 resp.msg = old_resp.msg
2634 del resp.headers['Content-encoding']
2636 if resp.headers.get('Content-encoding', '') == 'deflate':
2637 gz = io.BytesIO(self.deflate(resp.read()))
2638 resp = compat_urllib_request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
2639 resp.msg = old_resp.msg
2640 del resp.headers['Content-encoding']
2641 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
2642 # https://github.com/ytdl-org/youtube-dl/issues/6457).
2643 if 300 <= resp.code < 400:
2644 location = resp.headers.get('Location')
2646 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
2647 if sys.version_info >= (3, 0):
2648 location = location.encode('iso-8859-1').decode('utf-8')
2650 location = location.decode('utf-8')
2651 location_escaped = escape_url(location)
2652 if location != location_escaped:
2653 del resp.headers['Location']
2654 if sys.version_info < (3, 0):
2655 location_escaped = location_escaped.encode('utf-8')
2656 resp.headers['Location'] = location_escaped
2659 https_request = http_request
2660 https_response = http_response
2663 def make_socks_conn_class(base_class, socks_proxy):
2664 assert issubclass(base_class, (
2665 compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
2667 url_components = compat_urlparse.urlparse(socks_proxy)
2668 if url_components.scheme.lower() == 'socks5':
2669 socks_type = ProxyType.SOCKS5
2670 elif url_components.scheme.lower() in ('socks', 'socks4'):
2671 socks_type = ProxyType.SOCKS4
2672 elif url_components.scheme.lower() == 'socks4a':
2673 socks_type = ProxyType.SOCKS4A
2675 def unquote_if_non_empty(s):
2678 return compat_urllib_parse_unquote_plus(s)
2682 url_components.hostname, url_components.port or 1080,
2684 unquote_if_non_empty(url_components.username),
2685 unquote_if_non_empty(url_components.password),
2688 class SocksConnection(base_class):
2690 self.sock = sockssocket()
2691 self.sock.setproxy(*proxy_args)
2692 if type(self.timeout) in (int, float):
2693 self.sock.settimeout(self.timeout)
2694 self.sock.connect((self.host, self.port))
2696 if isinstance(self, compat_http_client.HTTPSConnection):
2697 if hasattr(self, '_context'): # Python > 2.6
2698 self.sock = self._context.wrap_socket(
2699 self.sock, server_hostname=self.host)
2701 self.sock = ssl.wrap_socket(self.sock)
2703 return SocksConnection
2706 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
2707 def __init__(self, params, https_conn_class=None, *args, **kwargs):
2708 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
2709 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
2710 self._params = params
2712 def https_open(self, req):
2714 conn_class = self._https_conn_class
2716 if hasattr(self, '_context'): # python > 2.6
2717 kwargs['context'] = self._context
2718 if hasattr(self, '_check_hostname'): # python 3.x
2719 kwargs['check_hostname'] = self._check_hostname
2721 socks_proxy = req.headers.get('Ytdl-socks-proxy')
2723 conn_class = make_socks_conn_class(conn_class, socks_proxy)
2724 del req.headers['Ytdl-socks-proxy']
2726 return self.do_open(functools.partial(
2727 _create_http_connection, self, conn_class, True),
2731 class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
2732 _HTTPONLY_PREFIX = '#HttpOnly_'
2734 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
2735 # Store session cookies with `expires` set to 0 instead of an empty
2738 if cookie.expires is None:
2740 compat_cookiejar.MozillaCookieJar.save(self, filename, ignore_discard, ignore_expires)
2742 def load(self, filename=None, ignore_discard=False, ignore_expires=False):
2743 """Load cookies from a file."""
2744 if filename is None:
2745 if self.filename is not None:
2746 filename = self.filename
2748 raise ValueError(compat_cookiejar.MISSING_FILENAME_TEXT)
2751 with open(filename) as f:
2753 if line.startswith(self._HTTPONLY_PREFIX):
2754 line = line[len(self._HTTPONLY_PREFIX):]
2755 cf.write(compat_str(line))
2757 self._really_load(cf, filename, ignore_discard, ignore_expires)
2758 # Session cookies are denoted by either `expires` field set to
2759 # an empty string or 0. MozillaCookieJar only recognizes the former
2760 # (see [1]). So we need force the latter to be recognized as session
2761 # cookies on our own.
2762 # Session cookies may be important for cookies-based authentication,
2763 # e.g. usually, when user does not check 'Remember me' check box while
2764 # logging in on a site, some important cookies are stored as session
2765 # cookies so that not recognizing them will result in failed login.
2766 # 1. https://bugs.python.org/issue17164
2768 # Treat `expires=0` cookies as session cookies
2769 if cookie.expires == 0:
2770 cookie.expires = None
2771 cookie.discard = True
2774 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
2775 def __init__(self, cookiejar=None):
2776 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
2778 def http_response(self, request, response):
2779 # Python 2 will choke on next HTTP request in row if there are non-ASCII
2780 # characters in Set-Cookie HTTP header of last response (see
2781 # https://github.com/ytdl-org/youtube-dl/issues/6769).
2782 # In order to at least prevent crashing we will percent encode Set-Cookie
2783 # header before HTTPCookieProcessor starts processing it.
2784 # if sys.version_info < (3, 0) and response.headers:
2785 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
2786 # set_cookie = response.headers.get(set_cookie_header)
2788 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
2789 # if set_cookie != set_cookie_escaped:
2790 # del response.headers[set_cookie_header]
2791 # response.headers[set_cookie_header] = set_cookie_escaped
2792 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
2794 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
2795 https_response = http_response
2798 def extract_timezone(date_str):
2800 r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
2803 timezone = datetime.timedelta()
2805 date_str = date_str[:-len(m.group('tz'))]
2806 if not m.group('sign'):
2807 timezone = datetime.timedelta()
2809 sign = 1 if m.group('sign') == '+' else -1
2810 timezone = datetime.timedelta(
2811 hours=sign * int(m.group('hours')),
2812 minutes=sign * int(m.group('minutes')))
2813 return timezone, date_str
2816 def parse_iso8601(date_str, delimiter='T', timezone=None):
2817 """ Return a UNIX timestamp from the given date """
2819 if date_str is None:
2822 date_str = re.sub(r'\.[0-9]+', '', date_str)
2824 if timezone is None:
2825 timezone, date_str = extract_timezone(date_str)
2828 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
2829 dt = datetime.datetime.strptime(date_str, date_format) - timezone
2830 return calendar.timegm(dt.timetuple())
2835 def date_formats(day_first=True):
2836 return DATE_FORMATS_DAY_FIRST if day_first else DATE_FORMATS_MONTH_FIRST
2839 def unified_strdate(date_str, day_first=True):
2840 """Return a string with the date in the format YYYYMMDD"""
2842 if date_str is None:
2846 date_str = date_str.replace(',', ' ')
2847 # Remove AM/PM + timezone
2848 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
2849 _, date_str = extract_timezone(date_str)
2851 for expression in date_formats(day_first):
2853 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
2856 if upload_date is None:
2857 timetuple = email.utils.parsedate_tz(date_str)
2860 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
2863 if upload_date is not None:
2864 return compat_str(upload_date)
2867 def unified_timestamp(date_str, day_first=True):
2868 if date_str is None:
2871 date_str = re.sub(r'[,|]', '', date_str)
2873 pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
2874 timezone, date_str = extract_timezone(date_str)
2876 # Remove AM/PM + timezone
2877 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
2879 # Remove unrecognized timezones from ISO 8601 alike timestamps
2880 m = re.search(r'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str)
2882 date_str = date_str[:-len(m.group('tz'))]
2884 # Python only supports microseconds, so remove nanoseconds
2885 m = re.search(r'^([0-9]{4,}-[0-9]{1,2}-[0-9]{1,2}T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]{6})[0-9]+$', date_str)
2887 date_str = m.group(1)
2889 for expression in date_formats(day_first):
2891 dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
2892 return calendar.timegm(dt.timetuple())
2895 timetuple = email.utils.parsedate_tz(date_str)
2897 return calendar.timegm(timetuple) + pm_delta * 3600
2900 def determine_ext(url, default_ext='unknown_video'):
2901 if url is None or '.' not in url:
2903 guess = url.partition('?')[0].rpartition('.')[2]
2904 if re.match(r'^[A-Za-z0-9]+$', guess):
2906 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
2907 elif guess.rstrip('/') in KNOWN_EXTENSIONS:
2908 return guess.rstrip('/')
2913 def subtitles_filename(filename, sub_lang, sub_format, expected_real_ext=None):
2914 return replace_extension(filename, sub_lang + '.' + sub_format, expected_real_ext)
2917 def date_from_str(date_str):
2919 Return a datetime object from a string in the format YYYYMMDD or
2920 (now|today)[+-][0-9](day|week|month|year)(s)?"""
2921 today = datetime.date.today()
2922 if date_str in ('now', 'today'):
2924 if date_str == 'yesterday':
2925 return today - datetime.timedelta(days=1)
2926 match = re.match(r'(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
2927 if match is not None:
2928 sign = match.group('sign')
2929 time = int(match.group('time'))
2932 unit = match.group('unit')
2933 # A bad approximation?
2937 elif unit == 'year':
2941 delta = datetime.timedelta(**{unit: time})
2942 return today + delta
2943 return datetime.datetime.strptime(date_str, '%Y%m%d').date()
2946 def hyphenate_date(date_str):
2948 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
2949 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
2950 if match is not None:
2951 return '-'.join(match.groups())
2956 class DateRange(object):
2957 """Represents a time interval between two dates"""
2959 def __init__(self, start=None, end=None):
2960 """start and end must be strings in the format accepted by date"""
2961 if start is not None:
2962 self.start = date_from_str(start)
2964 self.start = datetime.datetime.min.date()
2966 self.end = date_from_str(end)
2968 self.end = datetime.datetime.max.date()
2969 if self.start > self.end:
2970 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
2974 """Returns a range that only contains the given day"""
2975 return cls(day, day)
2977 def __contains__(self, date):
2978 """Check if the date is in the range"""
2979 if not isinstance(date, datetime.date):
2980 date = date_from_str(date)
2981 return self.start <= date <= self.end
2984 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
2987 def platform_name():
2988 """ Returns the platform name as a compat_str """
2989 res = platform.platform()
2990 if isinstance(res, bytes):
2991 res = res.decode(preferredencoding())
2993 assert isinstance(res, compat_str)
2997 def _windows_write_string(s, out):
2998 """ Returns True if the string was written using special methods,
2999 False if it has yet to be written out."""
3000 # Adapted from http://stackoverflow.com/a/3259271/35070
3003 import ctypes.wintypes
3011 fileno = out.fileno()
3012 except AttributeError:
3013 # If the output stream doesn't have a fileno, it's virtual
3015 except io.UnsupportedOperation:
3016 # Some strange Windows pseudo files?
3018 if fileno not in WIN_OUTPUT_IDS:
3021 GetStdHandle = compat_ctypes_WINFUNCTYPE(
3022 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
3023 ('GetStdHandle', ctypes.windll.kernel32))
3024 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
3026 WriteConsoleW = compat_ctypes_WINFUNCTYPE(
3027 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
3028 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
3029 ctypes.wintypes.LPVOID)(('WriteConsoleW', ctypes.windll.kernel32))
3030 written = ctypes.wintypes.DWORD(0)
3032 GetFileType = compat_ctypes_WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(('GetFileType', ctypes.windll.kernel32))
3033 FILE_TYPE_CHAR = 0x0002
3034 FILE_TYPE_REMOTE = 0x8000
3035 GetConsoleMode = compat_ctypes_WINFUNCTYPE(
3036 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
3037 ctypes.POINTER(ctypes.wintypes.DWORD))(
3038 ('GetConsoleMode', ctypes.windll.kernel32))
3039 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
3041 def not_a_console(handle):
3042 if handle == INVALID_HANDLE_VALUE or handle is None:
3044 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
3045 or GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
3047 if not_a_console(h):
3050 def next_nonbmp_pos(s):
3052 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
3053 except StopIteration:
3057 count = min(next_nonbmp_pos(s), 1024)
3059 ret = WriteConsoleW(
3060 h, s, count if count else 2, ctypes.byref(written), None)
3062 raise OSError('Failed to write string')
3063 if not count: # We just wrote a non-BMP character
3064 assert written.value == 2
3067 assert written.value > 0
3068 s = s[written.value:]
3072 def write_string(s, out=None, encoding=None):
3075 assert type(s) == compat_str
3077 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
3078 if _windows_write_string(s, out):
3081 if ('b' in getattr(out, 'mode', '')
3082 or sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
3083 byt = s.encode(encoding or preferredencoding(), 'ignore')
3085 elif hasattr(out, 'buffer'):
3086 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
3087 byt = s.encode(enc, 'ignore')
3088 out.buffer.write(byt)
3094 def bytes_to_intlist(bs):
3097 if isinstance(bs[0], int): # Python 3
3100 return [ord(c) for c in bs]
3103 def intlist_to_bytes(xs):
3106 return compat_struct_pack('%dB' % len(xs), *xs)
3109 # Cross-platform file locking
3110 if sys.platform == 'win32':
3111 import ctypes.wintypes
3114 class OVERLAPPED(ctypes.Structure):
3116 ('Internal', ctypes.wintypes.LPVOID),
3117 ('InternalHigh', ctypes.wintypes.LPVOID),
3118 ('Offset', ctypes.wintypes.DWORD),
3119 ('OffsetHigh', ctypes.wintypes.DWORD),
3120 ('hEvent', ctypes.wintypes.HANDLE),
3123 kernel32 = ctypes.windll.kernel32
3124 LockFileEx = kernel32.LockFileEx
3125 LockFileEx.argtypes = [
3126 ctypes.wintypes.HANDLE, # hFile
3127 ctypes.wintypes.DWORD, # dwFlags
3128 ctypes.wintypes.DWORD, # dwReserved
3129 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
3130 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
3131 ctypes.POINTER(OVERLAPPED) # Overlapped
3133 LockFileEx.restype = ctypes.wintypes.BOOL
3134 UnlockFileEx = kernel32.UnlockFileEx
3135 UnlockFileEx.argtypes = [
3136 ctypes.wintypes.HANDLE, # hFile
3137 ctypes.wintypes.DWORD, # dwReserved
3138 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
3139 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
3140 ctypes.POINTER(OVERLAPPED) # Overlapped
3142 UnlockFileEx.restype = ctypes.wintypes.BOOL
3143 whole_low = 0xffffffff
3144 whole_high = 0x7fffffff
3146 def _lock_file(f, exclusive):
3147 overlapped = OVERLAPPED()
3148 overlapped.Offset = 0
3149 overlapped.OffsetHigh = 0
3150 overlapped.hEvent = 0
3151 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
3152 handle = msvcrt.get_osfhandle(f.fileno())
3153 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
3154 whole_low, whole_high, f._lock_file_overlapped_p):
3155 raise OSError('Locking file failed: %r' % ctypes.FormatError())
3157 def _unlock_file(f):
3158 assert f._lock_file_overlapped_p
3159 handle = msvcrt.get_osfhandle(f.fileno())
3160 if not UnlockFileEx(handle, 0,
3161 whole_low, whole_high, f._lock_file_overlapped_p):
3162 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
3165 # Some platforms, such as Jython, is missing fcntl
3169 def _lock_file(f, exclusive):
3170 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
3172 def _unlock_file(f):
3173 fcntl.flock(f, fcntl.LOCK_UN)
3175 UNSUPPORTED_MSG = 'file locking is not supported on this platform'
3177 def _lock_file(f, exclusive):
3178 raise IOError(UNSUPPORTED_MSG)
3180 def _unlock_file(f):
3181 raise IOError(UNSUPPORTED_MSG)
3184 class locked_file(object):
3185 def __init__(self, filename, mode, encoding=None):
3186 assert mode in ['r', 'a', 'w']
3187 self.f = io.open(filename, mode, encoding=encoding)
3190 def __enter__(self):
3191 exclusive = self.mode != 'r'
3193 _lock_file(self.f, exclusive)
3199 def __exit__(self, etype, value, traceback):
3201 _unlock_file(self.f)
3208 def write(self, *args):
3209 return self.f.write(*args)
3211 def read(self, *args):
3212 return self.f.read(*args)
3215 def get_filesystem_encoding():
3216 encoding = sys.getfilesystemencoding()
3217 return encoding if encoding is not None else 'utf-8'
3220 def shell_quote(args):
3222 encoding = get_filesystem_encoding()
3224 if isinstance(a, bytes):
3225 # We may get a filename encoded with 'encodeFilename'
3226 a = a.decode(encoding)
3227 quoted_args.append(compat_shlex_quote(a))
3228 return ' '.join(quoted_args)
3231 def smuggle_url(url, data):
3232 """ Pass additional data in a URL for internal use. """
3234 url, idata = unsmuggle_url(url, {})
3236 sdata = compat_urllib_parse_urlencode(
3237 {'__youtubedl_smuggle': json.dumps(data)})
3238 return url + '#' + sdata
3241 def unsmuggle_url(smug_url, default=None):
3242 if '#__youtubedl_smuggle' not in smug_url:
3243 return smug_url, default
3244 url, _, sdata = smug_url.rpartition('#')
3245 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
3246 data = json.loads(jsond)
3250 def format_bytes(bytes):
3253 if type(bytes) is str:
3254 bytes = float(bytes)
3258 exponent = int(math.log(bytes, 1024.0))
3259 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
3260 converted = float(bytes) / float(1024 ** exponent)
3261 return '%.2f%s' % (converted, suffix)
3264 def lookup_unit_table(unit_table, s):
3265 units_re = '|'.join(re.escape(u) for u in unit_table)
3267 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
3270 num_str = m.group('num').replace(',', '.')
3271 mult = unit_table[m.group('unit')]
3272 return int(float(num_str) * mult)
3275 def parse_filesize(s):
3279 # The lower-case forms are of course incorrect and unofficial,
3280 # but we support those too
3297 'megabytes': 1000 ** 2,
3298 'mebibytes': 1024 ** 2,
3304 'gigabytes': 1000 ** 3,
3305 'gibibytes': 1024 ** 3,
3311 'terabytes': 1000 ** 4,
3312 'tebibytes': 1024 ** 4,
3318 'petabytes': 1000 ** 5,
3319 'pebibytes': 1024 ** 5,
3325 'exabytes': 1000 ** 6,
3326 'exbibytes': 1024 ** 6,
3332 'zettabytes': 1000 ** 7,
3333 'zebibytes': 1024 ** 7,
3339 'yottabytes': 1000 ** 8,
3340 'yobibytes': 1024 ** 8,
3343 return lookup_unit_table(_UNIT_TABLE, s)
3352 if re.match(r'^[\d,.]+$', s):
3353 return str_to_int(s)
3364 return lookup_unit_table(_UNIT_TABLE, s)
3367 def parse_resolution(s):
3371 mobj = re.search(r'\b(?P<w>\d+)\s*[xX×]\s*(?P<h>\d+)\b', s)
3374 'width': int(mobj.group('w')),
3375 'height': int(mobj.group('h')),
3378 mobj = re.search(r'\b(\d+)[pPiI]\b', s)
3380 return {'height': int(mobj.group(1))}
3382 mobj = re.search(r'\b([48])[kK]\b', s)
3384 return {'height': int(mobj.group(1)) * 540}
3389 def parse_bitrate(s):
3390 if not isinstance(s, compat_str):
3392 mobj = re.search(r'\b(\d+)\s*kbps', s)
3394 return int(mobj.group(1))
3397 def month_by_name(name, lang='en'):
3398 """ Return the number of a month by (locale-independently) English name """
3400 month_names = MONTH_NAMES.get(lang, MONTH_NAMES['en'])
3403 return month_names.index(name) + 1
3408 def month_by_abbreviation(abbrev):
3409 """ Return the number of a month by (locale-independently) English
3413 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
3418 def fix_xml_ampersands(xml_str):
3419 """Replace all the '&' by '&' in XML"""
3421 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
3426 def setproctitle(title):
3427 assert isinstance(title, compat_str)
3429 # ctypes in Jython is not complete
3430 # http://bugs.jython.org/issue2148
3431 if sys.platform.startswith('java'):
3435 libc = ctypes.cdll.LoadLibrary('libc.so.6')
3439 # LoadLibrary in Windows Python 2.7.13 only expects
3440 # a bytestring, but since unicode_literals turns
3441 # every string into a unicode string, it fails.
3443 title_bytes = title.encode('utf-8')
3444 buf = ctypes.create_string_buffer(len(title_bytes))
3445 buf.value = title_bytes
3447 libc.prctl(15, buf, 0, 0, 0)
3448 except AttributeError:
3449 return # Strange libc, just skip this
3452 def remove_start(s, start):
3453 return s[len(start):] if s is not None and s.startswith(start) else s
3456 def remove_end(s, end):
3457 return s[:-len(end)] if s is not None and s.endswith(end) else s
3460 def remove_quotes(s):
3461 if s is None or len(s) < 2:
3463 for quote in ('"', "'", ):
3464 if s[0] == quote and s[-1] == quote:
3469 def url_basename(url):
3470 path = compat_urlparse.urlparse(url).path
3471 return path.strip('/').split('/')[-1]
3475 return re.match(r'https?://[^?#&]+/', url).group()
3478 def urljoin(base, path):
3479 if isinstance(path, bytes):
3480 path = path.decode('utf-8')
3481 if not isinstance(path, compat_str) or not path:
3483 if re.match(r'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
3485 if isinstance(base, bytes):
3486 base = base.decode('utf-8')
3487 if not isinstance(base, compat_str) or not re.match(
3488 r'^(?:https?:)?//', base):
3490 return compat_urlparse.urljoin(base, path)
3493 class HEADRequest(compat_urllib_request.Request):
3494 def get_method(self):
3498 class PUTRequest(compat_urllib_request.Request):
3499 def get_method(self):
3503 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
3506 v = getattr(v, get_attr, None)
3512 return int(v) * invscale // scale
3513 except (ValueError, TypeError):
3517 def str_or_none(v, default=None):
3518 return default if v is None else compat_str(v)
3521 def str_to_int(int_str):
3522 """ A more relaxed version of int_or_none """
3523 if isinstance(int_str, compat_integer_types):
3525 elif isinstance(int_str, compat_str):
3526 int_str = re.sub(r'[,\.\+]', '', int_str)
3527 return int_or_none(int_str)
3530 def float_or_none(v, scale=1, invscale=1, default=None):
3534 return float(v) * invscale / scale
3535 except (ValueError, TypeError):
3539 def bool_or_none(v, default=None):
3540 return v if isinstance(v, bool) else default
3543 def strip_or_none(v, default=None):
3544 return v.strip() if isinstance(v, compat_str) else default
3547 def url_or_none(url):
3548 if not url or not isinstance(url, compat_str):
3551 return url if re.match(r'^(?:[a-zA-Z][\da-zA-Z.+-]*:)?//', url) else None
3554 def parse_duration(s):
3555 if not isinstance(s, compat_basestring):
3560 days, hours, mins, secs, ms = [None] * 5
3561 m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?Z?$', s)
3563 days, hours, mins, secs, ms = m.groups()
3568 [0-9]+\s*y(?:ears?)?\s*
3571 [0-9]+\s*m(?:onths?)?\s*
3574 [0-9]+\s*w(?:eeks?)?\s*
3577 (?P<days>[0-9]+)\s*d(?:ays?)?\s*
3581 (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
3584 (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
3587 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
3590 days, hours, mins, secs, ms = m.groups()
3592 m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)Z?$', s)
3594 hours, mins = m.groups()
3600 duration += float(secs)
3602 duration += float(mins) * 60
3604 duration += float(hours) * 60 * 60
3606 duration += float(days) * 24 * 60 * 60
3608 duration += float(ms)
3612 def prepend_extension(filename, ext, expected_real_ext=None):
3613 name, real_ext = os.path.splitext(filename)
3615 '{0}.{1}{2}'.format(name, ext, real_ext)
3616 if not expected_real_ext or real_ext[1:] == expected_real_ext
3617 else '{0}.{1}'.format(filename, ext))
3620 def replace_extension(filename, ext, expected_real_ext=None):
3621 name, real_ext = os.path.splitext(filename)
3622 return '{0}.{1}'.format(
3623 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
3627 def check_executable(exe, args=[]):
3628 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
3629 args can be a list of arguments for a short output (like -version) """
3631 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
3637 def get_exe_version(exe, args=['--version'],
3638 version_re=None, unrecognized='present'):
3639 """ Returns the version of the specified executable,
3640 or False if the executable is not present """
3642 # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
3643 # SIGTTOU if youtube-dl is run in the background.
3644 # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
3645 out, _ = subprocess.Popen(
3646 [encodeArgument(exe)] + args,
3647 stdin=subprocess.PIPE,
3648 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
3651 if isinstance(out, bytes): # Python 2.x
3652 out = out.decode('ascii', 'ignore')
3653 return detect_exe_version(out, version_re, unrecognized)
3656 def detect_exe_version(output, version_re=None, unrecognized='present'):
3657 assert isinstance(output, compat_str)
3658 if version_re is None:
3659 version_re = r'version\s+([-0-9._a-zA-Z]+)'
3660 m = re.search(version_re, output)
3667 class PagedList(object):
3669 # This is only useful for tests
3670 return len(self.getslice())
3673 class OnDemandPagedList(PagedList):
3674 def __init__(self, pagefunc, pagesize, use_cache=True):
3675 self._pagefunc = pagefunc
3676 self._pagesize = pagesize
3677 self._use_cache = use_cache
3681 def getslice(self, start=0, end=None):
3683 for pagenum in itertools.count(start // self._pagesize):
3684 firstid = pagenum * self._pagesize
3685 nextfirstid = pagenum * self._pagesize + self._pagesize
3686 if start >= nextfirstid:
3691 page_results = self._cache.get(pagenum)
3692 if page_results is None:
3693 page_results = list(self._pagefunc(pagenum))
3695 self._cache[pagenum] = page_results
3698 start % self._pagesize
3699 if firstid <= start < nextfirstid
3703 ((end - 1) % self._pagesize) + 1
3704 if (end is not None and firstid <= end <= nextfirstid)
3707 if startv != 0 or endv is not None:
3708 page_results = page_results[startv:endv]
3709 res.extend(page_results)
3711 # A little optimization - if current page is not "full", ie. does
3712 # not contain page_size videos then we can assume that this page
3713 # is the last one - there are no more ids on further pages -
3714 # i.e. no need to query again.
3715 if len(page_results) + startv < self._pagesize:
3718 # If we got the whole page, but the next page is not interesting,
3719 # break out early as well
3720 if end == nextfirstid:
3725 class InAdvancePagedList(PagedList):
3726 def __init__(self, pagefunc, pagecount, pagesize):
3727 self._pagefunc = pagefunc
3728 self._pagecount = pagecount
3729 self._pagesize = pagesize
3731 def getslice(self, start=0, end=None):
3733 start_page = start // self._pagesize
3735 self._pagecount if end is None else (end // self._pagesize + 1))
3736 skip_elems = start - start_page * self._pagesize
3737 only_more = None if end is None else end - start
3738 for pagenum in range(start_page, end_page):
3739 page = list(self._pagefunc(pagenum))
3741 page = page[skip_elems:]
3743 if only_more is not None:
3744 if len(page) < only_more:
3745 only_more -= len(page)
3747 page = page[:only_more]
3754 def uppercase_escape(s):
3755 unicode_escape = codecs.getdecoder('unicode_escape')
3757 r'\\U[0-9a-fA-F]{8}',
3758 lambda m: unicode_escape(m.group(0))[0],
3762 def lowercase_escape(s):
3763 unicode_escape = codecs.getdecoder('unicode_escape')
3765 r'\\u[0-9a-fA-F]{4}',
3766 lambda m: unicode_escape(m.group(0))[0],
3770 def escape_rfc3986(s):
3771 """Escape non-ASCII characters as suggested by RFC 3986"""
3772 if sys.version_info < (3, 0) and isinstance(s, compat_str):
3773 s = s.encode('utf-8')
3774 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
3777 def escape_url(url):
3778 """Escape URL as suggested by RFC 3986"""
3779 url_parsed = compat_urllib_parse_urlparse(url)
3780 return url_parsed._replace(
3781 netloc=url_parsed.netloc.encode('idna').decode('ascii'),
3782 path=escape_rfc3986(url_parsed.path),
3783 params=escape_rfc3986(url_parsed.params),
3784 query=escape_rfc3986(url_parsed.query),
3785 fragment=escape_rfc3986(url_parsed.fragment)
3789 def read_batch_urls(batch_fd):
3791 if not isinstance(url, compat_str):
3792 url = url.decode('utf-8', 'replace')
3793 BOM_UTF8 = '\xef\xbb\xbf'
3794 if url.startswith(BOM_UTF8):
3795 url = url[len(BOM_UTF8):]
3797 if url.startswith(('#', ';', ']')):
3801 with contextlib.closing(batch_fd) as fd:
3802 return [url for url in map(fixup, fd) if url]
3805 def urlencode_postdata(*args, **kargs):
3806 return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')
3809 def update_url_query(url, query):
3812 parsed_url = compat_urlparse.urlparse(url)
3813 qs = compat_parse_qs(parsed_url.query)
3815 return compat_urlparse.urlunparse(parsed_url._replace(
3816 query=compat_urllib_parse_urlencode(qs, True)))
3819 def update_Request(req, url=None, data=None, headers={}, query={}):
3820 req_headers = req.headers.copy()
3821 req_headers.update(headers)
3822 req_data = data or req.data
3823 req_url = update_url_query(url or req.get_full_url(), query)
3824 req_get_method = req.get_method()
3825 if req_get_method == 'HEAD':
3826 req_type = HEADRequest
3827 elif req_get_method == 'PUT':
3828 req_type = PUTRequest
3830 req_type = compat_urllib_request.Request
3832 req_url, data=req_data, headers=req_headers,
3833 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
3834 if hasattr(req, 'timeout'):
3835 new_req.timeout = req.timeout
3839 def _multipart_encode_impl(data, boundary):
3840 content_type = 'multipart/form-data; boundary=%s' % boundary
3843 for k, v in data.items():
3844 out += b'--' + boundary.encode('ascii') + b'\r\n'
3845 if isinstance(k, compat_str):
3846 k = k.encode('utf-8')
3847 if isinstance(v, compat_str):
3848 v = v.encode('utf-8')
3849 # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
3850 # suggests sending UTF-8 directly. Firefox sends UTF-8, too
3851 content = b'Content-Disposition: form-data; name="' + k + b'"\r\n\r\n' + v + b'\r\n'
3852 if boundary.encode('ascii') in content:
3853 raise ValueError('Boundary overlaps with data')
3856 out += b'--' + boundary.encode('ascii') + b'--\r\n'
3858 return out, content_type
3861 def multipart_encode(data, boundary=None):
3863 Encode a dict to RFC 7578-compliant form-data
3866 A dict where keys and values can be either Unicode or bytes-like
3869 If specified a Unicode object, it's used as the boundary. Otherwise
3870 a random boundary is generated.
3872 Reference: https://tools.ietf.org/html/rfc7578
3874 has_specified_boundary = boundary is not None
3877 if boundary is None:
3878 boundary = '---------------' + str(random.randrange(0x0fffffff, 0xffffffff))
3881 out, content_type = _multipart_encode_impl(data, boundary)
3884 if has_specified_boundary:
3888 return out, content_type
3891 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
3892 if isinstance(key_or_keys, (list, tuple)):
3893 for key in key_or_keys:
3894 if key not in d or d[key] is None or skip_false_values and not d[key]:
3898 return d.get(key_or_keys, default)
3901 def try_get(src, getter, expected_type=None):
3902 if not isinstance(getter, (list, tuple)):
3907 except (AttributeError, KeyError, TypeError, IndexError):
3910 if expected_type is None or isinstance(v, expected_type):
3914 def merge_dicts(*dicts):
3916 for a_dict in dicts:
3917 for k, v in a_dict.items():
3921 or (isinstance(v, compat_str) and v
3922 and isinstance(merged[k], compat_str)
3923 and not merged[k])):
3928 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
3929 return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
3941 TV_PARENTAL_GUIDELINES = {
3951 def parse_age_limit(s):
3953 return s if 0 <= s <= 21 else None
3954 if not isinstance(s, compat_basestring):
3956 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
3958 return int(m.group('age'))
3960 return US_RATINGS[s]
3961 m = re.match(r'^TV[_-]?(%s)$' % '|'.join(k[3:] for k in TV_PARENTAL_GUIDELINES), s)
3963 return TV_PARENTAL_GUIDELINES['TV-' + m.group(1)]
3967 def strip_jsonp(code):
3970 (?:window\.)?(?P<func_name>[a-zA-Z0-9_.$]*)
3971 (?:\s*&&\s*(?P=func_name))?
3972 \s*\(\s*(?P<callback_data>.*)\);?
3973 \s*?(?://[^\n]*)*$''',
3974 r'\g<callback_data>', code)
3977 def js_to_json(code):
3978 COMMENT_RE = r'/\*(?:(?!\*/).)*?\*/|//[^\n]*'
3979 SKIP_RE = r'\s*(?:{comment})?\s*'.format(comment=COMMENT_RE)
3981 (r'(?s)^(0[xX][0-9a-fA-F]+){skip}:?$'.format(skip=SKIP_RE), 16),
3982 (r'(?s)^(0+[0-7]+){skip}:?$'.format(skip=SKIP_RE), 8),
3987 if v in ('true', 'false', 'null'):
3989 elif v.startswith('/*') or v.startswith('//') or v == ',':
3992 if v[0] in ("'", '"'):
3993 v = re.sub(r'(?s)\\.|"', lambda m: {
3998 }.get(m.group(0), m.group(0)), v[1:-1])
4000 for regex, base in INTEGER_TABLE:
4001 im = re.match(regex, v)
4003 i = int(im.group(1), base)
4004 return '"%d":' % i if v.endswith(':') else '%d' % i
4008 return re.sub(r'''(?sx)
4009 "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
4010 '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
4011 {comment}|,(?={skip}[\]}}])|
4012 (?:(?<![0-9])[eE]|[a-df-zA-DF-Z_])[.a-zA-Z_0-9]*|
4013 \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:{skip}:)?|
4015 '''.format(comment=COMMENT_RE, skip=SKIP_RE), fix_kv, code)
4018 def qualities(quality_ids):
4019 """ Get a numeric quality value out of a list of possible values """
4022 return quality_ids.index(qid)
4028 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
4031 def limit_length(s, length):
4032 """ Add ellipses to overly long strings """
4037 return s[:length - len(ELLIPSES)] + ELLIPSES
4041 def version_tuple(v):
4042 return tuple(int(e) for e in re.split(r'[-.]', v))
4045 def is_outdated_version(version, limit, assume_new=True):
4047 return not assume_new
4049 return version_tuple(version) < version_tuple(limit)
4051 return not assume_new
4054 def ytdl_is_updateable():
4055 """ Returns if youtube-dl can be updated with -U """
4056 from zipimport import zipimporter
4058 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
4061 def args_to_str(args):
4062 # Get a short string representation for a subprocess command
4063 return ' '.join(compat_shlex_quote(a) for a in args)
4066 def error_to_compat_str(err):
4068 # On python 2 error byte string must be decoded with proper
4069 # encoding rather than ascii
4070 if sys.version_info[0] < 3:
4071 err_str = err_str.decode(preferredencoding())
4075 def mimetype2ext(mt):
4081 # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
4082 # it's the most popular one
4083 'audio/mpeg': 'mp3',
4088 _, _, res = mt.rpartition('/')
4089 res = res.split(';')[0].strip().lower()
4093 'smptett+xml': 'tt',
4097 'x-mp4-fragmented': 'mp4',
4098 'x-ms-sami': 'sami',
4101 'x-mpegurl': 'm3u8',
4102 'vnd.apple.mpegurl': 'm3u8',
4106 'vnd.ms-sstr+xml': 'ism',
4112 def parse_codecs(codecs_str):
4113 # http://tools.ietf.org/html/rfc6381
4116 splited_codecs = list(filter(None, map(
4117 lambda str: str.strip(), codecs_str.strip().strip(',').split(','))))
4118 vcodec, acodec = None, None
4119 for full_codec in splited_codecs:
4120 codec = full_codec.split('.')[0]
4121 if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01', 'theora'):
4124 elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
4128 write_string('WARNING: Unknown codec %s\n' % full_codec, sys.stderr)
4129 if not vcodec and not acodec:
4130 if len(splited_codecs) == 2:
4132 'vcodec': splited_codecs[0],
4133 'acodec': splited_codecs[1],
4137 'vcodec': vcodec or 'none',
4138 'acodec': acodec or 'none',
4143 def urlhandle_detect_ext(url_handle):
4144 getheader = url_handle.headers.get
4146 cd = getheader('Content-Disposition')
4148 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
4150 e = determine_ext(m.group('filename'), default_ext=None)
4154 return mimetype2ext(getheader('Content-Type'))
4157 def encode_data_uri(data, mime_type):
4158 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
4161 def age_restricted(content_limit, age_limit):
4162 """ Returns True iff the content should be blocked """
4164 if age_limit is None: # No limit set
4166 if content_limit is None:
4167 return False # Content available for everyone
4168 return age_limit < content_limit
4171 def is_html(first_bytes):
4172 """ Detect whether a file contains HTML by examining its first bytes. """
4175 (b'\xef\xbb\xbf', 'utf-8'),
4176 (b'\x00\x00\xfe\xff', 'utf-32-be'),
4177 (b'\xff\xfe\x00\x00', 'utf-32-le'),
4178 (b'\xff\xfe', 'utf-16-le'),
4179 (b'\xfe\xff', 'utf-16-be'),
4181 for bom, enc in BOMS:
4182 if first_bytes.startswith(bom):
4183 s = first_bytes[len(bom):].decode(enc, 'replace')
4186 s = first_bytes.decode('utf-8', 'replace')
4188 return re.match(r'^\s*<', s)
4191 def determine_protocol(info_dict):
4192 protocol = info_dict.get('protocol')
4193 if protocol is not None:
4196 url = info_dict['url']
4197 if url.startswith('rtmp'):
4199 elif url.startswith('mms'):
4201 elif url.startswith('rtsp'):
4204 ext = determine_ext(url)
4210 return compat_urllib_parse_urlparse(url).scheme
4213 def render_table(header_row, data):
4214 """ Render a list of rows, each as a list of values """
4215 table = [header_row] + data
4216 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
4217 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
4218 return '\n'.join(format_str % tuple(row) for row in table)
4221 def _match_one(filter_part, dct):
4222 COMPARISON_OPERATORS = {
4230 operator_rex = re.compile(r'''(?x)\s*
4232 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
4234 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
4235 (?P<quote>["\'])(?P<quotedstrval>(?:\\.|(?!(?P=quote)|\\).)+?)(?P=quote)|
4236 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
4239 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
4240 m = operator_rex.search(filter_part)
4242 op = COMPARISON_OPERATORS[m.group('op')]
4243 actual_value = dct.get(m.group('key'))
4244 if (m.group('quotedstrval') is not None
4245 or m.group('strval') is not None
4246 # If the original field is a string and matching comparisonvalue is
4247 # a number we should respect the origin of the original field
4248 # and process comparison value as a string (see
4249 # https://github.com/ytdl-org/youtube-dl/issues/11082).
4250 or actual_value is not None and m.group('intval') is not None
4251 and isinstance(actual_value, compat_str)):
4252 if m.group('op') not in ('=', '!='):
4254 'Operator %s does not support string values!' % m.group('op'))
4255 comparison_value = m.group('quotedstrval') or m.group('strval') or m.group('intval')
4256 quote = m.group('quote')
4257 if quote is not None:
4258 comparison_value = comparison_value.replace(r'\%s' % quote, quote)
4261 comparison_value = int(m.group('intval'))
4263 comparison_value = parse_filesize(m.group('intval'))
4264 if comparison_value is None:
4265 comparison_value = parse_filesize(m.group('intval') + 'B')
4266 if comparison_value is None:
4268 'Invalid integer value %r in filter part %r' % (
4269 m.group('intval'), filter_part))
4270 if actual_value is None:
4271 return m.group('none_inclusive')
4272 return op(actual_value, comparison_value)
4275 '': lambda v: (v is True) if isinstance(v, bool) else (v is not None),
4276 '!': lambda v: (v is False) if isinstance(v, bool) else (v is None),
4278 operator_rex = re.compile(r'''(?x)\s*
4279 (?P<op>%s)\s*(?P<key>[a-z_]+)
4281 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
4282 m = operator_rex.search(filter_part)
4284 op = UNARY_OPERATORS[m.group('op')]
4285 actual_value = dct.get(m.group('key'))
4286 return op(actual_value)
4288 raise ValueError('Invalid filter part %r' % filter_part)
4291 def match_str(filter_str, dct):
4292 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
4295 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
4298 def match_filter_func(filter_str):
4299 def _match_func(info_dict):
4300 if match_str(filter_str, info_dict):
4303 video_title = info_dict.get('title', info_dict.get('id', 'video'))
4304 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
4308 def parse_dfxp_time_expr(time_expr):
4312 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
4314 return float(mobj.group('time_offset'))
4316 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
4318 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
4321 def srt_subtitles_timecode(seconds):
4322 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
4325 def dfxp2srt(dfxp_data):
4327 @param dfxp_data A bytes-like object containing DFXP data
4328 @returns A unicode object containing converted SRT data
4330 LEGACY_NAMESPACES = (
4331 (b'http://www.w3.org/ns/ttml', [
4332 b'http://www.w3.org/2004/11/ttaf1',
4333 b'http://www.w3.org/2006/04/ttaf1',
4334 b'http://www.w3.org/2006/10/ttaf1',
4336 (b'http://www.w3.org/ns/ttml#styling', [
4337 b'http://www.w3.org/ns/ttml#style',
4341 SUPPORTED_STYLING = [
4350 _x = functools.partial(xpath_with_ns, ns_map={
4351 'xml': 'http://www.w3.org/XML/1998/namespace',
4352 'ttml': 'http://www.w3.org/ns/ttml',
4353 'tts': 'http://www.w3.org/ns/ttml#styling',
4359 class TTMLPElementParser(object):
4361 _unclosed_elements = []
4362 _applied_styles = []
4364 def start(self, tag, attrib):
4365 if tag in (_x('ttml:br'), 'br'):
4368 unclosed_elements = []
4370 element_style_id = attrib.get('style')
4372 style.update(default_style)
4373 if element_style_id:
4374 style.update(styles.get(element_style_id, {}))
4375 for prop in SUPPORTED_STYLING:
4376 prop_val = attrib.get(_x('tts:' + prop))
4378 style[prop] = prop_val
4381 for k, v in sorted(style.items()):
4382 if self._applied_styles and self._applied_styles[-1].get(k) == v:
4385 font += ' color="%s"' % v
4386 elif k == 'fontSize':
4387 font += ' size="%s"' % v
4388 elif k == 'fontFamily':
4389 font += ' face="%s"' % v
4390 elif k == 'fontWeight' and v == 'bold':
4392 unclosed_elements.append('b')
4393 elif k == 'fontStyle' and v == 'italic':
4395 unclosed_elements.append('i')
4396 elif k == 'textDecoration' and v == 'underline':
4398 unclosed_elements.append('u')
4400 self._out += '<font' + font + '>'
4401 unclosed_elements.append('font')
4403 if self._applied_styles:
4404 applied_style.update(self._applied_styles[-1])
4405 applied_style.update(style)
4406 self._applied_styles.append(applied_style)
4407 self._unclosed_elements.append(unclosed_elements)
4410 if tag not in (_x('ttml:br'), 'br'):
4411 unclosed_elements = self._unclosed_elements.pop()
4412 for element in reversed(unclosed_elements):
4413 self._out += '</%s>' % element
4414 if unclosed_elements and self._applied_styles:
4415 self._applied_styles.pop()
4417 def data(self, data):
4421 return self._out.strip()
4423 def parse_node(node):
4424 target = TTMLPElementParser()
4425 parser = xml.etree.ElementTree.XMLParser(target=target)
4426 parser.feed(xml.etree.ElementTree.tostring(node))
4427 return parser.close()
4429 for k, v in LEGACY_NAMESPACES:
4431 dfxp_data = dfxp_data.replace(ns, k)
4433 dfxp = compat_etree_fromstring(dfxp_data)
4435 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
4438 raise ValueError('Invalid dfxp/TTML subtitle')
4442 for style in dfxp.findall(_x('.//ttml:style')):
4443 style_id = style.get('id') or style.get(_x('xml:id'))
4446 parent_style_id = style.get('style')
4448 if parent_style_id not in styles:
4451 styles[style_id] = styles[parent_style_id].copy()
4452 for prop in SUPPORTED_STYLING:
4453 prop_val = style.get(_x('tts:' + prop))
4455 styles.setdefault(style_id, {})[prop] = prop_val
4461 for p in ('body', 'div'):
4462 ele = xpath_element(dfxp, [_x('.//ttml:' + p), './/' + p])
4465 style = styles.get(ele.get('style'))
4468 default_style.update(style)
4470 for para, index in zip(paras, itertools.count(1)):
4471 begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
4472 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
4473 dur = parse_dfxp_time_expr(para.attrib.get('dur'))
4474 if begin_time is None:
4479 end_time = begin_time + dur
4480 out.append('%d\n%s --> %s\n%s\n\n' % (
4482 srt_subtitles_timecode(begin_time),
4483 srt_subtitles_timecode(end_time),
4489 def cli_option(params, command_option, param):
4490 param = params.get(param)
4492 param = compat_str(param)
4493 return [command_option, param] if param is not None else []
4496 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
4497 param = params.get(param)
4500 assert isinstance(param, bool)
4502 return [command_option + separator + (true_value if param else false_value)]
4503 return [command_option, true_value if param else false_value]
4506 def cli_valueless_option(params, command_option, param, expected_value=True):
4507 param = params.get(param)
4508 return [command_option] if param == expected_value else []
4511 def cli_configuration_args(params, param, default=[]):
4512 ex_args = params.get(param)
4515 assert isinstance(ex_args, list)
4519 class ISO639Utils(object):
4520 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
4579 'iw': 'heb', # Replaced by he in 1989 revision
4589 'in': 'ind', # Replaced by id in 1989 revision
4704 'ji': 'yid', # Replaced by yi in 1989 revision
4712 def short2long(cls, code):
4713 """Convert language code from ISO 639-1 to ISO 639-2/T"""
4714 return cls._lang_map.get(code[:2])
4717 def long2short(cls, code):
4718 """Convert language code from ISO 639-2/T to ISO 639-1"""
4719 for short_name, long_name in cls._lang_map.items():
4720 if long_name == code:
4724 class ISO3166Utils(object):
4725 # From http://data.okfn.org/data/core/country-list
4727 'AF': 'Afghanistan',
4728 'AX': 'Åland Islands',
4731 'AS': 'American Samoa',
4736 'AG': 'Antigua and Barbuda',
4753 'BO': 'Bolivia, Plurinational State of',
4754 'BQ': 'Bonaire, Sint Eustatius and Saba',
4755 'BA': 'Bosnia and Herzegovina',
4757 'BV': 'Bouvet Island',
4759 'IO': 'British Indian Ocean Territory',
4760 'BN': 'Brunei Darussalam',
4762 'BF': 'Burkina Faso',
4768 'KY': 'Cayman Islands',
4769 'CF': 'Central African Republic',
4773 'CX': 'Christmas Island',
4774 'CC': 'Cocos (Keeling) Islands',
4778 'CD': 'Congo, the Democratic Republic of the',
4779 'CK': 'Cook Islands',
4781 'CI': 'Côte d\'Ivoire',
4786 'CZ': 'Czech Republic',
4790 'DO': 'Dominican Republic',
4793 'SV': 'El Salvador',
4794 'GQ': 'Equatorial Guinea',
4798 'FK': 'Falkland Islands (Malvinas)',
4799 'FO': 'Faroe Islands',
4803 'GF': 'French Guiana',
4804 'PF': 'French Polynesia',
4805 'TF': 'French Southern Territories',
4820 'GW': 'Guinea-Bissau',
4823 'HM': 'Heard Island and McDonald Islands',
4824 'VA': 'Holy See (Vatican City State)',
4831 'IR': 'Iran, Islamic Republic of',
4834 'IM': 'Isle of Man',
4844 'KP': 'Korea, Democratic People\'s Republic of',
4845 'KR': 'Korea, Republic of',
4848 'LA': 'Lao People\'s Democratic Republic',
4854 'LI': 'Liechtenstein',
4858 'MK': 'Macedonia, the Former Yugoslav Republic of',
4865 'MH': 'Marshall Islands',
4871 'FM': 'Micronesia, Federated States of',
4872 'MD': 'Moldova, Republic of',
4883 'NL': 'Netherlands',
4884 'NC': 'New Caledonia',
4885 'NZ': 'New Zealand',
4890 'NF': 'Norfolk Island',
4891 'MP': 'Northern Mariana Islands',
4896 'PS': 'Palestine, State of',
4898 'PG': 'Papua New Guinea',
4901 'PH': 'Philippines',
4905 'PR': 'Puerto Rico',
4909 'RU': 'Russian Federation',
4911 'BL': 'Saint Barthélemy',
4912 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
4913 'KN': 'Saint Kitts and Nevis',
4914 'LC': 'Saint Lucia',
4915 'MF': 'Saint Martin (French part)',
4916 'PM': 'Saint Pierre and Miquelon',
4917 'VC': 'Saint Vincent and the Grenadines',
4920 'ST': 'Sao Tome and Principe',
4921 'SA': 'Saudi Arabia',
4925 'SL': 'Sierra Leone',
4927 'SX': 'Sint Maarten (Dutch part)',
4930 'SB': 'Solomon Islands',
4932 'ZA': 'South Africa',
4933 'GS': 'South Georgia and the South Sandwich Islands',
4934 'SS': 'South Sudan',
4939 'SJ': 'Svalbard and Jan Mayen',
4942 'CH': 'Switzerland',
4943 'SY': 'Syrian Arab Republic',
4944 'TW': 'Taiwan, Province of China',
4946 'TZ': 'Tanzania, United Republic of',
4948 'TL': 'Timor-Leste',
4952 'TT': 'Trinidad and Tobago',
4955 'TM': 'Turkmenistan',
4956 'TC': 'Turks and Caicos Islands',
4960 'AE': 'United Arab Emirates',
4961 'GB': 'United Kingdom',
4962 'US': 'United States',
4963 'UM': 'United States Minor Outlying Islands',
4967 'VE': 'Venezuela, Bolivarian Republic of',
4969 'VG': 'Virgin Islands, British',
4970 'VI': 'Virgin Islands, U.S.',
4971 'WF': 'Wallis and Futuna',
4972 'EH': 'Western Sahara',
4979 def short2full(cls, code):
4980 """Convert an ISO 3166-2 country code to the corresponding full name"""
4981 return cls._country_map.get(code.upper())
4984 class GeoUtils(object):
4985 # Major IPv4 address blocks per country
4987 'AD': '46.172.224.0/19',
4988 'AE': '94.200.0.0/13',
4989 'AF': '149.54.0.0/17',
4990 'AG': '209.59.64.0/18',
4991 'AI': '204.14.248.0/21',
4992 'AL': '46.99.0.0/16',
4993 'AM': '46.70.0.0/15',
4994 'AO': '105.168.0.0/13',
4995 'AP': '182.50.184.0/21',
4996 'AQ': '23.154.160.0/24',
4997 'AR': '181.0.0.0/12',
4998 'AS': '202.70.112.0/20',
4999 'AT': '77.116.0.0/14',
5000 'AU': '1.128.0.0/11',
5001 'AW': '181.41.0.0/18',
5002 'AX': '185.217.4.0/22',
5003 'AZ': '5.197.0.0/16',
5004 'BA': '31.176.128.0/17',
5005 'BB': '65.48.128.0/17',
5006 'BD': '114.130.0.0/16',
5008 'BF': '102.178.0.0/15',
5009 'BG': '95.42.0.0/15',
5010 'BH': '37.131.0.0/17',
5011 'BI': '154.117.192.0/18',
5012 'BJ': '137.255.0.0/16',
5013 'BL': '185.212.72.0/23',
5014 'BM': '196.12.64.0/18',
5015 'BN': '156.31.0.0/16',
5016 'BO': '161.56.0.0/16',
5017 'BQ': '161.0.80.0/20',
5018 'BR': '191.128.0.0/12',
5019 'BS': '24.51.64.0/18',
5020 'BT': '119.2.96.0/19',
5021 'BW': '168.167.0.0/16',
5022 'BY': '178.120.0.0/13',
5023 'BZ': '179.42.192.0/18',
5024 'CA': '99.224.0.0/11',
5025 'CD': '41.243.0.0/16',
5026 'CF': '197.242.176.0/21',
5027 'CG': '160.113.0.0/16',
5028 'CH': '85.0.0.0/13',
5029 'CI': '102.136.0.0/14',
5030 'CK': '202.65.32.0/19',
5031 'CL': '152.172.0.0/14',
5032 'CM': '102.244.0.0/14',
5033 'CN': '36.128.0.0/10',
5034 'CO': '181.240.0.0/12',
5035 'CR': '201.192.0.0/12',
5036 'CU': '152.206.0.0/15',
5037 'CV': '165.90.96.0/19',
5038 'CW': '190.88.128.0/17',
5039 'CY': '31.153.0.0/16',
5040 'CZ': '88.100.0.0/14',
5042 'DJ': '197.241.0.0/17',
5043 'DK': '87.48.0.0/12',
5044 'DM': '192.243.48.0/20',
5045 'DO': '152.166.0.0/15',
5046 'DZ': '41.96.0.0/12',
5047 'EC': '186.68.0.0/15',
5048 'EE': '90.190.0.0/15',
5049 'EG': '156.160.0.0/11',
5050 'ER': '196.200.96.0/20',
5051 'ES': '88.0.0.0/11',
5052 'ET': '196.188.0.0/14',
5053 'EU': '2.16.0.0/13',
5054 'FI': '91.152.0.0/13',
5055 'FJ': '144.120.0.0/16',
5056 'FK': '80.73.208.0/21',
5057 'FM': '119.252.112.0/20',
5058 'FO': '88.85.32.0/19',
5060 'GA': '41.158.0.0/15',
5062 'GD': '74.122.88.0/21',
5063 'GE': '31.146.0.0/16',
5064 'GF': '161.22.64.0/18',
5065 'GG': '62.68.160.0/19',
5066 'GH': '154.160.0.0/12',
5067 'GI': '95.164.0.0/16',
5068 'GL': '88.83.0.0/19',
5069 'GM': '160.182.0.0/15',
5070 'GN': '197.149.192.0/18',
5071 'GP': '104.250.0.0/19',
5072 'GQ': '105.235.224.0/20',
5073 'GR': '94.64.0.0/13',
5074 'GT': '168.234.0.0/16',
5075 'GU': '168.123.0.0/16',
5076 'GW': '197.214.80.0/20',
5077 'GY': '181.41.64.0/18',
5078 'HK': '113.252.0.0/14',
5079 'HN': '181.210.0.0/16',
5080 'HR': '93.136.0.0/13',
5081 'HT': '148.102.128.0/17',
5082 'HU': '84.0.0.0/14',
5083 'ID': '39.192.0.0/10',
5084 'IE': '87.32.0.0/12',
5085 'IL': '79.176.0.0/13',
5086 'IM': '5.62.80.0/20',
5087 'IN': '117.192.0.0/10',
5088 'IO': '203.83.48.0/21',
5089 'IQ': '37.236.0.0/14',
5090 'IR': '2.176.0.0/12',
5091 'IS': '82.221.0.0/16',
5092 'IT': '79.0.0.0/10',
5093 'JE': '87.244.64.0/18',
5094 'JM': '72.27.0.0/17',
5095 'JO': '176.29.0.0/16',
5096 'JP': '133.0.0.0/8',
5097 'KE': '105.48.0.0/12',
5098 'KG': '158.181.128.0/17',
5099 'KH': '36.37.128.0/17',
5100 'KI': '103.25.140.0/22',
5101 'KM': '197.255.224.0/20',
5102 'KN': '198.167.192.0/19',
5103 'KP': '175.45.176.0/22',
5104 'KR': '175.192.0.0/10',
5105 'KW': '37.36.0.0/14',
5106 'KY': '64.96.0.0/15',
5107 'KZ': '2.72.0.0/13',
5108 'LA': '115.84.64.0/18',
5109 'LB': '178.135.0.0/16',
5110 'LC': '24.92.144.0/20',
5111 'LI': '82.117.0.0/19',
5112 'LK': '112.134.0.0/15',
5113 'LR': '102.183.0.0/16',
5114 'LS': '129.232.0.0/17',
5115 'LT': '78.56.0.0/13',
5116 'LU': '188.42.0.0/16',
5117 'LV': '46.109.0.0/16',
5118 'LY': '41.252.0.0/14',
5119 'MA': '105.128.0.0/11',
5120 'MC': '88.209.64.0/18',
5121 'MD': '37.246.0.0/16',
5122 'ME': '178.175.0.0/17',
5123 'MF': '74.112.232.0/21',
5124 'MG': '154.126.0.0/17',
5125 'MH': '117.103.88.0/21',
5126 'MK': '77.28.0.0/15',
5127 'ML': '154.118.128.0/18',
5128 'MM': '37.111.0.0/17',
5129 'MN': '49.0.128.0/17',
5130 'MO': '60.246.0.0/16',
5131 'MP': '202.88.64.0/20',
5132 'MQ': '109.203.224.0/19',
5133 'MR': '41.188.64.0/18',
5134 'MS': '208.90.112.0/22',
5135 'MT': '46.11.0.0/16',
5136 'MU': '105.16.0.0/12',
5137 'MV': '27.114.128.0/18',
5138 'MW': '102.70.0.0/15',
5139 'MX': '187.192.0.0/11',
5140 'MY': '175.136.0.0/13',
5141 'MZ': '197.218.0.0/15',
5142 'NA': '41.182.0.0/16',
5143 'NC': '101.101.0.0/18',
5144 'NE': '197.214.0.0/18',
5145 'NF': '203.17.240.0/22',
5146 'NG': '105.112.0.0/12',
5147 'NI': '186.76.0.0/15',
5148 'NL': '145.96.0.0/11',
5149 'NO': '84.208.0.0/13',
5150 'NP': '36.252.0.0/15',
5151 'NR': '203.98.224.0/19',
5152 'NU': '49.156.48.0/22',
5153 'NZ': '49.224.0.0/14',
5154 'OM': '5.36.0.0/15',
5155 'PA': '186.72.0.0/15',
5156 'PE': '186.160.0.0/14',
5157 'PF': '123.50.64.0/18',
5158 'PG': '124.240.192.0/19',
5159 'PH': '49.144.0.0/13',
5160 'PK': '39.32.0.0/11',
5161 'PL': '83.0.0.0/11',
5162 'PM': '70.36.0.0/20',
5163 'PR': '66.50.0.0/16',
5164 'PS': '188.161.0.0/16',
5165 'PT': '85.240.0.0/13',
5166 'PW': '202.124.224.0/20',
5167 'PY': '181.120.0.0/14',
5168 'QA': '37.210.0.0/15',
5169 'RE': '102.35.0.0/16',
5170 'RO': '79.112.0.0/13',
5171 'RS': '93.86.0.0/15',
5172 'RU': '5.136.0.0/13',
5173 'RW': '41.186.0.0/16',
5174 'SA': '188.48.0.0/13',
5175 'SB': '202.1.160.0/19',
5176 'SC': '154.192.0.0/11',
5177 'SD': '102.120.0.0/13',
5178 'SE': '78.64.0.0/12',
5179 'SG': '8.128.0.0/10',
5180 'SI': '188.196.0.0/14',
5181 'SK': '78.98.0.0/15',
5182 'SL': '102.143.0.0/17',
5183 'SM': '89.186.32.0/19',
5184 'SN': '41.82.0.0/15',
5185 'SO': '154.115.192.0/18',
5186 'SR': '186.179.128.0/17',
5187 'SS': '105.235.208.0/21',
5188 'ST': '197.159.160.0/19',
5189 'SV': '168.243.0.0/16',
5190 'SX': '190.102.0.0/20',
5192 'SZ': '41.84.224.0/19',
5193 'TC': '65.255.48.0/20',
5194 'TD': '154.68.128.0/19',
5195 'TG': '196.168.0.0/14',
5196 'TH': '171.96.0.0/13',
5197 'TJ': '85.9.128.0/18',
5198 'TK': '27.96.24.0/21',
5199 'TL': '180.189.160.0/20',
5200 'TM': '95.85.96.0/19',
5201 'TN': '197.0.0.0/11',
5202 'TO': '175.176.144.0/21',
5203 'TR': '78.160.0.0/11',
5204 'TT': '186.44.0.0/15',
5205 'TV': '202.2.96.0/19',
5206 'TW': '120.96.0.0/11',
5207 'TZ': '156.156.0.0/14',
5208 'UA': '37.52.0.0/14',
5209 'UG': '102.80.0.0/13',
5211 'UY': '167.56.0.0/13',
5212 'UZ': '84.54.64.0/18',
5213 'VA': '212.77.0.0/19',
5214 'VC': '207.191.240.0/21',
5215 'VE': '186.88.0.0/13',
5216 'VG': '66.81.192.0/20',
5217 'VI': '146.226.0.0/16',
5218 'VN': '14.160.0.0/11',
5219 'VU': '202.80.32.0/20',
5220 'WF': '117.20.32.0/21',
5221 'WS': '202.4.32.0/19',
5222 'YE': '134.35.0.0/16',
5223 'YT': '41.242.116.0/22',
5224 'ZA': '41.0.0.0/11',
5225 'ZM': '102.144.0.0/13',
5226 'ZW': '102.177.192.0/18',
5230 def random_ipv4(cls, code_or_block):
5231 if len(code_or_block) == 2:
5232 block = cls._country_ip_map.get(code_or_block.upper())
5236 block = code_or_block
5237 addr, preflen = block.split('/')
5238 addr_min = compat_struct_unpack('!L', socket.inet_aton(addr))[0]
5239 addr_max = addr_min | (0xffffffff >> int(preflen))
5240 return compat_str(socket.inet_ntoa(
5241 compat_struct_pack('!L', random.randint(addr_min, addr_max))))
5244 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
5245 def __init__(self, proxies=None):
5246 # Set default handlers
5247 for type in ('http', 'https'):
5248 setattr(self, '%s_open' % type,
5249 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
5250 meth(r, proxy, type))
5251 compat_urllib_request.ProxyHandler.__init__(self, proxies)
5253 def proxy_open(self, req, proxy, type):
5254 req_proxy = req.headers.get('Ytdl-request-proxy')
5255 if req_proxy is not None:
5257 del req.headers['Ytdl-request-proxy']
5259 if proxy == '__noproxy__':
5260 return None # No Proxy
5261 if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
5262 req.add_header('Ytdl-socks-proxy', proxy)
5263 # youtube-dl's http/https handlers do wrapping the socket with socks
5265 return compat_urllib_request.ProxyHandler.proxy_open(
5266 self, req, proxy, type)
5269 # Both long_to_bytes and bytes_to_long are adapted from PyCrypto, which is
5270 # released into Public Domain
5271 # https://github.com/dlitz/pycrypto/blob/master/lib/Crypto/Util/number.py#L387
5273 def long_to_bytes(n, blocksize=0):
5274 """long_to_bytes(n:long, blocksize:int) : string
5275 Convert a long integer to a byte string.
5277 If optional blocksize is given and greater than zero, pad the front of the
5278 byte string with binary zeros so that the length is a multiple of
5281 # after much testing, this algorithm was deemed to be the fastest
5285 s = compat_struct_pack('>I', n & 0xffffffff) + s
5287 # strip off leading zeros
5288 for i in range(len(s)):
5289 if s[i] != b'\000'[0]:
5292 # only happens when n == 0
5296 # add back some pad bytes. this could be done more efficiently w.r.t. the
5297 # de-padding being done above, but sigh...
5298 if blocksize > 0 and len(s) % blocksize:
5299 s = (blocksize - len(s) % blocksize) * b'\000' + s
5303 def bytes_to_long(s):
5304 """bytes_to_long(string) : long
5305 Convert a byte string to a long integer.
5307 This is (essentially) the inverse of long_to_bytes().
5312 extra = (4 - length % 4)
5313 s = b'\000' * extra + s
5314 length = length + extra
5315 for i in range(0, length, 4):
5316 acc = (acc << 32) + compat_struct_unpack('>I', s[i:i + 4])[0]
5320 def ohdave_rsa_encrypt(data, exponent, modulus):
5322 Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
5325 data: data to encrypt, bytes-like object
5326 exponent, modulus: parameter e and N of RSA algorithm, both integer
5327 Output: hex string of encrypted data
5329 Limitation: supports one block encryption only
5332 payload = int(binascii.hexlify(data[::-1]), 16)
5333 encrypted = pow(payload, exponent, modulus)
5334 return '%x' % encrypted
5337 def pkcs1pad(data, length):
5339 Padding input data with PKCS#1 scheme
5341 @param {int[]} data input data
5342 @param {int} length target length
5343 @returns {int[]} padded data
5345 if len(data) > length - 11:
5346 raise ValueError('Input data too long for PKCS#1 padding')
5348 pseudo_random = [random.randint(0, 254) for _ in range(length - len(data) - 3)]
5349 return [0, 2] + pseudo_random + [0] + data
5352 def encode_base_n(num, n, table=None):
5353 FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
5355 table = FULL_TABLE[:n]
5358 raise ValueError('base %d exceeds table length %d' % (n, len(table)))
5365 ret = table[num % n] + ret
5370 def decode_packed_codes(code):
5371 mobj = re.search(PACKED_CODES_RE, code)
5372 obfucasted_code, base, count, symbols = mobj.groups()
5375 symbols = symbols.split('|')
5380 base_n_count = encode_base_n(count, base)
5381 symbol_table[base_n_count] = symbols[count] or base_n_count
5384 r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
5388 def caesar(s, alphabet, shift):
5393 alphabet[(alphabet.index(c) + shift) % l] if c in alphabet else c
5398 return caesar(s, r'''!"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~''', 47)
5401 def parse_m3u8_attributes(attrib):
5403 for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
5404 if val.startswith('"'):
5410 def urshift(val, n):
5411 return val >> n if val >= 0 else (val + 0x100000000) >> n
5414 # Based on png2str() written by @gdkchan and improved by @yokrysty
5415 # Originally posted at https://github.com/ytdl-org/youtube-dl/issues/9706
5416 def decode_png(png_data):
5417 # Reference: https://www.w3.org/TR/PNG/
5418 header = png_data[8:]
5420 if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
5421 raise IOError('Not a valid PNG file.')
5423 int_map = {1: '>B', 2: '>H', 4: '>I'}
5424 unpack_integer = lambda x: compat_struct_unpack(int_map[len(x)], x)[0]
5429 length = unpack_integer(header[:4])
5432 chunk_type = header[:4]
5435 chunk_data = header[:length]
5436 header = header[length:]
5438 header = header[4:] # Skip CRC
5446 ihdr = chunks[0]['data']
5448 width = unpack_integer(ihdr[:4])
5449 height = unpack_integer(ihdr[4:8])
5453 for chunk in chunks:
5454 if chunk['type'] == b'IDAT':
5455 idat += chunk['data']
5458 raise IOError('Unable to read PNG data.')
5460 decompressed_data = bytearray(zlib.decompress(idat))
5465 def _get_pixel(idx):
5470 for y in range(height):
5471 basePos = y * (1 + stride)
5472 filter_type = decompressed_data[basePos]
5476 pixels.append(current_row)
5478 for x in range(stride):
5479 color = decompressed_data[1 + basePos + x]
5480 basex = y * stride + x
5485 left = _get_pixel(basex - 3)
5487 up = _get_pixel(basex - stride)
5489 if filter_type == 1: # Sub
5490 color = (color + left) & 0xff
5491 elif filter_type == 2: # Up
5492 color = (color + up) & 0xff
5493 elif filter_type == 3: # Average
5494 color = (color + ((left + up) >> 1)) & 0xff
5495 elif filter_type == 4: # Paeth
5501 c = _get_pixel(basex - stride - 3)
5509 if pa <= pb and pa <= pc:
5510 color = (color + a) & 0xff
5512 color = (color + b) & 0xff
5514 color = (color + c) & 0xff
5516 current_row.append(color)
5518 return width, height, pixels
5521 def write_xattr(path, key, value):
5522 # This mess below finds the best xattr tool for the job
5524 # try the pyxattr module...
5527 if hasattr(xattr, 'set'): # pyxattr
5528 # Unicode arguments are not supported in python-pyxattr until
5530 # See https://github.com/ytdl-org/youtube-dl/issues/5498
5531 pyxattr_required_version = '0.5.0'
5532 if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
5533 # TODO: fallback to CLI tools
5534 raise XAttrUnavailableError(
5535 'python-pyxattr is detected but is too old. '
5536 'youtube-dl requires %s or above while your version is %s. '
5537 'Falling back to other xattr implementations' % (
5538 pyxattr_required_version, xattr.__version__))
5540 setxattr = xattr.set
5542 setxattr = xattr.setxattr
5545 setxattr(path, key, value)
5546 except EnvironmentError as e:
5547 raise XAttrMetadataError(e.errno, e.strerror)
5550 if compat_os_name == 'nt':
5551 # Write xattrs to NTFS Alternate Data Streams:
5552 # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
5553 assert ':' not in key
5554 assert os.path.exists(path)
5556 ads_fn = path + ':' + key
5558 with open(ads_fn, 'wb') as f:
5560 except EnvironmentError as e:
5561 raise XAttrMetadataError(e.errno, e.strerror)
5563 user_has_setfattr = check_executable('setfattr', ['--version'])
5564 user_has_xattr = check_executable('xattr', ['-h'])
5566 if user_has_setfattr or user_has_xattr:
5568 value = value.decode('utf-8')
5569 if user_has_setfattr:
5570 executable = 'setfattr'
5571 opts = ['-n', key, '-v', value]
5572 elif user_has_xattr:
5573 executable = 'xattr'
5574 opts = ['-w', key, value]
5576 cmd = ([encodeFilename(executable, True)]
5577 + [encodeArgument(o) for o in opts]
5578 + [encodeFilename(path, True)])
5581 p = subprocess.Popen(
5582 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
5583 except EnvironmentError as e:
5584 raise XAttrMetadataError(e.errno, e.strerror)
5585 stdout, stderr = p.communicate()
5586 stderr = stderr.decode('utf-8', 'replace')
5587 if p.returncode != 0:
5588 raise XAttrMetadataError(p.returncode, stderr)
5591 # On Unix, and can't find pyxattr, setfattr, or xattr.
5592 if sys.platform.startswith('linux'):
5593 raise XAttrUnavailableError(
5594 "Couldn't find a tool to set the xattrs. "
5595 "Install either the python 'pyxattr' or 'xattr' "
5596 "modules, or the GNU 'attr' package "
5597 "(which contains the 'setfattr' tool).")
5599 raise XAttrUnavailableError(
5600 "Couldn't find a tool to set the xattrs. "
5601 "Install either the python 'xattr' module, "
5602 "or the 'xattr' binary.")
5605 def random_birthday(year_field, month_field, day_field):
5606 start_date = datetime.date(1950, 1, 1)
5607 end_date = datetime.date(1995, 12, 31)
5608 offset = random.randint(0, (end_date - start_date).days)
5609 random_date = start_date + datetime.timedelta(offset)
5611 year_field: str(random_date.year),
5612 month_field: str(random_date.month),
5613 day_field: str(random_date.day),
projects / youtube-dl / blob