4 from __future__ import unicode_literals
34 import xml.etree.ElementTree
38 compat_HTMLParseError,
43 compat_ctypes_WINFUNCTYPE,
44 compat_etree_fromstring,
47 compat_html_entities_html5,
58 compat_urllib_parse_urlencode,
59 compat_urllib_parse_urlparse,
60 compat_urllib_parse_unquote_plus,
61 compat_urllib_request,
72 def register_socks_protocols():
73 # "Register" SOCKS protocols
74 # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
75 # URLs with protocols not in urlparse.uses_netloc are not handled correctly
76 for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
77 if scheme not in compat_urlparse.uses_netloc:
78 compat_urlparse.uses_netloc.append(scheme)
81 # This is not clearly defined otherwise
82 compiled_regex_type = type(re.compile(''))
85 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0',
86 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
87 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
88 'Accept-Encoding': 'gzip, deflate',
89 'Accept-Language': 'en-us,en;q=0.5',
94 'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
100 ENGLISH_MONTH_NAMES = [
101 'January', 'February', 'March', 'April', 'May', 'June',
102 'July', 'August', 'September', 'October', 'November', 'December']
105 'en': ENGLISH_MONTH_NAMES,
107 'janvier', 'février', 'mars', 'avril', 'mai', 'juin',
108 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'],
112 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
113 'flv', 'f4v', 'f4a', 'f4b',
114 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
115 'mkv', 'mka', 'mk3d',
124 'f4f', 'f4m', 'm3u8', 'smil')
126 # needed for sanitizing filenames in restricted mode
127 ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
128 itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUY', ['TH', 'ss'],
129 'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuy', ['th'], 'y')))
152 '%Y-%m-%d %H:%M:%S.%f',
155 '%Y-%m-%dT%H:%M:%SZ',
156 '%Y-%m-%dT%H:%M:%S.%fZ',
157 '%Y-%m-%dT%H:%M:%S.%f0Z',
159 '%Y-%m-%dT%H:%M:%S.%f',
162 '%b %d %Y at %H:%M:%S',
164 '%B %d %Y at %H:%M:%S',
167 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
168 DATE_FORMATS_DAY_FIRST.extend([
177 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
178 DATE_FORMATS_MONTH_FIRST.extend([
186 PACKED_CODES_RE = r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
187 JSON_LD_RE = r'(?is)<script[^>]+type=(["\']?)application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>'
190 def preferredencoding():
191 """Get preferred encoding.
193 Returns the best encoding scheme for the system, based on
194 locale.getpreferredencoding() and some further tweaks.
197 pref = locale.getpreferredencoding()
205 def write_json_file(obj, fn):
206 """ Encode obj as JSON and write it to fn, atomically if possible """
208 fn = encodeFilename(fn)
209 if sys.version_info < (3, 0) and sys.platform != 'win32':
210 encoding = get_filesystem_encoding()
211 # os.path.basename returns a bytes object, but NamedTemporaryFile
212 # will fail if the filename contains non ascii characters unless we
213 # use a unicode object
214 path_basename = lambda f: os.path.basename(fn).decode(encoding)
215 # the same for os.path.dirname
216 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
218 path_basename = os.path.basename
219 path_dirname = os.path.dirname
223 'prefix': path_basename(fn) + '.',
224 'dir': path_dirname(fn),
228 # In Python 2.x, json.dump expects a bytestream.
229 # In Python 3.x, it writes to a character stream
230 if sys.version_info < (3, 0):
238 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
243 if sys.platform == 'win32':
244 # Need to remove existing file on Windows, else os.rename raises
245 # WindowsError or FileExistsError.
250 os.rename(tf.name, fn)
259 if sys.version_info >= (2, 7):
260 def find_xpath_attr(node, xpath, key, val=None):
261 """ Find the xpath xpath[@key=val] """
262 assert re.match(r'^[a-zA-Z_-]+$', key)
263 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
264 return node.find(expr)
266 def find_xpath_attr(node, xpath, key, val=None):
267 for f in node.findall(compat_xpath(xpath)):
268 if key not in f.attrib:
270 if val is None or f.attrib.get(key) == val:
274 # On python2.6 the xml.etree.ElementTree.Element methods don't support
275 # the namespace parameter
278 def xpath_with_ns(path, ns_map):
279 components = [c.split(':') for c in path.split('/')]
283 replaced.append(c[0])
286 replaced.append('{%s}%s' % (ns_map[ns], tag))
287 return '/'.join(replaced)
290 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
291 def _find_xpath(xpath):
292 return node.find(compat_xpath(xpath))
294 if isinstance(xpath, (str, compat_str)):
295 n = _find_xpath(xpath)
303 if default is not NO_DEFAULT:
306 name = xpath if name is None else name
307 raise ExtractorError('Could not find XML element %s' % name)
313 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
314 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
315 if n is None or n == default:
318 if default is not NO_DEFAULT:
321 name = xpath if name is None else name
322 raise ExtractorError('Could not find XML element\'s text %s' % name)
328 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
329 n = find_xpath_attr(node, xpath, key)
331 if default is not NO_DEFAULT:
334 name = '%s[@%s]' % (xpath, key) if name is None else name
335 raise ExtractorError('Could not find XML attribute %s' % name)
341 def get_element_by_id(id, html):
342 """Return the content of the tag with the specified ID in the passed HTML document"""
343 return get_element_by_attribute('id', id, html)
346 def get_element_by_class(class_name, html):
347 """Return the content of the first tag with the specified class in the passed HTML document"""
348 retval = get_elements_by_class(class_name, html)
349 return retval[0] if retval else None
352 def get_element_by_attribute(attribute, value, html, escape_value=True):
353 retval = get_elements_by_attribute(attribute, value, html, escape_value)
354 return retval[0] if retval else None
357 def get_elements_by_class(class_name, html):
358 """Return the content of all tags with the specified class in the passed HTML document as a list"""
359 return get_elements_by_attribute(
360 'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
361 html, escape_value=False)
364 def get_elements_by_attribute(attribute, value, html, escape_value=True):
365 """Return the content of the tag with the specified attribute in the passed HTML document"""
367 value = re.escape(value) if escape_value else value
370 for m in re.finditer(r'''(?xs)
372 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
374 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
378 ''' % (re.escape(attribute), value), html):
379 res = m.group('content')
381 if res.startswith('"') or res.startswith("'"):
384 retlist.append(unescapeHTML(res))
389 class HTMLAttributeParser(compat_HTMLParser):
390 """Trivial HTML parser to gather the attributes for a single element"""
393 compat_HTMLParser.__init__(self)
395 def handle_starttag(self, tag, attrs):
396 self.attrs = dict(attrs)
399 def extract_attributes(html_element):
400 """Given a string for an HTML element such as
402 a="foo" B="bar" c="&98;az" d=boz
403 empty= noval entity="&"
406 Decode and return a dictionary of attributes.
408 'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
409 'empty': '', 'noval': None, 'entity': '&',
410 'sq': '"', 'dq': '\''
412 NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
413 but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
415 parser = HTMLAttributeParser()
417 parser.feed(html_element)
419 # Older Python may throw HTMLParseError in case of malformed HTML
420 except compat_HTMLParseError:
425 def clean_html(html):
426 """Clean an HTML snippet into a readable string"""
428 if html is None: # Convenience for sanitizing descriptions etc.
432 html = html.replace('\n', ' ')
433 html = re.sub(r'(?u)\s*<\s*br\s*/?\s*>\s*', '\n', html)
434 html = re.sub(r'(?u)<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
436 html = re.sub('<.*?>', '', html)
437 # Replace html entities
438 html = unescapeHTML(html)
442 def sanitize_open(filename, open_mode):
443 """Try to open the given filename, and slightly tweak it if this fails.
445 Attempts to open the given filename. If this fails, it tries to change
446 the filename slightly, step by step, until it's either able to open it
447 or it fails and raises a final exception, like the standard open()
450 It returns the tuple (stream, definitive_file_name).
454 if sys.platform == 'win32':
456 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
457 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
458 stream = open(encodeFilename(filename), open_mode)
459 return (stream, filename)
460 except (IOError, OSError) as err:
461 if err.errno in (errno.EACCES,):
464 # In case of error, try to remove win32 forbidden chars
465 alt_filename = sanitize_path(filename)
466 if alt_filename == filename:
469 # An exception here should be caught in the caller
470 stream = open(encodeFilename(alt_filename), open_mode)
471 return (stream, alt_filename)
474 def timeconvert(timestr):
475 """Convert RFC 2822 defined time string into system timestamp"""
477 timetuple = email.utils.parsedate_tz(timestr)
478 if timetuple is not None:
479 timestamp = email.utils.mktime_tz(timetuple)
483 def sanitize_filename(s, restricted=False, is_id=False):
484 """Sanitizes a string so it could be used as part of a filename.
485 If restricted is set, use a stricter subset of allowed characters.
486 Set is_id if this is not an arbitrary string, but an ID that should be kept
489 def replace_insane(char):
490 if restricted and char in ACCENT_CHARS:
491 return ACCENT_CHARS[char]
492 if char == '?' or ord(char) < 32 or ord(char) == 127:
495 return '' if restricted else '\''
497 return '_-' if restricted else ' -'
498 elif char in '\\/|*<>':
500 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
502 if restricted and ord(char) > 127:
507 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
508 result = ''.join(map(replace_insane, s))
510 while '__' in result:
511 result = result.replace('__', '_')
512 result = result.strip('_')
513 # Common case of "Foreign band name - English song title"
514 if restricted and result.startswith('-_'):
516 if result.startswith('-'):
517 result = '_' + result[len('-'):]
518 result = result.lstrip('.')
524 def sanitize_path(s):
525 """Sanitizes and normalizes path on Windows"""
526 if sys.platform != 'win32':
528 drive_or_unc, _ = os.path.splitdrive(s)
529 if sys.version_info < (2, 7) and not drive_or_unc:
530 drive_or_unc, _ = os.path.splitunc(s)
531 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
535 path_part if path_part in ['.', '..'] else re.sub(r'(?:[/<>:"\|\\?\*]|[\s.]$)', '#', path_part)
536 for path_part in norm_path]
538 sanitized_path.insert(0, drive_or_unc + os.path.sep)
539 return os.path.join(*sanitized_path)
542 def sanitize_url(url):
543 # Prepend protocol-less URLs with `http:` scheme in order to mitigate
544 # the number of unwanted failures due to missing protocol
545 if url.startswith('//'):
546 return 'http:%s' % url
547 # Fix some common typos seen so far
549 # https://github.com/ytdl-org/youtube-dl/issues/15649
550 (r'^httpss://', r'https://'),
551 # https://bx1.be/lives/direct-tv/
552 (r'^rmtp([es]?)://', r'rtmp\1://'),
554 for mistake, fixup in COMMON_TYPOS:
555 if re.match(mistake, url):
556 return re.sub(mistake, fixup, url)
560 def sanitized_Request(url, *args, **kwargs):
561 return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)
565 """Expand shell variables and ~"""
566 return os.path.expandvars(compat_expanduser(s))
569 def orderedSet(iterable):
570 """ Remove all duplicates from the input iterable """
578 def _htmlentity_transform(entity_with_semicolon):
579 """Transforms an HTML entity to a character."""
580 entity = entity_with_semicolon[:-1]
582 # Known non-numeric HTML entity
583 if entity in compat_html_entities.name2codepoint:
584 return compat_chr(compat_html_entities.name2codepoint[entity])
586 # TODO: HTML5 allows entities without a semicolon. For example,
587 # 'Éric' should be decoded as 'Éric'.
588 if entity_with_semicolon in compat_html_entities_html5:
589 return compat_html_entities_html5[entity_with_semicolon]
591 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
593 numstr = mobj.group(1)
594 if numstr.startswith('x'):
596 numstr = '0%s' % numstr
599 # See https://github.com/ytdl-org/youtube-dl/issues/7518
601 return compat_chr(int(numstr, base))
605 # Unknown entity in name, return its literal representation
606 return '&%s;' % entity
612 assert type(s) == compat_str
615 r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
618 def get_subprocess_encoding():
619 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
620 # For subprocess calls, encode with locale encoding
621 # Refer to http://stackoverflow.com/a/9951851/35070
622 encoding = preferredencoding()
624 encoding = sys.getfilesystemencoding()
630 def encodeFilename(s, for_subprocess=False):
632 @param s The name of the file
635 assert type(s) == compat_str
637 # Python 3 has a Unicode API
638 if sys.version_info >= (3, 0):
641 # Pass '' directly to use Unicode APIs on Windows 2000 and up
642 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
643 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
644 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
647 # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
648 if sys.platform.startswith('java'):
651 return s.encode(get_subprocess_encoding(), 'ignore')
654 def decodeFilename(b, for_subprocess=False):
656 if sys.version_info >= (3, 0):
659 if not isinstance(b, bytes):
662 return b.decode(get_subprocess_encoding(), 'ignore')
665 def encodeArgument(s):
666 if not isinstance(s, compat_str):
667 # Legacy code that uses byte strings
668 # Uncomment the following line after fixing all post processors
669 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
670 s = s.decode('ascii')
671 return encodeFilename(s, True)
674 def decodeArgument(b):
675 return decodeFilename(b, True)
678 def decodeOption(optval):
681 if isinstance(optval, bytes):
682 optval = optval.decode(preferredencoding())
684 assert isinstance(optval, compat_str)
688 def formatSeconds(secs):
690 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
692 return '%d:%02d' % (secs // 60, secs % 60)
697 def make_HTTPS_handler(params, **kwargs):
698 opts_no_check_certificate = params.get('nocheckcertificate', False)
699 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
700 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
701 if opts_no_check_certificate:
702 context.check_hostname = False
703 context.verify_mode = ssl.CERT_NONE
705 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
708 # (create_default_context present but HTTPSHandler has no context=)
711 if sys.version_info < (3, 2):
712 return YoutubeDLHTTPSHandler(params, **kwargs)
714 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
715 context.verify_mode = (ssl.CERT_NONE
716 if opts_no_check_certificate
717 else ssl.CERT_REQUIRED)
718 context.set_default_verify_paths()
719 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
722 def bug_reports_message():
723 if ytdl_is_updateable():
724 update_cmd = 'type youtube-dl -U to update'
726 update_cmd = 'see https://yt-dl.org/update on how to update'
727 msg = '; please report this issue on https://yt-dl.org/bug .'
728 msg += ' Make sure you are using the latest version; %s.' % update_cmd
729 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
733 class YoutubeDLError(Exception):
734 """Base exception for YoutubeDL errors."""
738 class ExtractorError(YoutubeDLError):
739 """Error during info extraction."""
741 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
742 """ tb, if given, is the original traceback (so that it can be printed out).
743 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
746 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
748 if video_id is not None:
749 msg = video_id + ': ' + msg
751 msg += ' (caused by %r)' % cause
753 msg += bug_reports_message()
754 super(ExtractorError, self).__init__(msg)
757 self.exc_info = sys.exc_info() # preserve original exception
759 self.video_id = video_id
761 def format_traceback(self):
762 if self.traceback is None:
764 return ''.join(traceback.format_tb(self.traceback))
767 class UnsupportedError(ExtractorError):
768 def __init__(self, url):
769 super(UnsupportedError, self).__init__(
770 'Unsupported URL: %s' % url, expected=True)
774 class RegexNotFoundError(ExtractorError):
775 """Error when a regex didn't match"""
779 class GeoRestrictedError(ExtractorError):
780 """Geographic restriction Error exception.
782 This exception may be thrown when a video is not available from your
783 geographic location due to geographic restrictions imposed by a website.
785 def __init__(self, msg, countries=None):
786 super(GeoRestrictedError, self).__init__(msg, expected=True)
788 self.countries = countries
791 class DownloadError(YoutubeDLError):
792 """Download Error exception.
794 This exception may be thrown by FileDownloader objects if they are not
795 configured to continue on errors. They will contain the appropriate
799 def __init__(self, msg, exc_info=None):
800 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
801 super(DownloadError, self).__init__(msg)
802 self.exc_info = exc_info
805 class SameFileError(YoutubeDLError):
806 """Same File exception.
808 This exception will be thrown by FileDownloader objects if they detect
809 multiple files would have to be downloaded to the same file on disk.
814 class PostProcessingError(YoutubeDLError):
815 """Post Processing exception.
817 This exception may be raised by PostProcessor's .run() method to
818 indicate an error in the postprocessing task.
821 def __init__(self, msg):
822 super(PostProcessingError, self).__init__(msg)
826 class MaxDownloadsReached(YoutubeDLError):
827 """ --max-downloads limit has been reached. """
831 class UnavailableVideoError(YoutubeDLError):
832 """Unavailable Format exception.
834 This exception will be thrown when a video is requested
835 in a format that is not available for that video.
840 class ContentTooShortError(YoutubeDLError):
841 """Content Too Short exception.
843 This exception may be raised by FileDownloader objects when a file they
844 download is too small for what the server announced first, indicating
845 the connection was probably interrupted.
848 def __init__(self, downloaded, expected):
849 super(ContentTooShortError, self).__init__(
850 'Downloaded {0} bytes, expected {1} bytes'.format(downloaded, expected)
853 self.downloaded = downloaded
854 self.expected = expected
857 class XAttrMetadataError(YoutubeDLError):
858 def __init__(self, code=None, msg='Unknown error'):
859 super(XAttrMetadataError, self).__init__(msg)
863 # Parsing code and msg
864 if (self.code in (errno.ENOSPC, errno.EDQUOT)
865 or 'No space left' in self.msg or 'Disk quota excedded' in self.msg):
866 self.reason = 'NO_SPACE'
867 elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
868 self.reason = 'VALUE_TOO_LONG'
870 self.reason = 'NOT_SUPPORTED'
873 class XAttrUnavailableError(YoutubeDLError):
877 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
878 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
879 # expected HTTP responses to meet HTTP/1.0 or later (see also
880 # https://github.com/ytdl-org/youtube-dl/issues/6727)
881 if sys.version_info < (3, 0):
882 kwargs['strict'] = True
883 hc = http_class(*args, **compat_kwargs(kwargs))
884 source_address = ydl_handler._params.get('source_address')
886 if source_address is not None:
887 # This is to workaround _create_connection() from socket where it will try all
888 # address data from getaddrinfo() including IPv6. This filters the result from
889 # getaddrinfo() based on the source_address value.
890 # This is based on the cpython socket.create_connection() function.
891 # https://github.com/python/cpython/blob/master/Lib/socket.py#L691
892 def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
895 addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
896 af = socket.AF_INET if '.' in source_address[0] else socket.AF_INET6
897 ip_addrs = [addr for addr in addrs if addr[0] == af]
898 if addrs and not ip_addrs:
899 ip_version = 'v4' if af == socket.AF_INET else 'v6'
901 "No remote IP%s addresses available for connect, can't use '%s' as source address"
902 % (ip_version, source_address[0]))
904 af, socktype, proto, canonname, sa = res
907 sock = socket.socket(af, socktype, proto)
908 if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
909 sock.settimeout(timeout)
910 sock.bind(source_address)
912 err = None # Explicitly break reference cycle
914 except socket.error as _:
921 raise socket.error('getaddrinfo returns an empty list')
922 if hasattr(hc, '_create_connection'):
923 hc._create_connection = _create_connection
924 sa = (source_address, 0)
925 if hasattr(hc, 'source_address'): # Python 2.7+
926 hc.source_address = sa
928 def _hc_connect(self, *args, **kwargs):
929 sock = _create_connection(
930 (self.host, self.port), self.timeout, sa)
932 self.sock = ssl.wrap_socket(
933 sock, self.key_file, self.cert_file,
934 ssl_version=ssl.PROTOCOL_TLSv1)
937 hc.connect = functools.partial(_hc_connect, hc)
942 def handle_youtubedl_headers(headers):
943 filtered_headers = headers
945 if 'Youtubedl-no-compression' in filtered_headers:
946 filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
947 del filtered_headers['Youtubedl-no-compression']
949 return filtered_headers
952 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
953 """Handler for HTTP requests and responses.
955 This class, when installed with an OpenerDirector, automatically adds
956 the standard headers to every HTTP request and handles gzipped and
957 deflated responses from web servers. If compression is to be avoided in
958 a particular request, the original request in the program code only has
959 to include the HTTP header "Youtubedl-no-compression", which will be
960 removed before making the real request.
962 Part of this code was copied from:
964 http://techknack.net/python-urllib2-handlers/
966 Andrew Rowls, the author of that code, agreed to release it to the
970 def __init__(self, params, *args, **kwargs):
971 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
972 self._params = params
974 def http_open(self, req):
975 conn_class = compat_http_client.HTTPConnection
977 socks_proxy = req.headers.get('Ytdl-socks-proxy')
979 conn_class = make_socks_conn_class(conn_class, socks_proxy)
980 del req.headers['Ytdl-socks-proxy']
982 return self.do_open(functools.partial(
983 _create_http_connection, self, conn_class, False),
989 return zlib.decompress(data, -zlib.MAX_WBITS)
991 return zlib.decompress(data)
993 def http_request(self, req):
994 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
995 # always respected by websites, some tend to give out URLs with non percent-encoded
996 # non-ASCII characters (see telemb.py, ard.py [#3412])
997 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
998 # To work around aforementioned issue we will replace request's original URL with
999 # percent-encoded one
1000 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
1001 # the code of this workaround has been moved here from YoutubeDL.urlopen()
1002 url = req.get_full_url()
1003 url_escaped = escape_url(url)
1005 # Substitute URL if any change after escaping
1006 if url != url_escaped:
1007 req = update_Request(req, url=url_escaped)
1009 for h, v in std_headers.items():
1010 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
1011 # The dict keys are capitalized because of this bug by urllib
1012 if h.capitalize() not in req.headers:
1013 req.add_header(h, v)
1015 req.headers = handle_youtubedl_headers(req.headers)
1017 if sys.version_info < (2, 7) and '#' in req.get_full_url():
1018 # Python 2.6 is brain-dead when it comes to fragments
1019 req._Request__original = req._Request__original.partition('#')[0]
1020 req._Request__r_type = req._Request__r_type.partition('#')[0]
1024 def http_response(self, req, resp):
1027 if resp.headers.get('Content-encoding', '') == 'gzip':
1028 content = resp.read()
1029 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
1031 uncompressed = io.BytesIO(gz.read())
1032 except IOError as original_ioerror:
1033 # There may be junk add the end of the file
1034 # See http://stackoverflow.com/q/4928560/35070 for details
1035 for i in range(1, 1024):
1037 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
1038 uncompressed = io.BytesIO(gz.read())
1043 raise original_ioerror
1044 resp = compat_urllib_request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
1045 resp.msg = old_resp.msg
1046 del resp.headers['Content-encoding']
1048 if resp.headers.get('Content-encoding', '') == 'deflate':
1049 gz = io.BytesIO(self.deflate(resp.read()))
1050 resp = compat_urllib_request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
1051 resp.msg = old_resp.msg
1052 del resp.headers['Content-encoding']
1053 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
1054 # https://github.com/ytdl-org/youtube-dl/issues/6457).
1055 if 300 <= resp.code < 400:
1056 location = resp.headers.get('Location')
1058 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
1059 if sys.version_info >= (3, 0):
1060 location = location.encode('iso-8859-1').decode('utf-8')
1062 location = location.decode('utf-8')
1063 location_escaped = escape_url(location)
1064 if location != location_escaped:
1065 del resp.headers['Location']
1066 if sys.version_info < (3, 0):
1067 location_escaped = location_escaped.encode('utf-8')
1068 resp.headers['Location'] = location_escaped
1071 https_request = http_request
1072 https_response = http_response
1075 def make_socks_conn_class(base_class, socks_proxy):
1076 assert issubclass(base_class, (
1077 compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
1079 url_components = compat_urlparse.urlparse(socks_proxy)
1080 if url_components.scheme.lower() == 'socks5':
1081 socks_type = ProxyType.SOCKS5
1082 elif url_components.scheme.lower() in ('socks', 'socks4'):
1083 socks_type = ProxyType.SOCKS4
1084 elif url_components.scheme.lower() == 'socks4a':
1085 socks_type = ProxyType.SOCKS4A
1087 def unquote_if_non_empty(s):
1090 return compat_urllib_parse_unquote_plus(s)
1094 url_components.hostname, url_components.port or 1080,
1096 unquote_if_non_empty(url_components.username),
1097 unquote_if_non_empty(url_components.password),
1100 class SocksConnection(base_class):
1102 self.sock = sockssocket()
1103 self.sock.setproxy(*proxy_args)
1104 if type(self.timeout) in (int, float):
1105 self.sock.settimeout(self.timeout)
1106 self.sock.connect((self.host, self.port))
1108 if isinstance(self, compat_http_client.HTTPSConnection):
1109 if hasattr(self, '_context'): # Python > 2.6
1110 self.sock = self._context.wrap_socket(
1111 self.sock, server_hostname=self.host)
1113 self.sock = ssl.wrap_socket(self.sock)
1115 return SocksConnection
1118 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
1119 def __init__(self, params, https_conn_class=None, *args, **kwargs):
1120 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
1121 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
1122 self._params = params
1124 def https_open(self, req):
1126 conn_class = self._https_conn_class
1128 if hasattr(self, '_context'): # python > 2.6
1129 kwargs['context'] = self._context
1130 if hasattr(self, '_check_hostname'): # python 3.x
1131 kwargs['check_hostname'] = self._check_hostname
1133 socks_proxy = req.headers.get('Ytdl-socks-proxy')
1135 conn_class = make_socks_conn_class(conn_class, socks_proxy)
1136 del req.headers['Ytdl-socks-proxy']
1138 return self.do_open(functools.partial(
1139 _create_http_connection, self, conn_class, True),
1143 class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
1144 _HTTPONLY_PREFIX = '#HttpOnly_'
1146 def save(self, filename=None, ignore_discard=False, ignore_expires=False):
1147 # Store session cookies with `expires` set to 0 instead of an empty
1150 if cookie.expires is None:
1152 compat_cookiejar.MozillaCookieJar.save(self, filename, ignore_discard, ignore_expires)
1154 def load(self, filename=None, ignore_discard=False, ignore_expires=False):
1155 """Load cookies from a file."""
1156 if filename is None:
1157 if self.filename is not None:
1158 filename = self.filename
1160 raise ValueError(compat_cookiejar.MISSING_FILENAME_TEXT)
1163 with open(filename) as f:
1165 if line.startswith(self._HTTPONLY_PREFIX):
1166 line = line[len(self._HTTPONLY_PREFIX):]
1167 cf.write(compat_str(line))
1169 self._really_load(cf, filename, ignore_discard, ignore_expires)
1170 # Session cookies are denoted by either `expires` field set to
1171 # an empty string or 0. MozillaCookieJar only recognizes the former
1172 # (see [1]). So we need force the latter to be recognized as session
1173 # cookies on our own.
1174 # Session cookies may be important for cookies-based authentication,
1175 # e.g. usually, when user does not check 'Remember me' check box while
1176 # logging in on a site, some important cookies are stored as session
1177 # cookies so that not recognizing them will result in failed login.
1178 # 1. https://bugs.python.org/issue17164
1180 # Treat `expires=0` cookies as session cookies
1181 if cookie.expires == 0:
1182 cookie.expires = None
1183 cookie.discard = True
1186 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
1187 def __init__(self, cookiejar=None):
1188 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
1190 def http_response(self, request, response):
1191 # Python 2 will choke on next HTTP request in row if there are non-ASCII
1192 # characters in Set-Cookie HTTP header of last response (see
1193 # https://github.com/ytdl-org/youtube-dl/issues/6769).
1194 # In order to at least prevent crashing we will percent encode Set-Cookie
1195 # header before HTTPCookieProcessor starts processing it.
1196 # if sys.version_info < (3, 0) and response.headers:
1197 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
1198 # set_cookie = response.headers.get(set_cookie_header)
1200 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
1201 # if set_cookie != set_cookie_escaped:
1202 # del response.headers[set_cookie_header]
1203 # response.headers[set_cookie_header] = set_cookie_escaped
1204 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
1206 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
1207 https_response = http_response
1210 def extract_timezone(date_str):
1212 r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
1215 timezone = datetime.timedelta()
1217 date_str = date_str[:-len(m.group('tz'))]
1218 if not m.group('sign'):
1219 timezone = datetime.timedelta()
1221 sign = 1 if m.group('sign') == '+' else -1
1222 timezone = datetime.timedelta(
1223 hours=sign * int(m.group('hours')),
1224 minutes=sign * int(m.group('minutes')))
1225 return timezone, date_str
1228 def parse_iso8601(date_str, delimiter='T', timezone=None):
1229 """ Return a UNIX timestamp from the given date """
1231 if date_str is None:
1234 date_str = re.sub(r'\.[0-9]+', '', date_str)
1236 if timezone is None:
1237 timezone, date_str = extract_timezone(date_str)
1240 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
1241 dt = datetime.datetime.strptime(date_str, date_format) - timezone
1242 return calendar.timegm(dt.timetuple())
1247 def date_formats(day_first=True):
1248 return DATE_FORMATS_DAY_FIRST if day_first else DATE_FORMATS_MONTH_FIRST
1251 def unified_strdate(date_str, day_first=True):
1252 """Return a string with the date in the format YYYYMMDD"""
1254 if date_str is None:
1258 date_str = date_str.replace(',', ' ')
1259 # Remove AM/PM + timezone
1260 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
1261 _, date_str = extract_timezone(date_str)
1263 for expression in date_formats(day_first):
1265 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
1268 if upload_date is None:
1269 timetuple = email.utils.parsedate_tz(date_str)
1272 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
1275 if upload_date is not None:
1276 return compat_str(upload_date)
1279 def unified_timestamp(date_str, day_first=True):
1280 if date_str is None:
1283 date_str = re.sub(r'[,|]', '', date_str)
1285 pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
1286 timezone, date_str = extract_timezone(date_str)
1288 # Remove AM/PM + timezone
1289 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
1291 # Remove unrecognized timezones from ISO 8601 alike timestamps
1292 m = re.search(r'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str)
1294 date_str = date_str[:-len(m.group('tz'))]
1296 # Python only supports microseconds, so remove nanoseconds
1297 m = re.search(r'^([0-9]{4,}-[0-9]{1,2}-[0-9]{1,2}T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]{6})[0-9]+$', date_str)
1299 date_str = m.group(1)
1301 for expression in date_formats(day_first):
1303 dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
1304 return calendar.timegm(dt.timetuple())
1307 timetuple = email.utils.parsedate_tz(date_str)
1309 return calendar.timegm(timetuple) + pm_delta * 3600
1312 def determine_ext(url, default_ext='unknown_video'):
1313 if url is None or '.' not in url:
1315 guess = url.partition('?')[0].rpartition('.')[2]
1316 if re.match(r'^[A-Za-z0-9]+$', guess):
1318 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
1319 elif guess.rstrip('/') in KNOWN_EXTENSIONS:
1320 return guess.rstrip('/')
1325 def subtitles_filename(filename, sub_lang, sub_format):
1326 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
1329 def date_from_str(date_str):
1331 Return a datetime object from a string in the format YYYYMMDD or
1332 (now|today)[+-][0-9](day|week|month|year)(s)?"""
1333 today = datetime.date.today()
1334 if date_str in ('now', 'today'):
1336 if date_str == 'yesterday':
1337 return today - datetime.timedelta(days=1)
1338 match = re.match(r'(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
1339 if match is not None:
1340 sign = match.group('sign')
1341 time = int(match.group('time'))
1344 unit = match.group('unit')
1345 # A bad approximation?
1349 elif unit == 'year':
1353 delta = datetime.timedelta(**{unit: time})
1354 return today + delta
1355 return datetime.datetime.strptime(date_str, '%Y%m%d').date()
1358 def hyphenate_date(date_str):
1360 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
1361 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
1362 if match is not None:
1363 return '-'.join(match.groups())
1368 class DateRange(object):
1369 """Represents a time interval between two dates"""
1371 def __init__(self, start=None, end=None):
1372 """start and end must be strings in the format accepted by date"""
1373 if start is not None:
1374 self.start = date_from_str(start)
1376 self.start = datetime.datetime.min.date()
1378 self.end = date_from_str(end)
1380 self.end = datetime.datetime.max.date()
1381 if self.start > self.end:
1382 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
1386 """Returns a range that only contains the given day"""
1387 return cls(day, day)
1389 def __contains__(self, date):
1390 """Check if the date is in the range"""
1391 if not isinstance(date, datetime.date):
1392 date = date_from_str(date)
1393 return self.start <= date <= self.end
1396 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
1399 def platform_name():
1400 """ Returns the platform name as a compat_str """
1401 res = platform.platform()
1402 if isinstance(res, bytes):
1403 res = res.decode(preferredencoding())
1405 assert isinstance(res, compat_str)
1409 def _windows_write_string(s, out):
1410 """ Returns True if the string was written using special methods,
1411 False if it has yet to be written out."""
1412 # Adapted from http://stackoverflow.com/a/3259271/35070
1415 import ctypes.wintypes
1423 fileno = out.fileno()
1424 except AttributeError:
1425 # If the output stream doesn't have a fileno, it's virtual
1427 except io.UnsupportedOperation:
1428 # Some strange Windows pseudo files?
1430 if fileno not in WIN_OUTPUT_IDS:
1433 GetStdHandle = compat_ctypes_WINFUNCTYPE(
1434 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1435 ('GetStdHandle', ctypes.windll.kernel32))
1436 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1438 WriteConsoleW = compat_ctypes_WINFUNCTYPE(
1439 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1440 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1441 ctypes.wintypes.LPVOID)(('WriteConsoleW', ctypes.windll.kernel32))
1442 written = ctypes.wintypes.DWORD(0)
1444 GetFileType = compat_ctypes_WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(('GetFileType', ctypes.windll.kernel32))
1445 FILE_TYPE_CHAR = 0x0002
1446 FILE_TYPE_REMOTE = 0x8000
1447 GetConsoleMode = compat_ctypes_WINFUNCTYPE(
1448 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1449 ctypes.POINTER(ctypes.wintypes.DWORD))(
1450 ('GetConsoleMode', ctypes.windll.kernel32))
1451 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1453 def not_a_console(handle):
1454 if handle == INVALID_HANDLE_VALUE or handle is None:
1456 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
1457 or GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1459 if not_a_console(h):
1462 def next_nonbmp_pos(s):
1464 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1465 except StopIteration:
1469 count = min(next_nonbmp_pos(s), 1024)
1471 ret = WriteConsoleW(
1472 h, s, count if count else 2, ctypes.byref(written), None)
1474 raise OSError('Failed to write string')
1475 if not count: # We just wrote a non-BMP character
1476 assert written.value == 2
1479 assert written.value > 0
1480 s = s[written.value:]
1484 def write_string(s, out=None, encoding=None):
1487 assert type(s) == compat_str
1489 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1490 if _windows_write_string(s, out):
1493 if ('b' in getattr(out, 'mode', '')
1494 or sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1495 byt = s.encode(encoding or preferredencoding(), 'ignore')
1497 elif hasattr(out, 'buffer'):
1498 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1499 byt = s.encode(enc, 'ignore')
1500 out.buffer.write(byt)
1506 def bytes_to_intlist(bs):
1509 if isinstance(bs[0], int): # Python 3
1512 return [ord(c) for c in bs]
1515 def intlist_to_bytes(xs):
1518 return compat_struct_pack('%dB' % len(xs), *xs)
1521 # Cross-platform file locking
1522 if sys.platform == 'win32':
1523 import ctypes.wintypes
1526 class OVERLAPPED(ctypes.Structure):
1528 ('Internal', ctypes.wintypes.LPVOID),
1529 ('InternalHigh', ctypes.wintypes.LPVOID),
1530 ('Offset', ctypes.wintypes.DWORD),
1531 ('OffsetHigh', ctypes.wintypes.DWORD),
1532 ('hEvent', ctypes.wintypes.HANDLE),
1535 kernel32 = ctypes.windll.kernel32
1536 LockFileEx = kernel32.LockFileEx
1537 LockFileEx.argtypes = [
1538 ctypes.wintypes.HANDLE, # hFile
1539 ctypes.wintypes.DWORD, # dwFlags
1540 ctypes.wintypes.DWORD, # dwReserved
1541 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1542 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1543 ctypes.POINTER(OVERLAPPED) # Overlapped
1545 LockFileEx.restype = ctypes.wintypes.BOOL
1546 UnlockFileEx = kernel32.UnlockFileEx
1547 UnlockFileEx.argtypes = [
1548 ctypes.wintypes.HANDLE, # hFile
1549 ctypes.wintypes.DWORD, # dwReserved
1550 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1551 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1552 ctypes.POINTER(OVERLAPPED) # Overlapped
1554 UnlockFileEx.restype = ctypes.wintypes.BOOL
1555 whole_low = 0xffffffff
1556 whole_high = 0x7fffffff
1558 def _lock_file(f, exclusive):
1559 overlapped = OVERLAPPED()
1560 overlapped.Offset = 0
1561 overlapped.OffsetHigh = 0
1562 overlapped.hEvent = 0
1563 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1564 handle = msvcrt.get_osfhandle(f.fileno())
1565 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1566 whole_low, whole_high, f._lock_file_overlapped_p):
1567 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1569 def _unlock_file(f):
1570 assert f._lock_file_overlapped_p
1571 handle = msvcrt.get_osfhandle(f.fileno())
1572 if not UnlockFileEx(handle, 0,
1573 whole_low, whole_high, f._lock_file_overlapped_p):
1574 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1577 # Some platforms, such as Jython, is missing fcntl
1581 def _lock_file(f, exclusive):
1582 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1584 def _unlock_file(f):
1585 fcntl.flock(f, fcntl.LOCK_UN)
1587 UNSUPPORTED_MSG = 'file locking is not supported on this platform'
1589 def _lock_file(f, exclusive):
1590 raise IOError(UNSUPPORTED_MSG)
1592 def _unlock_file(f):
1593 raise IOError(UNSUPPORTED_MSG)
1596 class locked_file(object):
1597 def __init__(self, filename, mode, encoding=None):
1598 assert mode in ['r', 'a', 'w']
1599 self.f = io.open(filename, mode, encoding=encoding)
1602 def __enter__(self):
1603 exclusive = self.mode != 'r'
1605 _lock_file(self.f, exclusive)
1611 def __exit__(self, etype, value, traceback):
1613 _unlock_file(self.f)
1620 def write(self, *args):
1621 return self.f.write(*args)
1623 def read(self, *args):
1624 return self.f.read(*args)
1627 def get_filesystem_encoding():
1628 encoding = sys.getfilesystemencoding()
1629 return encoding if encoding is not None else 'utf-8'
1632 def shell_quote(args):
1634 encoding = get_filesystem_encoding()
1636 if isinstance(a, bytes):
1637 # We may get a filename encoded with 'encodeFilename'
1638 a = a.decode(encoding)
1639 quoted_args.append(compat_shlex_quote(a))
1640 return ' '.join(quoted_args)
1643 def smuggle_url(url, data):
1644 """ Pass additional data in a URL for internal use. """
1646 url, idata = unsmuggle_url(url, {})
1648 sdata = compat_urllib_parse_urlencode(
1649 {'__youtubedl_smuggle': json.dumps(data)})
1650 return url + '#' + sdata
1653 def unsmuggle_url(smug_url, default=None):
1654 if '#__youtubedl_smuggle' not in smug_url:
1655 return smug_url, default
1656 url, _, sdata = smug_url.rpartition('#')
1657 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1658 data = json.loads(jsond)
1662 def format_bytes(bytes):
1665 if type(bytes) is str:
1666 bytes = float(bytes)
1670 exponent = int(math.log(bytes, 1024.0))
1671 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1672 converted = float(bytes) / float(1024 ** exponent)
1673 return '%.2f%s' % (converted, suffix)
1676 def lookup_unit_table(unit_table, s):
1677 units_re = '|'.join(re.escape(u) for u in unit_table)
1679 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
1682 num_str = m.group('num').replace(',', '.')
1683 mult = unit_table[m.group('unit')]
1684 return int(float(num_str) * mult)
1687 def parse_filesize(s):
1691 # The lower-case forms are of course incorrect and unofficial,
1692 # but we support those too
1709 'megabytes': 1000 ** 2,
1710 'mebibytes': 1024 ** 2,
1716 'gigabytes': 1000 ** 3,
1717 'gibibytes': 1024 ** 3,
1723 'terabytes': 1000 ** 4,
1724 'tebibytes': 1024 ** 4,
1730 'petabytes': 1000 ** 5,
1731 'pebibytes': 1024 ** 5,
1737 'exabytes': 1000 ** 6,
1738 'exbibytes': 1024 ** 6,
1744 'zettabytes': 1000 ** 7,
1745 'zebibytes': 1024 ** 7,
1751 'yottabytes': 1000 ** 8,
1752 'yobibytes': 1024 ** 8,
1755 return lookup_unit_table(_UNIT_TABLE, s)
1764 if re.match(r'^[\d,.]+$', s):
1765 return str_to_int(s)
1776 return lookup_unit_table(_UNIT_TABLE, s)
1779 def parse_resolution(s):
1783 mobj = re.search(r'\b(?P<w>\d+)\s*[xX×]\s*(?P<h>\d+)\b', s)
1786 'width': int(mobj.group('w')),
1787 'height': int(mobj.group('h')),
1790 mobj = re.search(r'\b(\d+)[pPiI]\b', s)
1792 return {'height': int(mobj.group(1))}
1794 mobj = re.search(r'\b([48])[kK]\b', s)
1796 return {'height': int(mobj.group(1)) * 540}
1801 def parse_bitrate(s):
1802 if not isinstance(s, compat_str):
1804 mobj = re.search(r'\b(\d+)\s*kbps', s)
1806 return int(mobj.group(1))
1809 def month_by_name(name, lang='en'):
1810 """ Return the number of a month by (locale-independently) English name """
1812 month_names = MONTH_NAMES.get(lang, MONTH_NAMES['en'])
1815 return month_names.index(name) + 1
1820 def month_by_abbreviation(abbrev):
1821 """ Return the number of a month by (locale-independently) English
1825 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1830 def fix_xml_ampersands(xml_str):
1831 """Replace all the '&' by '&' in XML"""
1833 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1838 def setproctitle(title):
1839 assert isinstance(title, compat_str)
1841 # ctypes in Jython is not complete
1842 # http://bugs.jython.org/issue2148
1843 if sys.platform.startswith('java'):
1847 libc = ctypes.cdll.LoadLibrary('libc.so.6')
1851 # LoadLibrary in Windows Python 2.7.13 only expects
1852 # a bytestring, but since unicode_literals turns
1853 # every string into a unicode string, it fails.
1855 title_bytes = title.encode('utf-8')
1856 buf = ctypes.create_string_buffer(len(title_bytes))
1857 buf.value = title_bytes
1859 libc.prctl(15, buf, 0, 0, 0)
1860 except AttributeError:
1861 return # Strange libc, just skip this
1864 def remove_start(s, start):
1865 return s[len(start):] if s is not None and s.startswith(start) else s
1868 def remove_end(s, end):
1869 return s[:-len(end)] if s is not None and s.endswith(end) else s
1872 def remove_quotes(s):
1873 if s is None or len(s) < 2:
1875 for quote in ('"', "'", ):
1876 if s[0] == quote and s[-1] == quote:
1881 def url_basename(url):
1882 path = compat_urlparse.urlparse(url).path
1883 return path.strip('/').split('/')[-1]
1887 return re.match(r'https?://[^?#&]+/', url).group()
1890 def urljoin(base, path):
1891 if isinstance(path, bytes):
1892 path = path.decode('utf-8')
1893 if not isinstance(path, compat_str) or not path:
1895 if re.match(r'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
1897 if isinstance(base, bytes):
1898 base = base.decode('utf-8')
1899 if not isinstance(base, compat_str) or not re.match(
1900 r'^(?:https?:)?//', base):
1902 return compat_urlparse.urljoin(base, path)
1905 class HEADRequest(compat_urllib_request.Request):
1906 def get_method(self):
1910 class PUTRequest(compat_urllib_request.Request):
1911 def get_method(self):
1915 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1918 v = getattr(v, get_attr, None)
1924 return int(v) * invscale // scale
1925 except (ValueError, TypeError):
1929 def str_or_none(v, default=None):
1930 return default if v is None else compat_str(v)
1933 def str_to_int(int_str):
1934 """ A more relaxed version of int_or_none """
1937 int_str = re.sub(r'[,\.\+]', '', int_str)
1941 def float_or_none(v, scale=1, invscale=1, default=None):
1945 return float(v) * invscale / scale
1946 except (ValueError, TypeError):
1950 def bool_or_none(v, default=None):
1951 return v if isinstance(v, bool) else default
1954 def strip_or_none(v, default=None):
1955 return v.strip() if isinstance(v, compat_str) else default
1958 def url_or_none(url):
1959 if not url or not isinstance(url, compat_str):
1962 return url if re.match(r'^(?:[a-zA-Z][\da-zA-Z.+-]*:)?//', url) else None
1965 def parse_duration(s):
1966 if not isinstance(s, compat_basestring):
1971 days, hours, mins, secs, ms = [None] * 5
1972 m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?Z?$', s)
1974 days, hours, mins, secs, ms = m.groups()
1979 [0-9]+\s*y(?:ears?)?\s*
1982 [0-9]+\s*m(?:onths?)?\s*
1985 [0-9]+\s*w(?:eeks?)?\s*
1988 (?P<days>[0-9]+)\s*d(?:ays?)?\s*
1992 (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
1995 (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
1998 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
2001 days, hours, mins, secs, ms = m.groups()
2003 m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)Z?$', s)
2005 hours, mins = m.groups()
2011 duration += float(secs)
2013 duration += float(mins) * 60
2015 duration += float(hours) * 60 * 60
2017 duration += float(days) * 24 * 60 * 60
2019 duration += float(ms)
2023 def prepend_extension(filename, ext, expected_real_ext=None):
2024 name, real_ext = os.path.splitext(filename)
2026 '{0}.{1}{2}'.format(name, ext, real_ext)
2027 if not expected_real_ext or real_ext[1:] == expected_real_ext
2028 else '{0}.{1}'.format(filename, ext))
2031 def replace_extension(filename, ext, expected_real_ext=None):
2032 name, real_ext = os.path.splitext(filename)
2033 return '{0}.{1}'.format(
2034 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
2038 def check_executable(exe, args=[]):
2039 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
2040 args can be a list of arguments for a short output (like -version) """
2042 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
2048 def get_exe_version(exe, args=['--version'],
2049 version_re=None, unrecognized='present'):
2050 """ Returns the version of the specified executable,
2051 or False if the executable is not present """
2053 # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
2054 # SIGTTOU if youtube-dl is run in the background.
2055 # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
2056 out, _ = subprocess.Popen(
2057 [encodeArgument(exe)] + args,
2058 stdin=subprocess.PIPE,
2059 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
2062 if isinstance(out, bytes): # Python 2.x
2063 out = out.decode('ascii', 'ignore')
2064 return detect_exe_version(out, version_re, unrecognized)
2067 def detect_exe_version(output, version_re=None, unrecognized='present'):
2068 assert isinstance(output, compat_str)
2069 if version_re is None:
2070 version_re = r'version\s+([-0-9._a-zA-Z]+)'
2071 m = re.search(version_re, output)
2078 class PagedList(object):
2080 # This is only useful for tests
2081 return len(self.getslice())
2084 class OnDemandPagedList(PagedList):
2085 def __init__(self, pagefunc, pagesize, use_cache=True):
2086 self._pagefunc = pagefunc
2087 self._pagesize = pagesize
2088 self._use_cache = use_cache
2092 def getslice(self, start=0, end=None):
2094 for pagenum in itertools.count(start // self._pagesize):
2095 firstid = pagenum * self._pagesize
2096 nextfirstid = pagenum * self._pagesize + self._pagesize
2097 if start >= nextfirstid:
2102 page_results = self._cache.get(pagenum)
2103 if page_results is None:
2104 page_results = list(self._pagefunc(pagenum))
2106 self._cache[pagenum] = page_results
2109 start % self._pagesize
2110 if firstid <= start < nextfirstid
2114 ((end - 1) % self._pagesize) + 1
2115 if (end is not None and firstid <= end <= nextfirstid)
2118 if startv != 0 or endv is not None:
2119 page_results = page_results[startv:endv]
2120 res.extend(page_results)
2122 # A little optimization - if current page is not "full", ie. does
2123 # not contain page_size videos then we can assume that this page
2124 # is the last one - there are no more ids on further pages -
2125 # i.e. no need to query again.
2126 if len(page_results) + startv < self._pagesize:
2129 # If we got the whole page, but the next page is not interesting,
2130 # break out early as well
2131 if end == nextfirstid:
2136 class InAdvancePagedList(PagedList):
2137 def __init__(self, pagefunc, pagecount, pagesize):
2138 self._pagefunc = pagefunc
2139 self._pagecount = pagecount
2140 self._pagesize = pagesize
2142 def getslice(self, start=0, end=None):
2144 start_page = start // self._pagesize
2146 self._pagecount if end is None else (end // self._pagesize + 1))
2147 skip_elems = start - start_page * self._pagesize
2148 only_more = None if end is None else end - start
2149 for pagenum in range(start_page, end_page):
2150 page = list(self._pagefunc(pagenum))
2152 page = page[skip_elems:]
2154 if only_more is not None:
2155 if len(page) < only_more:
2156 only_more -= len(page)
2158 page = page[:only_more]
2165 def uppercase_escape(s):
2166 unicode_escape = codecs.getdecoder('unicode_escape')
2168 r'\\U[0-9a-fA-F]{8}',
2169 lambda m: unicode_escape(m.group(0))[0],
2173 def lowercase_escape(s):
2174 unicode_escape = codecs.getdecoder('unicode_escape')
2176 r'\\u[0-9a-fA-F]{4}',
2177 lambda m: unicode_escape(m.group(0))[0],
2181 def escape_rfc3986(s):
2182 """Escape non-ASCII characters as suggested by RFC 3986"""
2183 if sys.version_info < (3, 0) and isinstance(s, compat_str):
2184 s = s.encode('utf-8')
2185 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
2188 def escape_url(url):
2189 """Escape URL as suggested by RFC 3986"""
2190 url_parsed = compat_urllib_parse_urlparse(url)
2191 return url_parsed._replace(
2192 netloc=url_parsed.netloc.encode('idna').decode('ascii'),
2193 path=escape_rfc3986(url_parsed.path),
2194 params=escape_rfc3986(url_parsed.params),
2195 query=escape_rfc3986(url_parsed.query),
2196 fragment=escape_rfc3986(url_parsed.fragment)
2200 def read_batch_urls(batch_fd):
2202 if not isinstance(url, compat_str):
2203 url = url.decode('utf-8', 'replace')
2204 BOM_UTF8 = '\xef\xbb\xbf'
2205 if url.startswith(BOM_UTF8):
2206 url = url[len(BOM_UTF8):]
2208 if url.startswith(('#', ';', ']')):
2212 with contextlib.closing(batch_fd) as fd:
2213 return [url for url in map(fixup, fd) if url]
2216 def urlencode_postdata(*args, **kargs):
2217 return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')
2220 def update_url_query(url, query):
2223 parsed_url = compat_urlparse.urlparse(url)
2224 qs = compat_parse_qs(parsed_url.query)
2226 return compat_urlparse.urlunparse(parsed_url._replace(
2227 query=compat_urllib_parse_urlencode(qs, True)))
2230 def update_Request(req, url=None, data=None, headers={}, query={}):
2231 req_headers = req.headers.copy()
2232 req_headers.update(headers)
2233 req_data = data or req.data
2234 req_url = update_url_query(url or req.get_full_url(), query)
2235 req_get_method = req.get_method()
2236 if req_get_method == 'HEAD':
2237 req_type = HEADRequest
2238 elif req_get_method == 'PUT':
2239 req_type = PUTRequest
2241 req_type = compat_urllib_request.Request
2243 req_url, data=req_data, headers=req_headers,
2244 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
2245 if hasattr(req, 'timeout'):
2246 new_req.timeout = req.timeout
2250 def _multipart_encode_impl(data, boundary):
2251 content_type = 'multipart/form-data; boundary=%s' % boundary
2254 for k, v in data.items():
2255 out += b'--' + boundary.encode('ascii') + b'\r\n'
2256 if isinstance(k, compat_str):
2257 k = k.encode('utf-8')
2258 if isinstance(v, compat_str):
2259 v = v.encode('utf-8')
2260 # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
2261 # suggests sending UTF-8 directly. Firefox sends UTF-8, too
2262 content = b'Content-Disposition: form-data; name="' + k + b'"\r\n\r\n' + v + b'\r\n'
2263 if boundary.encode('ascii') in content:
2264 raise ValueError('Boundary overlaps with data')
2267 out += b'--' + boundary.encode('ascii') + b'--\r\n'
2269 return out, content_type
2272 def multipart_encode(data, boundary=None):
2274 Encode a dict to RFC 7578-compliant form-data
2277 A dict where keys and values can be either Unicode or bytes-like
2280 If specified a Unicode object, it's used as the boundary. Otherwise
2281 a random boundary is generated.
2283 Reference: https://tools.ietf.org/html/rfc7578
2285 has_specified_boundary = boundary is not None
2288 if boundary is None:
2289 boundary = '---------------' + str(random.randrange(0x0fffffff, 0xffffffff))
2292 out, content_type = _multipart_encode_impl(data, boundary)
2295 if has_specified_boundary:
2299 return out, content_type
2302 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
2303 if isinstance(key_or_keys, (list, tuple)):
2304 for key in key_or_keys:
2305 if key not in d or d[key] is None or skip_false_values and not d[key]:
2309 return d.get(key_or_keys, default)
2312 def try_get(src, getter, expected_type=None):
2313 if not isinstance(getter, (list, tuple)):
2318 except (AttributeError, KeyError, TypeError, IndexError):
2321 if expected_type is None or isinstance(v, expected_type):
2325 def merge_dicts(*dicts):
2327 for a_dict in dicts:
2328 for k, v in a_dict.items():
2332 or (isinstance(v, compat_str) and v
2333 and isinstance(merged[k], compat_str)
2334 and not merged[k])):
2339 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
2340 return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
2352 TV_PARENTAL_GUIDELINES = {
2362 def parse_age_limit(s):
2364 return s if 0 <= s <= 21 else None
2365 if not isinstance(s, compat_basestring):
2367 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
2369 return int(m.group('age'))
2371 return US_RATINGS[s]
2372 m = re.match(r'^TV[_-]?(%s)$' % '|'.join(k[3:] for k in TV_PARENTAL_GUIDELINES), s)
2374 return TV_PARENTAL_GUIDELINES['TV-' + m.group(1)]
2378 def strip_jsonp(code):
2381 (?:window\.)?(?P<func_name>[a-zA-Z0-9_.$]*)
2382 (?:\s*&&\s*(?P=func_name))?
2383 \s*\(\s*(?P<callback_data>.*)\);?
2384 \s*?(?://[^\n]*)*$''',
2385 r'\g<callback_data>', code)
2388 def js_to_json(code):
2389 COMMENT_RE = r'/\*(?:(?!\*/).)*?\*/|//[^\n]*'
2390 SKIP_RE = r'\s*(?:{comment})?\s*'.format(comment=COMMENT_RE)
2392 (r'(?s)^(0[xX][0-9a-fA-F]+){skip}:?$'.format(skip=SKIP_RE), 16),
2393 (r'(?s)^(0+[0-7]+){skip}:?$'.format(skip=SKIP_RE), 8),
2398 if v in ('true', 'false', 'null'):
2400 elif v.startswith('/*') or v.startswith('//') or v == ',':
2403 if v[0] in ("'", '"'):
2404 v = re.sub(r'(?s)\\.|"', lambda m: {
2409 }.get(m.group(0), m.group(0)), v[1:-1])
2411 for regex, base in INTEGER_TABLE:
2412 im = re.match(regex, v)
2414 i = int(im.group(1), base)
2415 return '"%d":' % i if v.endswith(':') else '%d' % i
2419 return re.sub(r'''(?sx)
2420 "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
2421 '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
2422 {comment}|,(?={skip}[\]}}])|
2423 (?:(?<![0-9])[eE]|[a-df-zA-DF-Z_])[.a-zA-Z_0-9]*|
2424 \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:{skip}:)?|
2426 '''.format(comment=COMMENT_RE, skip=SKIP_RE), fix_kv, code)
2429 def qualities(quality_ids):
2430 """ Get a numeric quality value out of a list of possible values """
2433 return quality_ids.index(qid)
2439 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
2442 def limit_length(s, length):
2443 """ Add ellipses to overly long strings """
2448 return s[:length - len(ELLIPSES)] + ELLIPSES
2452 def version_tuple(v):
2453 return tuple(int(e) for e in re.split(r'[-.]', v))
2456 def is_outdated_version(version, limit, assume_new=True):
2458 return not assume_new
2460 return version_tuple(version) < version_tuple(limit)
2462 return not assume_new
2465 def ytdl_is_updateable():
2466 """ Returns if youtube-dl can be updated with -U """
2467 from zipimport import zipimporter
2469 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
2472 def args_to_str(args):
2473 # Get a short string representation for a subprocess command
2474 return ' '.join(compat_shlex_quote(a) for a in args)
2477 def error_to_compat_str(err):
2479 # On python 2 error byte string must be decoded with proper
2480 # encoding rather than ascii
2481 if sys.version_info[0] < 3:
2482 err_str = err_str.decode(preferredencoding())
2486 def mimetype2ext(mt):
2492 # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
2493 # it's the most popular one
2494 'audio/mpeg': 'mp3',
2499 _, _, res = mt.rpartition('/')
2500 res = res.split(';')[0].strip().lower()
2504 'smptett+xml': 'tt',
2508 'x-mp4-fragmented': 'mp4',
2509 'x-ms-sami': 'sami',
2512 'x-mpegurl': 'm3u8',
2513 'vnd.apple.mpegurl': 'm3u8',
2517 'vnd.ms-sstr+xml': 'ism',
2523 def parse_codecs(codecs_str):
2524 # http://tools.ietf.org/html/rfc6381
2527 splited_codecs = list(filter(None, map(
2528 lambda str: str.strip(), codecs_str.strip().strip(',').split(','))))
2529 vcodec, acodec = None, None
2530 for full_codec in splited_codecs:
2531 codec = full_codec.split('.')[0]
2532 if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01', 'theora'):
2535 elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
2539 write_string('WARNING: Unknown codec %s\n' % full_codec, sys.stderr)
2540 if not vcodec and not acodec:
2541 if len(splited_codecs) == 2:
2543 'vcodec': splited_codecs[0],
2544 'acodec': splited_codecs[1],
2548 'vcodec': vcodec or 'none',
2549 'acodec': acodec or 'none',
2554 def urlhandle_detect_ext(url_handle):
2555 getheader = url_handle.headers.get
2557 cd = getheader('Content-Disposition')
2559 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
2561 e = determine_ext(m.group('filename'), default_ext=None)
2565 return mimetype2ext(getheader('Content-Type'))
2568 def encode_data_uri(data, mime_type):
2569 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
2572 def age_restricted(content_limit, age_limit):
2573 """ Returns True iff the content should be blocked """
2575 if age_limit is None: # No limit set
2577 if content_limit is None:
2578 return False # Content available for everyone
2579 return age_limit < content_limit
2582 def is_html(first_bytes):
2583 """ Detect whether a file contains HTML by examining its first bytes. """
2586 (b'\xef\xbb\xbf', 'utf-8'),
2587 (b'\x00\x00\xfe\xff', 'utf-32-be'),
2588 (b'\xff\xfe\x00\x00', 'utf-32-le'),
2589 (b'\xff\xfe', 'utf-16-le'),
2590 (b'\xfe\xff', 'utf-16-be'),
2592 for bom, enc in BOMS:
2593 if first_bytes.startswith(bom):
2594 s = first_bytes[len(bom):].decode(enc, 'replace')
2597 s = first_bytes.decode('utf-8', 'replace')
2599 return re.match(r'^\s*<', s)
2602 def determine_protocol(info_dict):
2603 protocol = info_dict.get('protocol')
2604 if protocol is not None:
2607 url = info_dict['url']
2608 if url.startswith('rtmp'):
2610 elif url.startswith('mms'):
2612 elif url.startswith('rtsp'):
2615 ext = determine_ext(url)
2621 return compat_urllib_parse_urlparse(url).scheme
2624 def render_table(header_row, data):
2625 """ Render a list of rows, each as a list of values """
2626 table = [header_row] + data
2627 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
2628 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
2629 return '\n'.join(format_str % tuple(row) for row in table)
2632 def _match_one(filter_part, dct):
2633 COMPARISON_OPERATORS = {
2641 operator_rex = re.compile(r'''(?x)\s*
2643 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
2645 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
2646 (?P<quote>["\'])(?P<quotedstrval>(?:\\.|(?!(?P=quote)|\\).)+?)(?P=quote)|
2647 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
2650 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
2651 m = operator_rex.search(filter_part)
2653 op = COMPARISON_OPERATORS[m.group('op')]
2654 actual_value = dct.get(m.group('key'))
2655 if (m.group('quotedstrval') is not None
2656 or m.group('strval') is not None
2657 # If the original field is a string and matching comparisonvalue is
2658 # a number we should respect the origin of the original field
2659 # and process comparison value as a string (see
2660 # https://github.com/ytdl-org/youtube-dl/issues/11082).
2661 or actual_value is not None and m.group('intval') is not None
2662 and isinstance(actual_value, compat_str)):
2663 if m.group('op') not in ('=', '!='):
2665 'Operator %s does not support string values!' % m.group('op'))
2666 comparison_value = m.group('quotedstrval') or m.group('strval') or m.group('intval')
2667 quote = m.group('quote')
2668 if quote is not None:
2669 comparison_value = comparison_value.replace(r'\%s' % quote, quote)
2672 comparison_value = int(m.group('intval'))
2674 comparison_value = parse_filesize(m.group('intval'))
2675 if comparison_value is None:
2676 comparison_value = parse_filesize(m.group('intval') + 'B')
2677 if comparison_value is None:
2679 'Invalid integer value %r in filter part %r' % (
2680 m.group('intval'), filter_part))
2681 if actual_value is None:
2682 return m.group('none_inclusive')
2683 return op(actual_value, comparison_value)
2686 '': lambda v: (v is True) if isinstance(v, bool) else (v is not None),
2687 '!': lambda v: (v is False) if isinstance(v, bool) else (v is None),
2689 operator_rex = re.compile(r'''(?x)\s*
2690 (?P<op>%s)\s*(?P<key>[a-z_]+)
2692 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
2693 m = operator_rex.search(filter_part)
2695 op = UNARY_OPERATORS[m.group('op')]
2696 actual_value = dct.get(m.group('key'))
2697 return op(actual_value)
2699 raise ValueError('Invalid filter part %r' % filter_part)
2702 def match_str(filter_str, dct):
2703 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
2706 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
2709 def match_filter_func(filter_str):
2710 def _match_func(info_dict):
2711 if match_str(filter_str, info_dict):
2714 video_title = info_dict.get('title', info_dict.get('id', 'video'))
2715 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
2719 def parse_dfxp_time_expr(time_expr):
2723 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
2725 return float(mobj.group('time_offset'))
2727 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
2729 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
2732 def srt_subtitles_timecode(seconds):
2733 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
2736 def dfxp2srt(dfxp_data):
2738 @param dfxp_data A bytes-like object containing DFXP data
2739 @returns A unicode object containing converted SRT data
2741 LEGACY_NAMESPACES = (
2742 (b'http://www.w3.org/ns/ttml', [
2743 b'http://www.w3.org/2004/11/ttaf1',
2744 b'http://www.w3.org/2006/04/ttaf1',
2745 b'http://www.w3.org/2006/10/ttaf1',
2747 (b'http://www.w3.org/ns/ttml#styling', [
2748 b'http://www.w3.org/ns/ttml#style',
2752 SUPPORTED_STYLING = [
2761 _x = functools.partial(xpath_with_ns, ns_map={
2762 'xml': 'http://www.w3.org/XML/1998/namespace',
2763 'ttml': 'http://www.w3.org/ns/ttml',
2764 'tts': 'http://www.w3.org/ns/ttml#styling',
2770 class TTMLPElementParser(object):
2772 _unclosed_elements = []
2773 _applied_styles = []
2775 def start(self, tag, attrib):
2776 if tag in (_x('ttml:br'), 'br'):
2779 unclosed_elements = []
2781 element_style_id = attrib.get('style')
2783 style.update(default_style)
2784 if element_style_id:
2785 style.update(styles.get(element_style_id, {}))
2786 for prop in SUPPORTED_STYLING:
2787 prop_val = attrib.get(_x('tts:' + prop))
2789 style[prop] = prop_val
2792 for k, v in sorted(style.items()):
2793 if self._applied_styles and self._applied_styles[-1].get(k) == v:
2796 font += ' color="%s"' % v
2797 elif k == 'fontSize':
2798 font += ' size="%s"' % v
2799 elif k == 'fontFamily':
2800 font += ' face="%s"' % v
2801 elif k == 'fontWeight' and v == 'bold':
2803 unclosed_elements.append('b')
2804 elif k == 'fontStyle' and v == 'italic':
2806 unclosed_elements.append('i')
2807 elif k == 'textDecoration' and v == 'underline':
2809 unclosed_elements.append('u')
2811 self._out += '<font' + font + '>'
2812 unclosed_elements.append('font')
2814 if self._applied_styles:
2815 applied_style.update(self._applied_styles[-1])
2816 applied_style.update(style)
2817 self._applied_styles.append(applied_style)
2818 self._unclosed_elements.append(unclosed_elements)
2821 if tag not in (_x('ttml:br'), 'br'):
2822 unclosed_elements = self._unclosed_elements.pop()
2823 for element in reversed(unclosed_elements):
2824 self._out += '</%s>' % element
2825 if unclosed_elements and self._applied_styles:
2826 self._applied_styles.pop()
2828 def data(self, data):
2832 return self._out.strip()
2834 def parse_node(node):
2835 target = TTMLPElementParser()
2836 parser = xml.etree.ElementTree.XMLParser(target=target)
2837 parser.feed(xml.etree.ElementTree.tostring(node))
2838 return parser.close()
2840 for k, v in LEGACY_NAMESPACES:
2842 dfxp_data = dfxp_data.replace(ns, k)
2844 dfxp = compat_etree_fromstring(dfxp_data)
2846 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
2849 raise ValueError('Invalid dfxp/TTML subtitle')
2853 for style in dfxp.findall(_x('.//ttml:style')):
2854 style_id = style.get('id') or style.get(_x('xml:id'))
2857 parent_style_id = style.get('style')
2859 if parent_style_id not in styles:
2862 styles[style_id] = styles[parent_style_id].copy()
2863 for prop in SUPPORTED_STYLING:
2864 prop_val = style.get(_x('tts:' + prop))
2866 styles.setdefault(style_id, {})[prop] = prop_val
2872 for p in ('body', 'div'):
2873 ele = xpath_element(dfxp, [_x('.//ttml:' + p), './/' + p])
2876 style = styles.get(ele.get('style'))
2879 default_style.update(style)
2881 for para, index in zip(paras, itertools.count(1)):
2882 begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
2883 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
2884 dur = parse_dfxp_time_expr(para.attrib.get('dur'))
2885 if begin_time is None:
2890 end_time = begin_time + dur
2891 out.append('%d\n%s --> %s\n%s\n\n' % (
2893 srt_subtitles_timecode(begin_time),
2894 srt_subtitles_timecode(end_time),
2900 def cli_option(params, command_option, param):
2901 param = params.get(param)
2903 param = compat_str(param)
2904 return [command_option, param] if param is not None else []
2907 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
2908 param = params.get(param)
2911 assert isinstance(param, bool)
2913 return [command_option + separator + (true_value if param else false_value)]
2914 return [command_option, true_value if param else false_value]
2917 def cli_valueless_option(params, command_option, param, expected_value=True):
2918 param = params.get(param)
2919 return [command_option] if param == expected_value else []
2922 def cli_configuration_args(params, param, default=[]):
2923 ex_args = params.get(param)
2926 assert isinstance(ex_args, list)
2930 class ISO639Utils(object):
2931 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2990 'iw': 'heb', # Replaced by he in 1989 revision
3000 'in': 'ind', # Replaced by id in 1989 revision
3115 'ji': 'yid', # Replaced by yi in 1989 revision
3123 def short2long(cls, code):
3124 """Convert language code from ISO 639-1 to ISO 639-2/T"""
3125 return cls._lang_map.get(code[:2])
3128 def long2short(cls, code):
3129 """Convert language code from ISO 639-2/T to ISO 639-1"""
3130 for short_name, long_name in cls._lang_map.items():
3131 if long_name == code:
3135 class ISO3166Utils(object):
3136 # From http://data.okfn.org/data/core/country-list
3138 'AF': 'Afghanistan',
3139 'AX': 'Åland Islands',
3142 'AS': 'American Samoa',
3147 'AG': 'Antigua and Barbuda',
3164 'BO': 'Bolivia, Plurinational State of',
3165 'BQ': 'Bonaire, Sint Eustatius and Saba',
3166 'BA': 'Bosnia and Herzegovina',
3168 'BV': 'Bouvet Island',
3170 'IO': 'British Indian Ocean Territory',
3171 'BN': 'Brunei Darussalam',
3173 'BF': 'Burkina Faso',
3179 'KY': 'Cayman Islands',
3180 'CF': 'Central African Republic',
3184 'CX': 'Christmas Island',
3185 'CC': 'Cocos (Keeling) Islands',
3189 'CD': 'Congo, the Democratic Republic of the',
3190 'CK': 'Cook Islands',
3192 'CI': 'Côte d\'Ivoire',
3197 'CZ': 'Czech Republic',
3201 'DO': 'Dominican Republic',
3204 'SV': 'El Salvador',
3205 'GQ': 'Equatorial Guinea',
3209 'FK': 'Falkland Islands (Malvinas)',
3210 'FO': 'Faroe Islands',
3214 'GF': 'French Guiana',
3215 'PF': 'French Polynesia',
3216 'TF': 'French Southern Territories',
3231 'GW': 'Guinea-Bissau',
3234 'HM': 'Heard Island and McDonald Islands',
3235 'VA': 'Holy See (Vatican City State)',
3242 'IR': 'Iran, Islamic Republic of',
3245 'IM': 'Isle of Man',
3255 'KP': 'Korea, Democratic People\'s Republic of',
3256 'KR': 'Korea, Republic of',
3259 'LA': 'Lao People\'s Democratic Republic',
3265 'LI': 'Liechtenstein',
3269 'MK': 'Macedonia, the Former Yugoslav Republic of',
3276 'MH': 'Marshall Islands',
3282 'FM': 'Micronesia, Federated States of',
3283 'MD': 'Moldova, Republic of',
3294 'NL': 'Netherlands',
3295 'NC': 'New Caledonia',
3296 'NZ': 'New Zealand',
3301 'NF': 'Norfolk Island',
3302 'MP': 'Northern Mariana Islands',
3307 'PS': 'Palestine, State of',
3309 'PG': 'Papua New Guinea',
3312 'PH': 'Philippines',
3316 'PR': 'Puerto Rico',
3320 'RU': 'Russian Federation',
3322 'BL': 'Saint Barthélemy',
3323 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
3324 'KN': 'Saint Kitts and Nevis',
3325 'LC': 'Saint Lucia',
3326 'MF': 'Saint Martin (French part)',
3327 'PM': 'Saint Pierre and Miquelon',
3328 'VC': 'Saint Vincent and the Grenadines',
3331 'ST': 'Sao Tome and Principe',
3332 'SA': 'Saudi Arabia',
3336 'SL': 'Sierra Leone',
3338 'SX': 'Sint Maarten (Dutch part)',
3341 'SB': 'Solomon Islands',
3343 'ZA': 'South Africa',
3344 'GS': 'South Georgia and the South Sandwich Islands',
3345 'SS': 'South Sudan',
3350 'SJ': 'Svalbard and Jan Mayen',
3353 'CH': 'Switzerland',
3354 'SY': 'Syrian Arab Republic',
3355 'TW': 'Taiwan, Province of China',
3357 'TZ': 'Tanzania, United Republic of',
3359 'TL': 'Timor-Leste',
3363 'TT': 'Trinidad and Tobago',
3366 'TM': 'Turkmenistan',
3367 'TC': 'Turks and Caicos Islands',
3371 'AE': 'United Arab Emirates',
3372 'GB': 'United Kingdom',
3373 'US': 'United States',
3374 'UM': 'United States Minor Outlying Islands',
3378 'VE': 'Venezuela, Bolivarian Republic of',
3380 'VG': 'Virgin Islands, British',
3381 'VI': 'Virgin Islands, U.S.',
3382 'WF': 'Wallis and Futuna',
3383 'EH': 'Western Sahara',
3390 def short2full(cls, code):
3391 """Convert an ISO 3166-2 country code to the corresponding full name"""
3392 return cls._country_map.get(code.upper())
3395 class GeoUtils(object):
3396 # Major IPv4 address blocks per country
3398 'AD': '85.94.160.0/19',
3399 'AE': '94.200.0.0/13',
3400 'AF': '149.54.0.0/17',
3401 'AG': '209.59.64.0/18',
3402 'AI': '204.14.248.0/21',
3403 'AL': '46.99.0.0/16',
3404 'AM': '46.70.0.0/15',
3405 'AO': '105.168.0.0/13',
3406 'AP': '159.117.192.0/21',
3407 'AR': '181.0.0.0/12',
3408 'AS': '202.70.112.0/20',
3409 'AT': '84.112.0.0/13',
3410 'AU': '1.128.0.0/11',
3411 'AW': '181.41.0.0/18',
3412 'AZ': '5.191.0.0/16',
3413 'BA': '31.176.128.0/17',
3414 'BB': '65.48.128.0/17',
3415 'BD': '114.130.0.0/16',
3417 'BF': '129.45.128.0/17',
3418 'BG': '95.42.0.0/15',
3419 'BH': '37.131.0.0/17',
3420 'BI': '154.117.192.0/18',
3421 'BJ': '137.255.0.0/16',
3422 'BL': '192.131.134.0/24',
3423 'BM': '196.12.64.0/18',
3424 'BN': '156.31.0.0/16',
3425 'BO': '161.56.0.0/16',
3426 'BQ': '161.0.80.0/20',
3427 'BR': '152.240.0.0/12',
3428 'BS': '24.51.64.0/18',
3429 'BT': '119.2.96.0/19',
3430 'BW': '168.167.0.0/16',
3431 'BY': '178.120.0.0/13',
3432 'BZ': '179.42.192.0/18',
3433 'CA': '99.224.0.0/11',
3434 'CD': '41.243.0.0/16',
3435 'CF': '196.32.200.0/21',
3436 'CG': '197.214.128.0/17',
3437 'CH': '85.0.0.0/13',
3438 'CI': '154.232.0.0/14',
3439 'CK': '202.65.32.0/19',
3440 'CL': '152.172.0.0/14',
3441 'CM': '165.210.0.0/15',
3442 'CN': '36.128.0.0/10',
3443 'CO': '181.240.0.0/12',
3444 'CR': '201.192.0.0/12',
3445 'CU': '152.206.0.0/15',
3446 'CV': '165.90.96.0/19',
3447 'CW': '190.88.128.0/17',
3448 'CY': '46.198.0.0/15',
3449 'CZ': '88.100.0.0/14',
3451 'DJ': '197.241.0.0/17',
3452 'DK': '87.48.0.0/12',
3453 'DM': '192.243.48.0/20',
3454 'DO': '152.166.0.0/15',
3455 'DZ': '41.96.0.0/12',
3456 'EC': '186.68.0.0/15',
3457 'EE': '90.190.0.0/15',
3458 'EG': '156.160.0.0/11',
3459 'ER': '196.200.96.0/20',
3460 'ES': '88.0.0.0/11',
3461 'ET': '196.188.0.0/14',
3462 'EU': '2.16.0.0/13',
3463 'FI': '91.152.0.0/13',
3464 'FJ': '144.120.0.0/16',
3465 'FM': '119.252.112.0/20',
3466 'FO': '88.85.32.0/19',
3468 'GA': '41.158.0.0/15',
3470 'GD': '74.122.88.0/21',
3471 'GE': '31.146.0.0/16',
3472 'GF': '161.22.64.0/18',
3473 'GG': '62.68.160.0/19',
3474 'GH': '45.208.0.0/14',
3475 'GI': '85.115.128.0/19',
3476 'GL': '88.83.0.0/19',
3477 'GM': '160.182.0.0/15',
3478 'GN': '197.149.192.0/18',
3479 'GP': '104.250.0.0/19',
3480 'GQ': '105.235.224.0/20',
3481 'GR': '94.64.0.0/13',
3482 'GT': '168.234.0.0/16',
3483 'GU': '168.123.0.0/16',
3484 'GW': '197.214.80.0/20',
3485 'GY': '181.41.64.0/18',
3486 'HK': '113.252.0.0/14',
3487 'HN': '181.210.0.0/16',
3488 'HR': '93.136.0.0/13',
3489 'HT': '148.102.128.0/17',
3490 'HU': '84.0.0.0/14',
3491 'ID': '39.192.0.0/10',
3492 'IE': '87.32.0.0/12',
3493 'IL': '79.176.0.0/13',
3494 'IM': '5.62.80.0/20',
3495 'IN': '117.192.0.0/10',
3496 'IO': '203.83.48.0/21',
3497 'IQ': '37.236.0.0/14',
3498 'IR': '2.176.0.0/12',
3499 'IS': '82.221.0.0/16',
3500 'IT': '79.0.0.0/10',
3501 'JE': '87.244.64.0/18',
3502 'JM': '72.27.0.0/17',
3503 'JO': '176.29.0.0/16',
3504 'JP': '126.0.0.0/8',
3505 'KE': '105.48.0.0/12',
3506 'KG': '158.181.128.0/17',
3507 'KH': '36.37.128.0/17',
3508 'KI': '103.25.140.0/22',
3509 'KM': '197.255.224.0/20',
3510 'KN': '198.32.32.0/19',
3511 'KP': '175.45.176.0/22',
3512 'KR': '175.192.0.0/10',
3513 'KW': '37.36.0.0/14',
3514 'KY': '64.96.0.0/15',
3515 'KZ': '2.72.0.0/13',
3516 'LA': '115.84.64.0/18',
3517 'LB': '178.135.0.0/16',
3518 'LC': '192.147.231.0/24',
3519 'LI': '82.117.0.0/19',
3520 'LK': '112.134.0.0/15',
3521 'LR': '41.86.0.0/19',
3522 'LS': '129.232.0.0/17',
3523 'LT': '78.56.0.0/13',
3524 'LU': '188.42.0.0/16',
3525 'LV': '46.109.0.0/16',
3526 'LY': '41.252.0.0/14',
3527 'MA': '105.128.0.0/11',
3528 'MC': '88.209.64.0/18',
3529 'MD': '37.246.0.0/16',
3530 'ME': '178.175.0.0/17',
3531 'MF': '74.112.232.0/21',
3532 'MG': '154.126.0.0/17',
3533 'MH': '117.103.88.0/21',
3534 'MK': '77.28.0.0/15',
3535 'ML': '154.118.128.0/18',
3536 'MM': '37.111.0.0/17',
3537 'MN': '49.0.128.0/17',
3538 'MO': '60.246.0.0/16',
3539 'MP': '202.88.64.0/20',
3540 'MQ': '109.203.224.0/19',
3541 'MR': '41.188.64.0/18',
3542 'MS': '208.90.112.0/22',
3543 'MT': '46.11.0.0/16',
3544 'MU': '105.16.0.0/12',
3545 'MV': '27.114.128.0/18',
3546 'MW': '105.234.0.0/16',
3547 'MX': '187.192.0.0/11',
3548 'MY': '175.136.0.0/13',
3549 'MZ': '197.218.0.0/15',
3550 'NA': '41.182.0.0/16',
3551 'NC': '101.101.0.0/18',
3552 'NE': '197.214.0.0/18',
3553 'NF': '203.17.240.0/22',
3554 'NG': '105.112.0.0/12',
3555 'NI': '186.76.0.0/15',
3556 'NL': '145.96.0.0/11',
3557 'NO': '84.208.0.0/13',
3558 'NP': '36.252.0.0/15',
3559 'NR': '203.98.224.0/19',
3560 'NU': '49.156.48.0/22',
3561 'NZ': '49.224.0.0/14',
3562 'OM': '5.36.0.0/15',
3563 'PA': '186.72.0.0/15',
3564 'PE': '186.160.0.0/14',
3565 'PF': '123.50.64.0/18',
3566 'PG': '124.240.192.0/19',
3567 'PH': '49.144.0.0/13',
3568 'PK': '39.32.0.0/11',
3569 'PL': '83.0.0.0/11',
3570 'PM': '70.36.0.0/20',
3571 'PR': '66.50.0.0/16',
3572 'PS': '188.161.0.0/16',
3573 'PT': '85.240.0.0/13',
3574 'PW': '202.124.224.0/20',
3575 'PY': '181.120.0.0/14',
3576 'QA': '37.210.0.0/15',
3577 'RE': '139.26.0.0/16',
3578 'RO': '79.112.0.0/13',
3579 'RS': '178.220.0.0/14',
3580 'RU': '5.136.0.0/13',
3581 'RW': '105.178.0.0/15',
3582 'SA': '188.48.0.0/13',
3583 'SB': '202.1.160.0/19',
3584 'SC': '154.192.0.0/11',
3585 'SD': '154.96.0.0/13',
3586 'SE': '78.64.0.0/12',
3587 'SG': '152.56.0.0/14',
3588 'SI': '188.196.0.0/14',
3589 'SK': '78.98.0.0/15',
3590 'SL': '197.215.0.0/17',
3591 'SM': '89.186.32.0/19',
3592 'SN': '41.82.0.0/15',
3593 'SO': '197.220.64.0/19',
3594 'SR': '186.179.128.0/17',
3595 'SS': '105.235.208.0/21',
3596 'ST': '197.159.160.0/19',
3597 'SV': '168.243.0.0/16',
3598 'SX': '190.102.0.0/20',
3600 'SZ': '41.84.224.0/19',
3601 'TC': '65.255.48.0/20',
3602 'TD': '154.68.128.0/19',
3603 'TG': '196.168.0.0/14',
3604 'TH': '171.96.0.0/13',
3605 'TJ': '85.9.128.0/18',
3606 'TK': '27.96.24.0/21',
3607 'TL': '180.189.160.0/20',
3608 'TM': '95.85.96.0/19',
3609 'TN': '197.0.0.0/11',
3610 'TO': '175.176.144.0/21',
3611 'TR': '78.160.0.0/11',
3612 'TT': '186.44.0.0/15',
3613 'TV': '202.2.96.0/19',
3614 'TW': '120.96.0.0/11',
3615 'TZ': '156.156.0.0/14',
3616 'UA': '93.72.0.0/13',
3617 'UG': '154.224.0.0/13',
3619 'UY': '167.56.0.0/13',
3620 'UZ': '82.215.64.0/18',
3621 'VA': '212.77.0.0/19',
3622 'VC': '24.92.144.0/20',
3623 'VE': '186.88.0.0/13',
3624 'VG': '172.103.64.0/18',
3625 'VI': '146.226.0.0/16',
3626 'VN': '14.160.0.0/11',
3627 'VU': '202.80.32.0/20',
3628 'WF': '117.20.32.0/21',
3629 'WS': '202.4.32.0/19',
3630 'YE': '134.35.0.0/16',
3631 'YT': '41.242.116.0/22',
3632 'ZA': '41.0.0.0/11',
3633 'ZM': '165.56.0.0/13',
3634 'ZW': '41.85.192.0/19',
3638 def random_ipv4(cls, code_or_block):
3639 if len(code_or_block) == 2:
3640 block = cls._country_ip_map.get(code_or_block.upper())
3644 block = code_or_block
3645 addr, preflen = block.split('/')
3646 addr_min = compat_struct_unpack('!L', socket.inet_aton(addr))[0]
3647 addr_max = addr_min | (0xffffffff >> int(preflen))
3648 return compat_str(socket.inet_ntoa(
3649 compat_struct_pack('!L', random.randint(addr_min, addr_max))))
3652 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
3653 def __init__(self, proxies=None):
3654 # Set default handlers
3655 for type in ('http', 'https'):
3656 setattr(self, '%s_open' % type,
3657 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
3658 meth(r, proxy, type))
3659 compat_urllib_request.ProxyHandler.__init__(self, proxies)
3661 def proxy_open(self, req, proxy, type):
3662 req_proxy = req.headers.get('Ytdl-request-proxy')
3663 if req_proxy is not None:
3665 del req.headers['Ytdl-request-proxy']
3667 if proxy == '__noproxy__':
3668 return None # No Proxy
3669 if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
3670 req.add_header('Ytdl-socks-proxy', proxy)
3671 # youtube-dl's http/https handlers do wrapping the socket with socks
3673 return compat_urllib_request.ProxyHandler.proxy_open(
3674 self, req, proxy, type)
3677 # Both long_to_bytes and bytes_to_long are adapted from PyCrypto, which is
3678 # released into Public Domain
3679 # https://github.com/dlitz/pycrypto/blob/master/lib/Crypto/Util/number.py#L387
3681 def long_to_bytes(n, blocksize=0):
3682 """long_to_bytes(n:long, blocksize:int) : string
3683 Convert a long integer to a byte string.
3685 If optional blocksize is given and greater than zero, pad the front of the
3686 byte string with binary zeros so that the length is a multiple of
3689 # after much testing, this algorithm was deemed to be the fastest
3693 s = compat_struct_pack('>I', n & 0xffffffff) + s
3695 # strip off leading zeros
3696 for i in range(len(s)):
3697 if s[i] != b'\000'[0]:
3700 # only happens when n == 0
3704 # add back some pad bytes. this could be done more efficiently w.r.t. the
3705 # de-padding being done above, but sigh...
3706 if blocksize > 0 and len(s) % blocksize:
3707 s = (blocksize - len(s) % blocksize) * b'\000' + s
3711 def bytes_to_long(s):
3712 """bytes_to_long(string) : long
3713 Convert a byte string to a long integer.
3715 This is (essentially) the inverse of long_to_bytes().
3720 extra = (4 - length % 4)
3721 s = b'\000' * extra + s
3722 length = length + extra
3723 for i in range(0, length, 4):
3724 acc = (acc << 32) + compat_struct_unpack('>I', s[i:i + 4])[0]
3728 def ohdave_rsa_encrypt(data, exponent, modulus):
3730 Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
3733 data: data to encrypt, bytes-like object
3734 exponent, modulus: parameter e and N of RSA algorithm, both integer
3735 Output: hex string of encrypted data
3737 Limitation: supports one block encryption only
3740 payload = int(binascii.hexlify(data[::-1]), 16)
3741 encrypted = pow(payload, exponent, modulus)
3742 return '%x' % encrypted
3745 def pkcs1pad(data, length):
3747 Padding input data with PKCS#1 scheme
3749 @param {int[]} data input data
3750 @param {int} length target length
3751 @returns {int[]} padded data
3753 if len(data) > length - 11:
3754 raise ValueError('Input data too long for PKCS#1 padding')
3756 pseudo_random = [random.randint(0, 254) for _ in range(length - len(data) - 3)]
3757 return [0, 2] + pseudo_random + [0] + data
3760 def encode_base_n(num, n, table=None):
3761 FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
3763 table = FULL_TABLE[:n]
3766 raise ValueError('base %d exceeds table length %d' % (n, len(table)))
3773 ret = table[num % n] + ret
3778 def decode_packed_codes(code):
3779 mobj = re.search(PACKED_CODES_RE, code)
3780 obfucasted_code, base, count, symbols = mobj.groups()
3783 symbols = symbols.split('|')
3788 base_n_count = encode_base_n(count, base)
3789 symbol_table[base_n_count] = symbols[count] or base_n_count
3792 r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
3796 def parse_m3u8_attributes(attrib):
3798 for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
3799 if val.startswith('"'):
3805 def urshift(val, n):
3806 return val >> n if val >= 0 else (val + 0x100000000) >> n
3809 # Based on png2str() written by @gdkchan and improved by @yokrysty
3810 # Originally posted at https://github.com/ytdl-org/youtube-dl/issues/9706
3811 def decode_png(png_data):
3812 # Reference: https://www.w3.org/TR/PNG/
3813 header = png_data[8:]
3815 if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
3816 raise IOError('Not a valid PNG file.')
3818 int_map = {1: '>B', 2: '>H', 4: '>I'}
3819 unpack_integer = lambda x: compat_struct_unpack(int_map[len(x)], x)[0]
3824 length = unpack_integer(header[:4])
3827 chunk_type = header[:4]
3830 chunk_data = header[:length]
3831 header = header[length:]
3833 header = header[4:] # Skip CRC
3841 ihdr = chunks[0]['data']
3843 width = unpack_integer(ihdr[:4])
3844 height = unpack_integer(ihdr[4:8])
3848 for chunk in chunks:
3849 if chunk['type'] == b'IDAT':
3850 idat += chunk['data']
3853 raise IOError('Unable to read PNG data.')
3855 decompressed_data = bytearray(zlib.decompress(idat))
3860 def _get_pixel(idx):
3865 for y in range(height):
3866 basePos = y * (1 + stride)
3867 filter_type = decompressed_data[basePos]
3871 pixels.append(current_row)
3873 for x in range(stride):
3874 color = decompressed_data[1 + basePos + x]
3875 basex = y * stride + x
3880 left = _get_pixel(basex - 3)
3882 up = _get_pixel(basex - stride)
3884 if filter_type == 1: # Sub
3885 color = (color + left) & 0xff
3886 elif filter_type == 2: # Up
3887 color = (color + up) & 0xff
3888 elif filter_type == 3: # Average
3889 color = (color + ((left + up) >> 1)) & 0xff
3890 elif filter_type == 4: # Paeth
3896 c = _get_pixel(basex - stride - 3)
3904 if pa <= pb and pa <= pc:
3905 color = (color + a) & 0xff
3907 color = (color + b) & 0xff
3909 color = (color + c) & 0xff
3911 current_row.append(color)
3913 return width, height, pixels
3916 def write_xattr(path, key, value):
3917 # This mess below finds the best xattr tool for the job
3919 # try the pyxattr module...
3922 if hasattr(xattr, 'set'): # pyxattr
3923 # Unicode arguments are not supported in python-pyxattr until
3925 # See https://github.com/ytdl-org/youtube-dl/issues/5498
3926 pyxattr_required_version = '0.5.0'
3927 if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
3928 # TODO: fallback to CLI tools
3929 raise XAttrUnavailableError(
3930 'python-pyxattr is detected but is too old. '
3931 'youtube-dl requires %s or above while your version is %s. '
3932 'Falling back to other xattr implementations' % (
3933 pyxattr_required_version, xattr.__version__))
3935 setxattr = xattr.set
3937 setxattr = xattr.setxattr
3940 setxattr(path, key, value)
3941 except EnvironmentError as e:
3942 raise XAttrMetadataError(e.errno, e.strerror)
3945 if compat_os_name == 'nt':
3946 # Write xattrs to NTFS Alternate Data Streams:
3947 # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
3948 assert ':' not in key
3949 assert os.path.exists(path)
3951 ads_fn = path + ':' + key
3953 with open(ads_fn, 'wb') as f:
3955 except EnvironmentError as e:
3956 raise XAttrMetadataError(e.errno, e.strerror)
3958 user_has_setfattr = check_executable('setfattr', ['--version'])
3959 user_has_xattr = check_executable('xattr', ['-h'])
3961 if user_has_setfattr or user_has_xattr:
3963 value = value.decode('utf-8')
3964 if user_has_setfattr:
3965 executable = 'setfattr'
3966 opts = ['-n', key, '-v', value]
3967 elif user_has_xattr:
3968 executable = 'xattr'
3969 opts = ['-w', key, value]
3971 cmd = ([encodeFilename(executable, True)]
3972 + [encodeArgument(o) for o in opts]
3973 + [encodeFilename(path, True)])
3976 p = subprocess.Popen(
3977 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
3978 except EnvironmentError as e:
3979 raise XAttrMetadataError(e.errno, e.strerror)
3980 stdout, stderr = p.communicate()
3981 stderr = stderr.decode('utf-8', 'replace')
3982 if p.returncode != 0:
3983 raise XAttrMetadataError(p.returncode, stderr)
3986 # On Unix, and can't find pyxattr, setfattr, or xattr.
3987 if sys.platform.startswith('linux'):
3988 raise XAttrUnavailableError(
3989 "Couldn't find a tool to set the xattrs. "
3990 "Install either the python 'pyxattr' or 'xattr' "
3991 "modules, or the GNU 'attr' package "
3992 "(which contains the 'setfattr' tool).")
3994 raise XAttrUnavailableError(
3995 "Couldn't find a tool to set the xattrs. "
3996 "Install either the python 'xattr' module, "
3997 "or the 'xattr' binary.")
4000 def random_birthday(year_field, month_field, day_field):
4001 start_date = datetime.date(1950, 1, 1)
4002 end_date = datetime.date(1995, 12, 31)
4003 offset = random.randint(0, (end_date - start_date).days)
4004 random_date = start_date + datetime.timedelta(offset)
4006 year_field: str(random_date.year),
4007 month_field: str(random_date.month),
4008 day_field: str(random_date.day),