2 # -*- coding: utf-8 -*-
4 from __future__ import unicode_literals
34 import xml.etree.ElementTree
40 compat_etree_fromstring,
45 compat_socket_create_connection,
49 compat_urllib_parse_urlparse,
50 compat_urllib_request,
56 # This is not clearly defined otherwise
57 compiled_regex_type = type(re.compile(''))
60 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/44.0 (Chrome)',
61 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
62 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
63 'Accept-Encoding': 'gzip, deflate',
64 'Accept-Language': 'en-us,en;q=0.5',
70 ENGLISH_MONTH_NAMES = [
71 'January', 'February', 'March', 'April', 'May', 'June',
72 'July', 'August', 'September', 'October', 'November', 'December']
75 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
76 'flv', 'f4v', 'f4a', 'f4b',
77 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
87 'f4f', 'f4m', 'm3u8', 'smil')
90 def preferredencoding():
91 """Get preferred encoding.
93 Returns the best encoding scheme for the system, based on
94 locale.getpreferredencoding() and some further tweaks.
97 pref = locale.getpreferredencoding()
105 def write_json_file(obj, fn):
106 """ Encode obj as JSON and write it to fn, atomically if possible """
108 fn = encodeFilename(fn)
109 if sys.version_info < (3, 0) and sys.platform != 'win32':
110 encoding = get_filesystem_encoding()
111 # os.path.basename returns a bytes object, but NamedTemporaryFile
112 # will fail if the filename contains non ascii characters unless we
113 # use a unicode object
114 path_basename = lambda f: os.path.basename(fn).decode(encoding)
115 # the same for os.path.dirname
116 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
118 path_basename = os.path.basename
119 path_dirname = os.path.dirname
123 'prefix': path_basename(fn) + '.',
124 'dir': path_dirname(fn),
128 # In Python 2.x, json.dump expects a bytestream.
129 # In Python 3.x, it writes to a character stream
130 if sys.version_info < (3, 0):
138 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
143 if sys.platform == 'win32':
144 # Need to remove existing file on Windows, else os.rename raises
145 # WindowsError or FileExistsError.
150 os.rename(tf.name, fn)
159 if sys.version_info >= (2, 7):
160 def find_xpath_attr(node, xpath, key, val=None):
161 """ Find the xpath xpath[@key=val] """
162 assert re.match(r'^[a-zA-Z_-]+$', key)
164 assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
165 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
166 return node.find(expr)
168 def find_xpath_attr(node, xpath, key, val=None):
169 # Here comes the crazy part: In 2.6, if the xpath is a unicode,
170 # .//node does not match if a node is a direct child of . !
171 if isinstance(xpath, compat_str):
172 xpath = xpath.encode('ascii')
174 for f in node.findall(xpath):
175 if key not in f.attrib:
177 if val is None or f.attrib.get(key) == val:
181 # On python2.6 the xml.etree.ElementTree.Element methods don't support
182 # the namespace parameter
185 def xpath_with_ns(path, ns_map):
186 components = [c.split(':') for c in path.split('/')]
190 replaced.append(c[0])
193 replaced.append('{%s}%s' % (ns_map[ns], tag))
194 return '/'.join(replaced)
197 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
198 def _find_xpath(xpath):
199 if sys.version_info < (2, 7): # Crazy 2.6
200 xpath = xpath.encode('ascii')
201 return node.find(xpath)
203 if isinstance(xpath, (str, compat_str)):
204 n = _find_xpath(xpath)
212 if default is not NO_DEFAULT:
215 name = xpath if name is None else name
216 raise ExtractorError('Could not find XML element %s' % name)
222 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
223 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
224 if n is None or n == default:
227 if default is not NO_DEFAULT:
230 name = xpath if name is None else name
231 raise ExtractorError('Could not find XML element\'s text %s' % name)
237 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
238 n = find_xpath_attr(node, xpath, key)
240 if default is not NO_DEFAULT:
243 name = '%s[@%s]' % (xpath, key) if name is None else name
244 raise ExtractorError('Could not find XML attribute %s' % name)
250 def get_element_by_id(id, html):
251 """Return the content of the tag with the specified ID in the passed HTML document"""
252 return get_element_by_attribute('id', id, html)
255 def get_element_by_attribute(attribute, value, html):
256 """Return the content of the tag with the specified attribute in the passed HTML document"""
258 m = re.search(r'''(?xs)
260 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
262 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
266 ''' % (re.escape(attribute), re.escape(value)), html)
270 res = m.group('content')
272 if res.startswith('"') or res.startswith("'"):
275 return unescapeHTML(res)
278 def clean_html(html):
279 """Clean an HTML snippet into a readable string"""
281 if html is None: # Convenience for sanitizing descriptions etc.
285 html = html.replace('\n', ' ')
286 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
287 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
289 html = re.sub('<.*?>', '', html)
290 # Replace html entities
291 html = unescapeHTML(html)
295 def sanitize_open(filename, open_mode):
296 """Try to open the given filename, and slightly tweak it if this fails.
298 Attempts to open the given filename. If this fails, it tries to change
299 the filename slightly, step by step, until it's either able to open it
300 or it fails and raises a final exception, like the standard open()
303 It returns the tuple (stream, definitive_file_name).
307 if sys.platform == 'win32':
309 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
310 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
311 stream = open(encodeFilename(filename), open_mode)
312 return (stream, filename)
313 except (IOError, OSError) as err:
314 if err.errno in (errno.EACCES,):
317 # In case of error, try to remove win32 forbidden chars
318 alt_filename = sanitize_path(filename)
319 if alt_filename == filename:
322 # An exception here should be caught in the caller
323 stream = open(encodeFilename(alt_filename), open_mode)
324 return (stream, alt_filename)
327 def timeconvert(timestr):
328 """Convert RFC 2822 defined time string into system timestamp"""
330 timetuple = email.utils.parsedate_tz(timestr)
331 if timetuple is not None:
332 timestamp = email.utils.mktime_tz(timetuple)
336 def sanitize_filename(s, restricted=False, is_id=False):
337 """Sanitizes a string so it could be used as part of a filename.
338 If restricted is set, use a stricter subset of allowed characters.
339 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
341 def replace_insane(char):
342 if char == '?' or ord(char) < 32 or ord(char) == 127:
345 return '' if restricted else '\''
347 return '_-' if restricted else ' -'
348 elif char in '\\/|*<>':
350 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
352 if restricted and ord(char) > 127:
357 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
358 result = ''.join(map(replace_insane, s))
360 while '__' in result:
361 result = result.replace('__', '_')
362 result = result.strip('_')
363 # Common case of "Foreign band name - English song title"
364 if restricted and result.startswith('-_'):
366 if result.startswith('-'):
367 result = '_' + result[len('-'):]
368 result = result.lstrip('.')
374 def sanitize_path(s):
375 """Sanitizes and normalizes path on Windows"""
376 if sys.platform != 'win32':
378 drive_or_unc, _ = os.path.splitdrive(s)
379 if sys.version_info < (2, 7) and not drive_or_unc:
380 drive_or_unc, _ = os.path.splitunc(s)
381 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
385 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part)
386 for path_part in norm_path]
388 sanitized_path.insert(0, drive_or_unc + os.path.sep)
389 return os.path.join(*sanitized_path)
392 # Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
393 # unwanted failures due to missing protocol
394 def sanitized_Request(url, *args, **kwargs):
395 return compat_urllib_request.Request(
396 'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
399 def orderedSet(iterable):
400 """ Remove all duplicates from the input iterable """
408 def _htmlentity_transform(entity):
409 """Transforms an HTML entity to a character."""
410 # Known non-numeric HTML entity
411 if entity in compat_html_entities.name2codepoint:
412 return compat_chr(compat_html_entities.name2codepoint[entity])
414 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
416 numstr = mobj.group(1)
417 if numstr.startswith('x'):
419 numstr = '0%s' % numstr
422 # See https://github.com/rg3/youtube-dl/issues/7518
424 return compat_chr(int(numstr, base))
428 # Unknown entity in name, return its literal representation
429 return '&%s;' % entity
435 assert type(s) == compat_str
438 r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
441 def get_subprocess_encoding():
442 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
443 # For subprocess calls, encode with locale encoding
444 # Refer to http://stackoverflow.com/a/9951851/35070
445 encoding = preferredencoding()
447 encoding = sys.getfilesystemencoding()
453 def encodeFilename(s, for_subprocess=False):
455 @param s The name of the file
458 assert type(s) == compat_str
460 # Python 3 has a Unicode API
461 if sys.version_info >= (3, 0):
464 # Pass '' directly to use Unicode APIs on Windows 2000 and up
465 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
466 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
467 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
470 return s.encode(get_subprocess_encoding(), 'ignore')
473 def decodeFilename(b, for_subprocess=False):
475 if sys.version_info >= (3, 0):
478 if not isinstance(b, bytes):
481 return b.decode(get_subprocess_encoding(), 'ignore')
484 def encodeArgument(s):
485 if not isinstance(s, compat_str):
486 # Legacy code that uses byte strings
487 # Uncomment the following line after fixing all post processors
488 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
489 s = s.decode('ascii')
490 return encodeFilename(s, True)
493 def decodeArgument(b):
494 return decodeFilename(b, True)
497 def decodeOption(optval):
500 if isinstance(optval, bytes):
501 optval = optval.decode(preferredencoding())
503 assert isinstance(optval, compat_str)
507 def formatSeconds(secs):
509 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
511 return '%d:%02d' % (secs // 60, secs % 60)
516 def make_HTTPS_handler(params, **kwargs):
517 opts_no_check_certificate = params.get('nocheckcertificate', False)
518 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
519 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
520 if opts_no_check_certificate:
521 context.check_hostname = False
522 context.verify_mode = ssl.CERT_NONE
524 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
527 # (create_default_context present but HTTPSHandler has no context=)
530 if sys.version_info < (3, 2):
531 return YoutubeDLHTTPSHandler(params, **kwargs)
533 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
534 context.verify_mode = (ssl.CERT_NONE
535 if opts_no_check_certificate
536 else ssl.CERT_REQUIRED)
537 context.set_default_verify_paths()
538 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
541 def bug_reports_message():
542 if ytdl_is_updateable():
543 update_cmd = 'type youtube-dl -U to update'
545 update_cmd = 'see https://yt-dl.org/update on how to update'
546 msg = '; please report this issue on https://yt-dl.org/bug .'
547 msg += ' Make sure you are using the latest version; %s.' % update_cmd
548 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
552 class ExtractorError(Exception):
553 """Error during info extraction."""
555 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
556 """ tb, if given, is the original traceback (so that it can be printed out).
557 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
560 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
562 if video_id is not None:
563 msg = video_id + ': ' + msg
565 msg += ' (caused by %r)' % cause
567 msg += bug_reports_message()
568 super(ExtractorError, self).__init__(msg)
571 self.exc_info = sys.exc_info() # preserve original exception
573 self.video_id = video_id
575 def format_traceback(self):
576 if self.traceback is None:
578 return ''.join(traceback.format_tb(self.traceback))
581 class UnsupportedError(ExtractorError):
582 def __init__(self, url):
583 super(UnsupportedError, self).__init__(
584 'Unsupported URL: %s' % url, expected=True)
588 class RegexNotFoundError(ExtractorError):
589 """Error when a regex didn't match"""
593 class DownloadError(Exception):
594 """Download Error exception.
596 This exception may be thrown by FileDownloader objects if they are not
597 configured to continue on errors. They will contain the appropriate
601 def __init__(self, msg, exc_info=None):
602 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
603 super(DownloadError, self).__init__(msg)
604 self.exc_info = exc_info
607 class SameFileError(Exception):
608 """Same File exception.
610 This exception will be thrown by FileDownloader objects if they detect
611 multiple files would have to be downloaded to the same file on disk.
616 class PostProcessingError(Exception):
617 """Post Processing exception.
619 This exception may be raised by PostProcessor's .run() method to
620 indicate an error in the postprocessing task.
623 def __init__(self, msg):
627 class MaxDownloadsReached(Exception):
628 """ --max-downloads limit has been reached. """
632 class UnavailableVideoError(Exception):
633 """Unavailable Format exception.
635 This exception will be thrown when a video is requested
636 in a format that is not available for that video.
641 class ContentTooShortError(Exception):
642 """Content Too Short exception.
644 This exception may be raised by FileDownloader objects when a file they
645 download is too small for what the server announced first, indicating
646 the connection was probably interrupted.
649 def __init__(self, downloaded, expected):
651 self.downloaded = downloaded
652 self.expected = expected
655 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
656 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
657 # expected HTTP responses to meet HTTP/1.0 or later (see also
658 # https://github.com/rg3/youtube-dl/issues/6727)
659 if sys.version_info < (3, 0):
660 kwargs[b'strict'] = True
661 hc = http_class(*args, **kwargs)
662 source_address = ydl_handler._params.get('source_address')
663 if source_address is not None:
664 sa = (source_address, 0)
665 if hasattr(hc, 'source_address'): # Python 2.7+
666 hc.source_address = sa
668 def _hc_connect(self, *args, **kwargs):
669 sock = compat_socket_create_connection(
670 (self.host, self.port), self.timeout, sa)
672 self.sock = ssl.wrap_socket(
673 sock, self.key_file, self.cert_file,
674 ssl_version=ssl.PROTOCOL_TLSv1)
677 hc.connect = functools.partial(_hc_connect, hc)
682 def handle_youtubedl_headers(headers):
683 filtered_headers = headers
685 if 'Youtubedl-no-compression' in filtered_headers:
686 filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
687 del filtered_headers['Youtubedl-no-compression']
689 return filtered_headers
692 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
693 """Handler for HTTP requests and responses.
695 This class, when installed with an OpenerDirector, automatically adds
696 the standard headers to every HTTP request and handles gzipped and
697 deflated responses from web servers. If compression is to be avoided in
698 a particular request, the original request in the program code only has
699 to include the HTTP header "Youtubedl-no-compression", which will be
700 removed before making the real request.
702 Part of this code was copied from:
704 http://techknack.net/python-urllib2-handlers/
706 Andrew Rowls, the author of that code, agreed to release it to the
710 def __init__(self, params, *args, **kwargs):
711 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
712 self._params = params
714 def http_open(self, req):
715 return self.do_open(functools.partial(
716 _create_http_connection, self, compat_http_client.HTTPConnection, False),
722 return zlib.decompress(data, -zlib.MAX_WBITS)
724 return zlib.decompress(data)
727 def addinfourl_wrapper(stream, headers, url, code):
728 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
729 return compat_urllib_request.addinfourl(stream, headers, url, code)
730 ret = compat_urllib_request.addinfourl(stream, headers, url)
734 def http_request(self, req):
735 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
736 # always respected by websites, some tend to give out URLs with non percent-encoded
737 # non-ASCII characters (see telemb.py, ard.py [#3412])
738 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
739 # To work around aforementioned issue we will replace request's original URL with
740 # percent-encoded one
741 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
742 # the code of this workaround has been moved here from YoutubeDL.urlopen()
743 url = req.get_full_url()
744 url_escaped = escape_url(url)
746 # Substitute URL if any change after escaping
747 if url != url_escaped:
748 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
750 url_escaped, data=req.data, headers=req.headers,
751 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
752 new_req.timeout = req.timeout
755 for h, v in std_headers.items():
756 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
757 # The dict keys are capitalized because of this bug by urllib
758 if h.capitalize() not in req.headers:
761 req.headers = handle_youtubedl_headers(req.headers)
763 if sys.version_info < (2, 7) and '#' in req.get_full_url():
764 # Python 2.6 is brain-dead when it comes to fragments
765 req._Request__original = req._Request__original.partition('#')[0]
766 req._Request__r_type = req._Request__r_type.partition('#')[0]
770 def http_response(self, req, resp):
773 if resp.headers.get('Content-encoding', '') == 'gzip':
774 content = resp.read()
775 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
777 uncompressed = io.BytesIO(gz.read())
778 except IOError as original_ioerror:
779 # There may be junk add the end of the file
780 # See http://stackoverflow.com/q/4928560/35070 for details
781 for i in range(1, 1024):
783 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
784 uncompressed = io.BytesIO(gz.read())
789 raise original_ioerror
790 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
791 resp.msg = old_resp.msg
792 del resp.headers['Content-encoding']
794 if resp.headers.get('Content-encoding', '') == 'deflate':
795 gz = io.BytesIO(self.deflate(resp.read()))
796 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
797 resp.msg = old_resp.msg
798 del resp.headers['Content-encoding']
799 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
800 # https://github.com/rg3/youtube-dl/issues/6457).
801 if 300 <= resp.code < 400:
802 location = resp.headers.get('Location')
804 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
805 if sys.version_info >= (3, 0):
806 location = location.encode('iso-8859-1').decode('utf-8')
807 location_escaped = escape_url(location)
808 if location != location_escaped:
809 del resp.headers['Location']
810 resp.headers['Location'] = location_escaped
813 https_request = http_request
814 https_response = http_response
817 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
818 def __init__(self, params, https_conn_class=None, *args, **kwargs):
819 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
820 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
821 self._params = params
823 def https_open(self, req):
825 if hasattr(self, '_context'): # python > 2.6
826 kwargs['context'] = self._context
827 if hasattr(self, '_check_hostname'): # python 3.x
828 kwargs['check_hostname'] = self._check_hostname
829 return self.do_open(functools.partial(
830 _create_http_connection, self, self._https_conn_class, True),
834 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
835 def __init__(self, cookiejar=None):
836 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
838 def http_response(self, request, response):
839 # Python 2 will choke on next HTTP request in row if there are non-ASCII
840 # characters in Set-Cookie HTTP header of last response (see
841 # https://github.com/rg3/youtube-dl/issues/6769).
842 # In order to at least prevent crashing we will percent encode Set-Cookie
843 # header before HTTPCookieProcessor starts processing it.
844 # if sys.version_info < (3, 0) and response.headers:
845 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
846 # set_cookie = response.headers.get(set_cookie_header)
848 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
849 # if set_cookie != set_cookie_escaped:
850 # del response.headers[set_cookie_header]
851 # response.headers[set_cookie_header] = set_cookie_escaped
852 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
854 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
855 https_response = http_response
858 def parse_iso8601(date_str, delimiter='T', timezone=None):
859 """ Return a UNIX timestamp from the given date """
864 date_str = re.sub(r'\.[0-9]+', '', date_str)
868 r'(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
871 timezone = datetime.timedelta()
873 date_str = date_str[:-len(m.group(0))]
874 if not m.group('sign'):
875 timezone = datetime.timedelta()
877 sign = 1 if m.group('sign') == '+' else -1
878 timezone = datetime.timedelta(
879 hours=sign * int(m.group('hours')),
880 minutes=sign * int(m.group('minutes')))
882 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
883 dt = datetime.datetime.strptime(date_str, date_format) - timezone
884 return calendar.timegm(dt.timetuple())
889 def unified_strdate(date_str, day_first=True):
890 """Return a string with the date in the format YYYYMMDD"""
896 date_str = date_str.replace(',', ' ')
897 # %z (UTC offset) is only supported in python>=3.2
898 if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
899 date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
900 # Remove AM/PM + timezone
901 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
903 format_expressions = [
908 '%b %dst %Y %I:%M%p',
909 '%b %dnd %Y %I:%M%p',
910 '%b %dth %Y %I:%M%p',
916 '%Y-%m-%d %H:%M:%S.%f',
919 '%Y-%m-%dT%H:%M:%SZ',
920 '%Y-%m-%dT%H:%M:%S.%fZ',
921 '%Y-%m-%dT%H:%M:%S.%f0Z',
923 '%Y-%m-%dT%H:%M:%S.%f',
927 format_expressions.extend([
935 format_expressions.extend([
942 for expression in format_expressions:
944 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
947 if upload_date is None:
948 timetuple = email.utils.parsedate_tz(date_str)
950 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
951 if upload_date is not None:
952 return compat_str(upload_date)
955 def determine_ext(url, default_ext='unknown_video'):
958 guess = url.partition('?')[0].rpartition('.')[2]
959 if re.match(r'^[A-Za-z0-9]+$', guess):
961 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
962 elif guess.rstrip('/') in KNOWN_EXTENSIONS:
963 return guess.rstrip('/')
968 def subtitles_filename(filename, sub_lang, sub_format):
969 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
972 def date_from_str(date_str):
974 Return a datetime object from a string in the format YYYYMMDD or
975 (now|today)[+-][0-9](day|week|month|year)(s)?"""
976 today = datetime.date.today()
977 if date_str in ('now', 'today'):
979 if date_str == 'yesterday':
980 return today - datetime.timedelta(days=1)
981 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
982 if match is not None:
983 sign = match.group('sign')
984 time = int(match.group('time'))
987 unit = match.group('unit')
988 # A bad approximation?
996 delta = datetime.timedelta(**{unit: time})
998 return datetime.datetime.strptime(date_str, '%Y%m%d').date()
1001 def hyphenate_date(date_str):
1003 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
1004 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
1005 if match is not None:
1006 return '-'.join(match.groups())
1011 class DateRange(object):
1012 """Represents a time interval between two dates"""
1014 def __init__(self, start=None, end=None):
1015 """start and end must be strings in the format accepted by date"""
1016 if start is not None:
1017 self.start = date_from_str(start)
1019 self.start = datetime.datetime.min.date()
1021 self.end = date_from_str(end)
1023 self.end = datetime.datetime.max.date()
1024 if self.start > self.end:
1025 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
1029 """Returns a range that only contains the given day"""
1030 return cls(day, day)
1032 def __contains__(self, date):
1033 """Check if the date is in the range"""
1034 if not isinstance(date, datetime.date):
1035 date = date_from_str(date)
1036 return self.start <= date <= self.end
1039 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
1042 def platform_name():
1043 """ Returns the platform name as a compat_str """
1044 res = platform.platform()
1045 if isinstance(res, bytes):
1046 res = res.decode(preferredencoding())
1048 assert isinstance(res, compat_str)
1052 def _windows_write_string(s, out):
1053 """ Returns True if the string was written using special methods,
1054 False if it has yet to be written out."""
1055 # Adapted from http://stackoverflow.com/a/3259271/35070
1058 import ctypes.wintypes
1066 fileno = out.fileno()
1067 except AttributeError:
1068 # If the output stream doesn't have a fileno, it's virtual
1070 except io.UnsupportedOperation:
1071 # Some strange Windows pseudo files?
1073 if fileno not in WIN_OUTPUT_IDS:
1076 GetStdHandle = ctypes.WINFUNCTYPE(
1077 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1078 (b'GetStdHandle', ctypes.windll.kernel32))
1079 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1081 WriteConsoleW = ctypes.WINFUNCTYPE(
1082 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1083 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1084 ctypes.wintypes.LPVOID)((b'WriteConsoleW', ctypes.windll.kernel32))
1085 written = ctypes.wintypes.DWORD(0)
1087 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b'GetFileType', ctypes.windll.kernel32))
1088 FILE_TYPE_CHAR = 0x0002
1089 FILE_TYPE_REMOTE = 0x8000
1090 GetConsoleMode = ctypes.WINFUNCTYPE(
1091 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1092 ctypes.POINTER(ctypes.wintypes.DWORD))(
1093 (b'GetConsoleMode', ctypes.windll.kernel32))
1094 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1096 def not_a_console(handle):
1097 if handle == INVALID_HANDLE_VALUE or handle is None:
1099 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1100 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1102 if not_a_console(h):
1105 def next_nonbmp_pos(s):
1107 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1108 except StopIteration:
1112 count = min(next_nonbmp_pos(s), 1024)
1114 ret = WriteConsoleW(
1115 h, s, count if count else 2, ctypes.byref(written), None)
1117 raise OSError('Failed to write string')
1118 if not count: # We just wrote a non-BMP character
1119 assert written.value == 2
1122 assert written.value > 0
1123 s = s[written.value:]
1127 def write_string(s, out=None, encoding=None):
1130 assert type(s) == compat_str
1132 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1133 if _windows_write_string(s, out):
1136 if ('b' in getattr(out, 'mode', '') or
1137 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1138 byt = s.encode(encoding or preferredencoding(), 'ignore')
1140 elif hasattr(out, 'buffer'):
1141 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1142 byt = s.encode(enc, 'ignore')
1143 out.buffer.write(byt)
1149 def bytes_to_intlist(bs):
1152 if isinstance(bs[0], int): # Python 3
1155 return [ord(c) for c in bs]
1158 def intlist_to_bytes(xs):
1161 return struct_pack('%dB' % len(xs), *xs)
1164 # Cross-platform file locking
1165 if sys.platform == 'win32':
1166 import ctypes.wintypes
1169 class OVERLAPPED(ctypes.Structure):
1171 ('Internal', ctypes.wintypes.LPVOID),
1172 ('InternalHigh', ctypes.wintypes.LPVOID),
1173 ('Offset', ctypes.wintypes.DWORD),
1174 ('OffsetHigh', ctypes.wintypes.DWORD),
1175 ('hEvent', ctypes.wintypes.HANDLE),
1178 kernel32 = ctypes.windll.kernel32
1179 LockFileEx = kernel32.LockFileEx
1180 LockFileEx.argtypes = [
1181 ctypes.wintypes.HANDLE, # hFile
1182 ctypes.wintypes.DWORD, # dwFlags
1183 ctypes.wintypes.DWORD, # dwReserved
1184 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1185 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1186 ctypes.POINTER(OVERLAPPED) # Overlapped
1188 LockFileEx.restype = ctypes.wintypes.BOOL
1189 UnlockFileEx = kernel32.UnlockFileEx
1190 UnlockFileEx.argtypes = [
1191 ctypes.wintypes.HANDLE, # hFile
1192 ctypes.wintypes.DWORD, # dwReserved
1193 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1194 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1195 ctypes.POINTER(OVERLAPPED) # Overlapped
1197 UnlockFileEx.restype = ctypes.wintypes.BOOL
1198 whole_low = 0xffffffff
1199 whole_high = 0x7fffffff
1201 def _lock_file(f, exclusive):
1202 overlapped = OVERLAPPED()
1203 overlapped.Offset = 0
1204 overlapped.OffsetHigh = 0
1205 overlapped.hEvent = 0
1206 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1207 handle = msvcrt.get_osfhandle(f.fileno())
1208 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1209 whole_low, whole_high, f._lock_file_overlapped_p):
1210 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1212 def _unlock_file(f):
1213 assert f._lock_file_overlapped_p
1214 handle = msvcrt.get_osfhandle(f.fileno())
1215 if not UnlockFileEx(handle, 0,
1216 whole_low, whole_high, f._lock_file_overlapped_p):
1217 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1220 # Some platforms, such as Jython, is missing fcntl
1224 def _lock_file(f, exclusive):
1225 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1227 def _unlock_file(f):
1228 fcntl.flock(f, fcntl.LOCK_UN)
1230 UNSUPPORTED_MSG = 'file locking is not supported on this platform'
1232 def _lock_file(f, exclusive):
1233 raise IOError(UNSUPPORTED_MSG)
1235 def _unlock_file(f):
1236 raise IOError(UNSUPPORTED_MSG)
1239 class locked_file(object):
1240 def __init__(self, filename, mode, encoding=None):
1241 assert mode in ['r', 'a', 'w']
1242 self.f = io.open(filename, mode, encoding=encoding)
1245 def __enter__(self):
1246 exclusive = self.mode != 'r'
1248 _lock_file(self.f, exclusive)
1254 def __exit__(self, etype, value, traceback):
1256 _unlock_file(self.f)
1263 def write(self, *args):
1264 return self.f.write(*args)
1266 def read(self, *args):
1267 return self.f.read(*args)
1270 def get_filesystem_encoding():
1271 encoding = sys.getfilesystemencoding()
1272 return encoding if encoding is not None else 'utf-8'
1275 def shell_quote(args):
1277 encoding = get_filesystem_encoding()
1279 if isinstance(a, bytes):
1280 # We may get a filename encoded with 'encodeFilename'
1281 a = a.decode(encoding)
1282 quoted_args.append(pipes.quote(a))
1283 return ' '.join(quoted_args)
1286 def smuggle_url(url, data):
1287 """ Pass additional data in a URL for internal use. """
1289 sdata = compat_urllib_parse.urlencode(
1290 {'__youtubedl_smuggle': json.dumps(data)})
1291 return url + '#' + sdata
1294 def unsmuggle_url(smug_url, default=None):
1295 if '#__youtubedl_smuggle' not in smug_url:
1296 return smug_url, default
1297 url, _, sdata = smug_url.rpartition('#')
1298 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1299 data = json.loads(jsond)
1303 def format_bytes(bytes):
1306 if type(bytes) is str:
1307 bytes = float(bytes)
1311 exponent = int(math.log(bytes, 1024.0))
1312 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1313 converted = float(bytes) / float(1024 ** exponent)
1314 return '%.2f%s' % (converted, suffix)
1317 def parse_filesize(s):
1321 # The lower-case forms are of course incorrect and unofficial,
1322 # but we support those too
1360 units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
1362 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
1366 num_str = m.group('num').replace(',', '.')
1367 mult = _UNIT_TABLE[m.group('unit')]
1368 return int(float(num_str) * mult)
1371 def month_by_name(name):
1372 """ Return the number of a month by (locale-independently) English name """
1375 return ENGLISH_MONTH_NAMES.index(name) + 1
1380 def month_by_abbreviation(abbrev):
1381 """ Return the number of a month by (locale-independently) English
1385 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1390 def fix_xml_ampersands(xml_str):
1391 """Replace all the '&' by '&' in XML"""
1393 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1398 def setproctitle(title):
1399 assert isinstance(title, compat_str)
1401 # ctypes in Jython is not complete
1402 # http://bugs.jython.org/issue2148
1403 if sys.platform.startswith('java'):
1407 libc = ctypes.cdll.LoadLibrary('libc.so.6')
1410 title_bytes = title.encode('utf-8')
1411 buf = ctypes.create_string_buffer(len(title_bytes))
1412 buf.value = title_bytes
1414 libc.prctl(15, buf, 0, 0, 0)
1415 except AttributeError:
1416 return # Strange libc, just skip this
1419 def remove_start(s, start):
1420 if s.startswith(start):
1421 return s[len(start):]
1425 def remove_end(s, end):
1427 return s[:-len(end)]
1431 def remove_quotes(s):
1432 if s is None or len(s) < 2:
1434 for quote in ('"', "'", ):
1435 if s[0] == quote and s[-1] == quote:
1440 def url_basename(url):
1441 path = compat_urlparse.urlparse(url).path
1442 return path.strip('/').split('/')[-1]
1445 class HEADRequest(compat_urllib_request.Request):
1446 def get_method(self):
1450 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1453 v = getattr(v, get_attr, None)
1459 return int(v) * invscale // scale
1464 def str_or_none(v, default=None):
1465 return default if v is None else compat_str(v)
1468 def str_to_int(int_str):
1469 """ A more relaxed version of int_or_none """
1472 int_str = re.sub(r'[,\.\+]', '', int_str)
1476 def float_or_none(v, scale=1, invscale=1, default=None):
1480 return float(v) * invscale / scale
1485 def parse_duration(s):
1486 if not isinstance(s, compat_basestring):
1494 (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
1495 (?P<only_hours>[0-9.]+)\s*(?:hours?)|
1497 \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
1500 (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
1501 (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
1503 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
1505 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
1510 if m.group('only_mins'):
1511 return float_or_none(m.group('only_mins'), invscale=60)
1512 if m.group('only_hours'):
1513 return float_or_none(m.group('only_hours'), invscale=60 * 60)
1515 res += int(m.group('secs'))
1516 if m.group('mins_reversed'):
1517 res += int(m.group('mins_reversed')) * 60
1519 res += int(m.group('mins')) * 60
1520 if m.group('hours'):
1521 res += int(m.group('hours')) * 60 * 60
1522 if m.group('hours_reversed'):
1523 res += int(m.group('hours_reversed')) * 60 * 60
1525 res += int(m.group('days')) * 24 * 60 * 60
1527 res += float(m.group('ms'))
1531 def prepend_extension(filename, ext, expected_real_ext=None):
1532 name, real_ext = os.path.splitext(filename)
1534 '{0}.{1}{2}'.format(name, ext, real_ext)
1535 if not expected_real_ext or real_ext[1:] == expected_real_ext
1536 else '{0}.{1}'.format(filename, ext))
1539 def replace_extension(filename, ext, expected_real_ext=None):
1540 name, real_ext = os.path.splitext(filename)
1541 return '{0}.{1}'.format(
1542 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1546 def check_executable(exe, args=[]):
1547 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1548 args can be a list of arguments for a short output (like -version) """
1550 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1556 def get_exe_version(exe, args=['--version'],
1557 version_re=None, unrecognized='present'):
1558 """ Returns the version of the specified executable,
1559 or False if the executable is not present """
1561 out, _ = subprocess.Popen(
1562 [encodeArgument(exe)] + args,
1563 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1566 if isinstance(out, bytes): # Python 2.x
1567 out = out.decode('ascii', 'ignore')
1568 return detect_exe_version(out, version_re, unrecognized)
1571 def detect_exe_version(output, version_re=None, unrecognized='present'):
1572 assert isinstance(output, compat_str)
1573 if version_re is None:
1574 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1575 m = re.search(version_re, output)
1582 class PagedList(object):
1584 # This is only useful for tests
1585 return len(self.getslice())
1588 class OnDemandPagedList(PagedList):
1589 def __init__(self, pagefunc, pagesize):
1590 self._pagefunc = pagefunc
1591 self._pagesize = pagesize
1593 def getslice(self, start=0, end=None):
1595 for pagenum in itertools.count(start // self._pagesize):
1596 firstid = pagenum * self._pagesize
1597 nextfirstid = pagenum * self._pagesize + self._pagesize
1598 if start >= nextfirstid:
1601 page_results = list(self._pagefunc(pagenum))
1604 start % self._pagesize
1605 if firstid <= start < nextfirstid
1609 ((end - 1) % self._pagesize) + 1
1610 if (end is not None and firstid <= end <= nextfirstid)
1613 if startv != 0 or endv is not None:
1614 page_results = page_results[startv:endv]
1615 res.extend(page_results)
1617 # A little optimization - if current page is not "full", ie. does
1618 # not contain page_size videos then we can assume that this page
1619 # is the last one - there are no more ids on further pages -
1620 # i.e. no need to query again.
1621 if len(page_results) + startv < self._pagesize:
1624 # If we got the whole page, but the next page is not interesting,
1625 # break out early as well
1626 if end == nextfirstid:
1631 class InAdvancePagedList(PagedList):
1632 def __init__(self, pagefunc, pagecount, pagesize):
1633 self._pagefunc = pagefunc
1634 self._pagecount = pagecount
1635 self._pagesize = pagesize
1637 def getslice(self, start=0, end=None):
1639 start_page = start // self._pagesize
1641 self._pagecount if end is None else (end // self._pagesize + 1))
1642 skip_elems = start - start_page * self._pagesize
1643 only_more = None if end is None else end - start
1644 for pagenum in range(start_page, end_page):
1645 page = list(self._pagefunc(pagenum))
1647 page = page[skip_elems:]
1649 if only_more is not None:
1650 if len(page) < only_more:
1651 only_more -= len(page)
1653 page = page[:only_more]
1660 def uppercase_escape(s):
1661 unicode_escape = codecs.getdecoder('unicode_escape')
1663 r'\\U[0-9a-fA-F]{8}',
1664 lambda m: unicode_escape(m.group(0))[0],
1668 def lowercase_escape(s):
1669 unicode_escape = codecs.getdecoder('unicode_escape')
1671 r'\\u[0-9a-fA-F]{4}',
1672 lambda m: unicode_escape(m.group(0))[0],
1676 def escape_rfc3986(s):
1677 """Escape non-ASCII characters as suggested by RFC 3986"""
1678 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1679 s = s.encode('utf-8')
1680 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1683 def escape_url(url):
1684 """Escape URL as suggested by RFC 3986"""
1685 url_parsed = compat_urllib_parse_urlparse(url)
1686 return url_parsed._replace(
1687 path=escape_rfc3986(url_parsed.path),
1688 params=escape_rfc3986(url_parsed.params),
1689 query=escape_rfc3986(url_parsed.query),
1690 fragment=escape_rfc3986(url_parsed.fragment)
1694 struct.pack('!I', 0)
1696 # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1697 def struct_pack(spec, *args):
1698 if isinstance(spec, compat_str):
1699 spec = spec.encode('ascii')
1700 return struct.pack(spec, *args)
1702 def struct_unpack(spec, *args):
1703 if isinstance(spec, compat_str):
1704 spec = spec.encode('ascii')
1705 return struct.unpack(spec, *args)
1707 struct_pack = struct.pack
1708 struct_unpack = struct.unpack
1711 def read_batch_urls(batch_fd):
1713 if not isinstance(url, compat_str):
1714 url = url.decode('utf-8', 'replace')
1715 BOM_UTF8 = '\xef\xbb\xbf'
1716 if url.startswith(BOM_UTF8):
1717 url = url[len(BOM_UTF8):]
1719 if url.startswith(('#', ';', ']')):
1723 with contextlib.closing(batch_fd) as fd:
1724 return [url for url in map(fixup, fd) if url]
1727 def urlencode_postdata(*args, **kargs):
1728 return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
1731 def encode_dict(d, encoding='utf-8'):
1733 return v.encode(encoding) if isinstance(v, compat_basestring) else v
1734 return dict((encode(k), encode(v)) for k, v in d.items())
1737 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
1738 if isinstance(key_or_keys, (list, tuple)):
1739 for key in key_or_keys:
1740 if key not in d or d[key] is None or skip_false_values and not d[key]:
1744 return d.get(key_or_keys, default)
1747 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
1748 return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
1760 def parse_age_limit(s):
1763 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1764 return int(m.group('age')) if m else US_RATINGS.get(s)
1767 def strip_jsonp(code):
1769 r'(?s)^[a-zA-Z0-9_.]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1772 def js_to_json(code):
1775 if v in ('true', 'false', 'null'):
1777 if v.startswith('"'):
1778 v = re.sub(r"\\'", "'", v[1:-1])
1779 elif v.startswith("'"):
1781 v = re.sub(r"\\\\|\\'|\"", lambda m: {
1788 res = re.sub(r'''(?x)
1789 "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
1790 '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
1791 [a-zA-Z_][.a-zA-Z_0-9]*
1793 res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
1797 def qualities(quality_ids):
1798 """ Get a numeric quality value out of a list of possible values """
1801 return quality_ids.index(qid)
1807 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1810 def limit_length(s, length):
1811 """ Add ellipses to overly long strings """
1816 return s[:length - len(ELLIPSES)] + ELLIPSES
1820 def version_tuple(v):
1821 return tuple(int(e) for e in re.split(r'[-.]', v))
1824 def is_outdated_version(version, limit, assume_new=True):
1826 return not assume_new
1828 return version_tuple(version) < version_tuple(limit)
1830 return not assume_new
1833 def ytdl_is_updateable():
1834 """ Returns if youtube-dl can be updated with -U """
1835 from zipimport import zipimporter
1837 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
1840 def args_to_str(args):
1841 # Get a short string representation for a subprocess command
1842 return ' '.join(shlex_quote(a) for a in args)
1845 def error_to_compat_str(err):
1847 # On python 2 error byte string must be decoded with proper
1848 # encoding rather than ascii
1849 if sys.version_info[0] < 3:
1850 err_str = err_str.decode(preferredencoding())
1854 def mimetype2ext(mt):
1861 _, _, res = mt.rpartition('/')
1867 'x-mp4-fragmented': 'mp4',
1872 def urlhandle_detect_ext(url_handle):
1875 getheader = lambda h: url_handle.headers[h]
1876 except AttributeError: # Python < 3
1877 getheader = url_handle.info().getheader
1879 cd = getheader('Content-Disposition')
1881 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
1883 e = determine_ext(m.group('filename'), default_ext=None)
1887 return mimetype2ext(getheader('Content-Type'))
1890 def encode_data_uri(data, mime_type):
1891 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
1894 def age_restricted(content_limit, age_limit):
1895 """ Returns True iff the content should be blocked """
1897 if age_limit is None: # No limit set
1899 if content_limit is None:
1900 return False # Content available for everyone
1901 return age_limit < content_limit
1904 def is_html(first_bytes):
1905 """ Detect whether a file contains HTML by examining its first bytes. """
1908 (b'\xef\xbb\xbf', 'utf-8'),
1909 (b'\x00\x00\xfe\xff', 'utf-32-be'),
1910 (b'\xff\xfe\x00\x00', 'utf-32-le'),
1911 (b'\xff\xfe', 'utf-16-le'),
1912 (b'\xfe\xff', 'utf-16-be'),
1914 for bom, enc in BOMS:
1915 if first_bytes.startswith(bom):
1916 s = first_bytes[len(bom):].decode(enc, 'replace')
1919 s = first_bytes.decode('utf-8', 'replace')
1921 return re.match(r'^\s*<', s)
1924 def determine_protocol(info_dict):
1925 protocol = info_dict.get('protocol')
1926 if protocol is not None:
1929 url = info_dict['url']
1930 if url.startswith('rtmp'):
1932 elif url.startswith('mms'):
1934 elif url.startswith('rtsp'):
1937 ext = determine_ext(url)
1943 return compat_urllib_parse_urlparse(url).scheme
1946 def render_table(header_row, data):
1947 """ Render a list of rows, each as a list of values """
1948 table = [header_row] + data
1949 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
1950 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
1951 return '\n'.join(format_str % tuple(row) for row in table)
1954 def _match_one(filter_part, dct):
1955 COMPARISON_OPERATORS = {
1963 operator_rex = re.compile(r'''(?x)\s*
1965 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1967 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
1968 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
1971 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
1972 m = operator_rex.search(filter_part)
1974 op = COMPARISON_OPERATORS[m.group('op')]
1975 if m.group('strval') is not None:
1976 if m.group('op') not in ('=', '!='):
1978 'Operator %s does not support string values!' % m.group('op'))
1979 comparison_value = m.group('strval')
1982 comparison_value = int(m.group('intval'))
1984 comparison_value = parse_filesize(m.group('intval'))
1985 if comparison_value is None:
1986 comparison_value = parse_filesize(m.group('intval') + 'B')
1987 if comparison_value is None:
1989 'Invalid integer value %r in filter part %r' % (
1990 m.group('intval'), filter_part))
1991 actual_value = dct.get(m.group('key'))
1992 if actual_value is None:
1993 return m.group('none_inclusive')
1994 return op(actual_value, comparison_value)
1997 '': lambda v: v is not None,
1998 '!': lambda v: v is None,
2000 operator_rex = re.compile(r'''(?x)\s*
2001 (?P<op>%s)\s*(?P<key>[a-z_]+)
2003 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
2004 m = operator_rex.search(filter_part)
2006 op = UNARY_OPERATORS[m.group('op')]
2007 actual_value = dct.get(m.group('key'))
2008 return op(actual_value)
2010 raise ValueError('Invalid filter part %r' % filter_part)
2013 def match_str(filter_str, dct):
2014 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
2017 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
2020 def match_filter_func(filter_str):
2021 def _match_func(info_dict):
2022 if match_str(filter_str, info_dict):
2025 video_title = info_dict.get('title', info_dict.get('id', 'video'))
2026 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
2030 def parse_dfxp_time_expr(time_expr):
2034 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
2036 return float(mobj.group('time_offset'))
2038 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
2040 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
2043 def srt_subtitles_timecode(seconds):
2044 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
2047 def dfxp2srt(dfxp_data):
2048 _x = functools.partial(xpath_with_ns, ns_map={
2049 'ttml': 'http://www.w3.org/ns/ttml',
2050 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
2053 class TTMLPElementParser(object):
2056 def start(self, tag, attrib):
2057 if tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
2063 def data(self, data):
2067 return self.out.strip()
2069 def parse_node(node):
2070 target = TTMLPElementParser()
2071 parser = xml.etree.ElementTree.XMLParser(target=target)
2072 parser.feed(xml.etree.ElementTree.tostring(node))
2073 return parser.close()
2075 dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
2077 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
2080 raise ValueError('Invalid dfxp/TTML subtitle')
2082 for para, index in zip(paras, itertools.count(1)):
2083 begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
2084 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
2085 dur = parse_dfxp_time_expr(para.attrib.get('dur'))
2086 if begin_time is None:
2091 end_time = begin_time + dur
2092 out.append('%d\n%s --> %s\n%s\n\n' % (
2094 srt_subtitles_timecode(begin_time),
2095 srt_subtitles_timecode(end_time),
2101 def cli_option(params, command_option, param):
2102 param = params.get(param)
2103 return [command_option, param] if param is not None else []
2106 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
2107 param = params.get(param)
2108 assert isinstance(param, bool)
2110 return [command_option + separator + (true_value if param else false_value)]
2111 return [command_option, true_value if param else false_value]
2114 def cli_valueless_option(params, command_option, param, expected_value=True):
2115 param = params.get(param)
2116 return [command_option] if param == expected_value else []
2119 def cli_configuration_args(params, param, default=[]):
2120 ex_args = params.get(param)
2123 assert isinstance(ex_args, list)
2127 class ISO639Utils(object):
2128 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2317 def short2long(cls, code):
2318 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2319 return cls._lang_map.get(code[:2])
2322 def long2short(cls, code):
2323 """Convert language code from ISO 639-2/T to ISO 639-1"""
2324 for short_name, long_name in cls._lang_map.items():
2325 if long_name == code:
2329 class ISO3166Utils(object):
2330 # From http://data.okfn.org/data/core/country-list
2332 'AF': 'Afghanistan',
2333 'AX': 'Ã…land Islands',
2336 'AS': 'American Samoa',
2341 'AG': 'Antigua and Barbuda',
2358 'BO': 'Bolivia, Plurinational State of',
2359 'BQ': 'Bonaire, Sint Eustatius and Saba',
2360 'BA': 'Bosnia and Herzegovina',
2362 'BV': 'Bouvet Island',
2364 'IO': 'British Indian Ocean Territory',
2365 'BN': 'Brunei Darussalam',
2367 'BF': 'Burkina Faso',
2373 'KY': 'Cayman Islands',
2374 'CF': 'Central African Republic',
2378 'CX': 'Christmas Island',
2379 'CC': 'Cocos (Keeling) Islands',
2383 'CD': 'Congo, the Democratic Republic of the',
2384 'CK': 'Cook Islands',
2386 'CI': 'Côte d\'Ivoire',
2391 'CZ': 'Czech Republic',
2395 'DO': 'Dominican Republic',
2398 'SV': 'El Salvador',
2399 'GQ': 'Equatorial Guinea',
2403 'FK': 'Falkland Islands (Malvinas)',
2404 'FO': 'Faroe Islands',
2408 'GF': 'French Guiana',
2409 'PF': 'French Polynesia',
2410 'TF': 'French Southern Territories',
2425 'GW': 'Guinea-Bissau',
2428 'HM': 'Heard Island and McDonald Islands',
2429 'VA': 'Holy See (Vatican City State)',
2436 'IR': 'Iran, Islamic Republic of',
2439 'IM': 'Isle of Man',
2449 'KP': 'Korea, Democratic People\'s Republic of',
2450 'KR': 'Korea, Republic of',
2453 'LA': 'Lao People\'s Democratic Republic',
2459 'LI': 'Liechtenstein',
2463 'MK': 'Macedonia, the Former Yugoslav Republic of',
2470 'MH': 'Marshall Islands',
2476 'FM': 'Micronesia, Federated States of',
2477 'MD': 'Moldova, Republic of',
2488 'NL': 'Netherlands',
2489 'NC': 'New Caledonia',
2490 'NZ': 'New Zealand',
2495 'NF': 'Norfolk Island',
2496 'MP': 'Northern Mariana Islands',
2501 'PS': 'Palestine, State of',
2503 'PG': 'Papua New Guinea',
2506 'PH': 'Philippines',
2510 'PR': 'Puerto Rico',
2514 'RU': 'Russian Federation',
2516 'BL': 'Saint Barthélemy',
2517 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2518 'KN': 'Saint Kitts and Nevis',
2519 'LC': 'Saint Lucia',
2520 'MF': 'Saint Martin (French part)',
2521 'PM': 'Saint Pierre and Miquelon',
2522 'VC': 'Saint Vincent and the Grenadines',
2525 'ST': 'Sao Tome and Principe',
2526 'SA': 'Saudi Arabia',
2530 'SL': 'Sierra Leone',
2532 'SX': 'Sint Maarten (Dutch part)',
2535 'SB': 'Solomon Islands',
2537 'ZA': 'South Africa',
2538 'GS': 'South Georgia and the South Sandwich Islands',
2539 'SS': 'South Sudan',
2544 'SJ': 'Svalbard and Jan Mayen',
2547 'CH': 'Switzerland',
2548 'SY': 'Syrian Arab Republic',
2549 'TW': 'Taiwan, Province of China',
2551 'TZ': 'Tanzania, United Republic of',
2553 'TL': 'Timor-Leste',
2557 'TT': 'Trinidad and Tobago',
2560 'TM': 'Turkmenistan',
2561 'TC': 'Turks and Caicos Islands',
2565 'AE': 'United Arab Emirates',
2566 'GB': 'United Kingdom',
2567 'US': 'United States',
2568 'UM': 'United States Minor Outlying Islands',
2572 'VE': 'Venezuela, Bolivarian Republic of',
2574 'VG': 'Virgin Islands, British',
2575 'VI': 'Virgin Islands, U.S.',
2576 'WF': 'Wallis and Futuna',
2577 'EH': 'Western Sahara',
2584 def short2full(cls, code):
2585 """Convert an ISO 3166-2 country code to the corresponding full name"""
2586 return cls._country_map.get(code.upper())
2589 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2590 def __init__(self, proxies=None):
2591 # Set default handlers
2592 for type in ('http', 'https'):
2593 setattr(self, '%s_open' % type,
2594 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2595 meth(r, proxy, type))
2596 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2598 def proxy_open(self, req, proxy, type):
2599 req_proxy = req.headers.get('Ytdl-request-proxy')
2600 if req_proxy is not None:
2602 del req.headers['Ytdl-request-proxy']
2604 if proxy == '__noproxy__':
2605 return None # No Proxy
2606 return compat_urllib_request.ProxyHandler.proxy_open(
2607 self, req, proxy, type)
2610 def ohdave_rsa_encrypt(data, exponent, modulus):
2612 Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
2615 data: data to encrypt, bytes-like object
2616 exponent, modulus: parameter e and N of RSA algorithm, both integer
2617 Output: hex string of encrypted data
2619 Limitation: supports one block encryption only
2622 payload = int(binascii.hexlify(data[::-1]), 16)
2623 encrypted = pow(payload, exponent, modulus)
2624 return '%x' % encrypted