2 # -*- coding: utf-8 -*-
4 from __future__ import unicode_literals
33 import xml.etree.ElementTree
39 compat_etree_fromstring,
44 compat_socket_create_connection,
48 compat_urllib_parse_urlparse,
49 compat_urllib_request,
55 # This is not clearly defined otherwise
56 compiled_regex_type = type(re.compile(''))
59 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
60 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
61 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
62 'Accept-Encoding': 'gzip, deflate',
63 'Accept-Language': 'en-us,en;q=0.5',
69 ENGLISH_MONTH_NAMES = [
70 'January', 'February', 'March', 'April', 'May', 'June',
71 'July', 'August', 'September', 'October', 'November', 'December']
74 def preferredencoding():
75 """Get preferred encoding.
77 Returns the best encoding scheme for the system, based on
78 locale.getpreferredencoding() and some further tweaks.
81 pref = locale.getpreferredencoding()
89 def write_json_file(obj, fn):
90 """ Encode obj as JSON and write it to fn, atomically if possible """
92 fn = encodeFilename(fn)
93 if sys.version_info < (3, 0) and sys.platform != 'win32':
94 encoding = get_filesystem_encoding()
95 # os.path.basename returns a bytes object, but NamedTemporaryFile
96 # will fail if the filename contains non ascii characters unless we
97 # use a unicode object
98 path_basename = lambda f: os.path.basename(fn).decode(encoding)
99 # the same for os.path.dirname
100 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
102 path_basename = os.path.basename
103 path_dirname = os.path.dirname
107 'prefix': path_basename(fn) + '.',
108 'dir': path_dirname(fn),
112 # In Python 2.x, json.dump expects a bytestream.
113 # In Python 3.x, it writes to a character stream
114 if sys.version_info < (3, 0):
122 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
127 if sys.platform == 'win32':
128 # Need to remove existing file on Windows, else os.rename raises
129 # WindowsError or FileExistsError.
134 os.rename(tf.name, fn)
143 if sys.version_info >= (2, 7):
144 def find_xpath_attr(node, xpath, key, val=None):
145 """ Find the xpath xpath[@key=val] """
146 assert re.match(r'^[a-zA-Z_-]+$', key)
148 assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
149 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
150 return node.find(expr)
152 def find_xpath_attr(node, xpath, key, val=None):
153 # Here comes the crazy part: In 2.6, if the xpath is a unicode,
154 # .//node does not match if a node is a direct child of . !
155 if isinstance(xpath, compat_str):
156 xpath = xpath.encode('ascii')
158 for f in node.findall(xpath):
159 if key not in f.attrib:
161 if val is None or f.attrib.get(key) == val:
165 # On python2.6 the xml.etree.ElementTree.Element methods don't support
166 # the namespace parameter
169 def xpath_with_ns(path, ns_map):
170 components = [c.split(':') for c in path.split('/')]
174 replaced.append(c[0])
177 replaced.append('{%s}%s' % (ns_map[ns], tag))
178 return '/'.join(replaced)
181 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
182 def _find_xpath(xpath):
183 if sys.version_info < (2, 7): # Crazy 2.6
184 xpath = xpath.encode('ascii')
185 return node.find(xpath)
187 if isinstance(xpath, (str, compat_str)):
188 n = _find_xpath(xpath)
196 if default is not NO_DEFAULT:
199 name = xpath if name is None else name
200 raise ExtractorError('Could not find XML element %s' % name)
206 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
207 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
208 if n is None or n == default:
211 if default is not NO_DEFAULT:
214 name = xpath if name is None else name
215 raise ExtractorError('Could not find XML element\'s text %s' % name)
221 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
222 n = find_xpath_attr(node, xpath, key)
224 if default is not NO_DEFAULT:
227 name = '%s[@%s]' % (xpath, key) if name is None else name
228 raise ExtractorError('Could not find XML attribute %s' % name)
234 def get_element_by_id(id, html):
235 """Return the content of the tag with the specified ID in the passed HTML document"""
236 return get_element_by_attribute("id", id, html)
239 def get_element_by_attribute(attribute, value, html):
240 """Return the content of the tag with the specified attribute in the passed HTML document"""
242 m = re.search(r'''(?xs)
244 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
246 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
250 ''' % (re.escape(attribute), re.escape(value)), html)
254 res = m.group('content')
256 if res.startswith('"') or res.startswith("'"):
259 return unescapeHTML(res)
262 def clean_html(html):
263 """Clean an HTML snippet into a readable string"""
265 if html is None: # Convenience for sanitizing descriptions etc.
269 html = html.replace('\n', ' ')
270 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
271 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
273 html = re.sub('<.*?>', '', html)
274 # Replace html entities
275 html = unescapeHTML(html)
279 def sanitize_open(filename, open_mode):
280 """Try to open the given filename, and slightly tweak it if this fails.
282 Attempts to open the given filename. If this fails, it tries to change
283 the filename slightly, step by step, until it's either able to open it
284 or it fails and raises a final exception, like the standard open()
287 It returns the tuple (stream, definitive_file_name).
291 if sys.platform == 'win32':
293 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
294 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
295 stream = open(encodeFilename(filename), open_mode)
296 return (stream, filename)
297 except (IOError, OSError) as err:
298 if err.errno in (errno.EACCES,):
301 # In case of error, try to remove win32 forbidden chars
302 alt_filename = sanitize_path(filename)
303 if alt_filename == filename:
306 # An exception here should be caught in the caller
307 stream = open(encodeFilename(alt_filename), open_mode)
308 return (stream, alt_filename)
311 def timeconvert(timestr):
312 """Convert RFC 2822 defined time string into system timestamp"""
314 timetuple = email.utils.parsedate_tz(timestr)
315 if timetuple is not None:
316 timestamp = email.utils.mktime_tz(timetuple)
320 def sanitize_filename(s, restricted=False, is_id=False):
321 """Sanitizes a string so it could be used as part of a filename.
322 If restricted is set, use a stricter subset of allowed characters.
323 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
325 def replace_insane(char):
326 if char == '?' or ord(char) < 32 or ord(char) == 127:
329 return '' if restricted else '\''
331 return '_-' if restricted else ' -'
332 elif char in '\\/|*<>':
334 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
336 if restricted and ord(char) > 127:
341 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
342 result = ''.join(map(replace_insane, s))
344 while '__' in result:
345 result = result.replace('__', '_')
346 result = result.strip('_')
347 # Common case of "Foreign band name - English song title"
348 if restricted and result.startswith('-_'):
350 if result.startswith('-'):
351 result = '_' + result[len('-'):]
352 result = result.lstrip('.')
358 def sanitize_path(s):
359 """Sanitizes and normalizes path on Windows"""
360 if sys.platform != 'win32':
362 drive_or_unc, _ = os.path.splitdrive(s)
363 if sys.version_info < (2, 7) and not drive_or_unc:
364 drive_or_unc, _ = os.path.splitunc(s)
365 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
369 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part)
370 for path_part in norm_path]
372 sanitized_path.insert(0, drive_or_unc + os.path.sep)
373 return os.path.join(*sanitized_path)
376 # Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
377 # unwanted failures due to missing protocol
378 def sanitized_Request(url, *args, **kwargs):
379 return compat_urllib_request.Request(
380 'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
383 def orderedSet(iterable):
384 """ Remove all duplicates from the input iterable """
392 def _htmlentity_transform(entity):
393 """Transforms an HTML entity to a character."""
394 # Known non-numeric HTML entity
395 if entity in compat_html_entities.name2codepoint:
396 return compat_chr(compat_html_entities.name2codepoint[entity])
398 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
400 numstr = mobj.group(1)
401 if numstr.startswith('x'):
403 numstr = '0%s' % numstr
406 # See https://github.com/rg3/youtube-dl/issues/7518
408 return compat_chr(int(numstr, base))
412 # Unknown entity in name, return its literal representation
413 return '&%s;' % entity
419 assert type(s) == compat_str
422 r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
425 def get_subprocess_encoding():
426 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
427 # For subprocess calls, encode with locale encoding
428 # Refer to http://stackoverflow.com/a/9951851/35070
429 encoding = preferredencoding()
431 encoding = sys.getfilesystemencoding()
437 def encodeFilename(s, for_subprocess=False):
439 @param s The name of the file
442 assert type(s) == compat_str
444 # Python 3 has a Unicode API
445 if sys.version_info >= (3, 0):
448 # Pass '' directly to use Unicode APIs on Windows 2000 and up
449 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
450 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
451 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
454 return s.encode(get_subprocess_encoding(), 'ignore')
457 def decodeFilename(b, for_subprocess=False):
459 if sys.version_info >= (3, 0):
462 if not isinstance(b, bytes):
465 return b.decode(get_subprocess_encoding(), 'ignore')
468 def encodeArgument(s):
469 if not isinstance(s, compat_str):
470 # Legacy code that uses byte strings
471 # Uncomment the following line after fixing all post processors
472 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
473 s = s.decode('ascii')
474 return encodeFilename(s, True)
477 def decodeArgument(b):
478 return decodeFilename(b, True)
481 def decodeOption(optval):
484 if isinstance(optval, bytes):
485 optval = optval.decode(preferredencoding())
487 assert isinstance(optval, compat_str)
491 def formatSeconds(secs):
493 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
495 return '%d:%02d' % (secs // 60, secs % 60)
500 def make_HTTPS_handler(params, **kwargs):
501 opts_no_check_certificate = params.get('nocheckcertificate', False)
502 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
503 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
504 if opts_no_check_certificate:
505 context.check_hostname = False
506 context.verify_mode = ssl.CERT_NONE
508 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
511 # (create_default_context present but HTTPSHandler has no context=)
514 if sys.version_info < (3, 2):
515 return YoutubeDLHTTPSHandler(params, **kwargs)
517 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
518 context.verify_mode = (ssl.CERT_NONE
519 if opts_no_check_certificate
520 else ssl.CERT_REQUIRED)
521 context.set_default_verify_paths()
522 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
525 def bug_reports_message():
526 if ytdl_is_updateable():
527 update_cmd = 'type youtube-dl -U to update'
529 update_cmd = 'see https://yt-dl.org/update on how to update'
530 msg = '; please report this issue on https://yt-dl.org/bug .'
531 msg += ' Make sure you are using the latest version; %s.' % update_cmd
532 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
536 class ExtractorError(Exception):
537 """Error during info extraction."""
539 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
540 """ tb, if given, is the original traceback (so that it can be printed out).
541 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
544 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
546 if video_id is not None:
547 msg = video_id + ': ' + msg
549 msg += ' (caused by %r)' % cause
551 msg += bug_reports_message()
552 super(ExtractorError, self).__init__(msg)
555 self.exc_info = sys.exc_info() # preserve original exception
557 self.video_id = video_id
559 def format_traceback(self):
560 if self.traceback is None:
562 return ''.join(traceback.format_tb(self.traceback))
565 class UnsupportedError(ExtractorError):
566 def __init__(self, url):
567 super(UnsupportedError, self).__init__(
568 'Unsupported URL: %s' % url, expected=True)
572 class RegexNotFoundError(ExtractorError):
573 """Error when a regex didn't match"""
577 class DownloadError(Exception):
578 """Download Error exception.
580 This exception may be thrown by FileDownloader objects if they are not
581 configured to continue on errors. They will contain the appropriate
585 def __init__(self, msg, exc_info=None):
586 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
587 super(DownloadError, self).__init__(msg)
588 self.exc_info = exc_info
591 class SameFileError(Exception):
592 """Same File exception.
594 This exception will be thrown by FileDownloader objects if they detect
595 multiple files would have to be downloaded to the same file on disk.
600 class PostProcessingError(Exception):
601 """Post Processing exception.
603 This exception may be raised by PostProcessor's .run() method to
604 indicate an error in the postprocessing task.
607 def __init__(self, msg):
611 class MaxDownloadsReached(Exception):
612 """ --max-downloads limit has been reached. """
616 class UnavailableVideoError(Exception):
617 """Unavailable Format exception.
619 This exception will be thrown when a video is requested
620 in a format that is not available for that video.
625 class ContentTooShortError(Exception):
626 """Content Too Short exception.
628 This exception may be raised by FileDownloader objects when a file they
629 download is too small for what the server announced first, indicating
630 the connection was probably interrupted.
633 def __init__(self, downloaded, expected):
635 self.downloaded = downloaded
636 self.expected = expected
639 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
640 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
641 # expected HTTP responses to meet HTTP/1.0 or later (see also
642 # https://github.com/rg3/youtube-dl/issues/6727)
643 if sys.version_info < (3, 0):
644 kwargs[b'strict'] = True
645 hc = http_class(*args, **kwargs)
646 source_address = ydl_handler._params.get('source_address')
647 if source_address is not None:
648 sa = (source_address, 0)
649 if hasattr(hc, 'source_address'): # Python 2.7+
650 hc.source_address = sa
652 def _hc_connect(self, *args, **kwargs):
653 sock = compat_socket_create_connection(
654 (self.host, self.port), self.timeout, sa)
656 self.sock = ssl.wrap_socket(
657 sock, self.key_file, self.cert_file,
658 ssl_version=ssl.PROTOCOL_TLSv1)
661 hc.connect = functools.partial(_hc_connect, hc)
666 def handle_youtubedl_headers(headers):
667 filtered_headers = headers
669 if 'Youtubedl-no-compression' in filtered_headers:
670 filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
671 del filtered_headers['Youtubedl-no-compression']
673 return filtered_headers
676 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
677 """Handler for HTTP requests and responses.
679 This class, when installed with an OpenerDirector, automatically adds
680 the standard headers to every HTTP request and handles gzipped and
681 deflated responses from web servers. If compression is to be avoided in
682 a particular request, the original request in the program code only has
683 to include the HTTP header "Youtubedl-no-compression", which will be
684 removed before making the real request.
686 Part of this code was copied from:
688 http://techknack.net/python-urllib2-handlers/
690 Andrew Rowls, the author of that code, agreed to release it to the
694 def __init__(self, params, *args, **kwargs):
695 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
696 self._params = params
698 def http_open(self, req):
699 return self.do_open(functools.partial(
700 _create_http_connection, self, compat_http_client.HTTPConnection, False),
706 return zlib.decompress(data, -zlib.MAX_WBITS)
708 return zlib.decompress(data)
711 def addinfourl_wrapper(stream, headers, url, code):
712 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
713 return compat_urllib_request.addinfourl(stream, headers, url, code)
714 ret = compat_urllib_request.addinfourl(stream, headers, url)
718 def http_request(self, req):
719 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
720 # always respected by websites, some tend to give out URLs with non percent-encoded
721 # non-ASCII characters (see telemb.py, ard.py [#3412])
722 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
723 # To work around aforementioned issue we will replace request's original URL with
724 # percent-encoded one
725 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
726 # the code of this workaround has been moved here from YoutubeDL.urlopen()
727 url = req.get_full_url()
728 url_escaped = escape_url(url)
730 # Substitute URL if any change after escaping
731 if url != url_escaped:
732 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
734 url_escaped, data=req.data, headers=req.headers,
735 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
736 new_req.timeout = req.timeout
739 for h, v in std_headers.items():
740 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
741 # The dict keys are capitalized because of this bug by urllib
742 if h.capitalize() not in req.headers:
745 req.headers = handle_youtubedl_headers(req.headers)
747 if sys.version_info < (2, 7) and '#' in req.get_full_url():
748 # Python 2.6 is brain-dead when it comes to fragments
749 req._Request__original = req._Request__original.partition('#')[0]
750 req._Request__r_type = req._Request__r_type.partition('#')[0]
754 def http_response(self, req, resp):
757 if resp.headers.get('Content-encoding', '') == 'gzip':
758 content = resp.read()
759 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
761 uncompressed = io.BytesIO(gz.read())
762 except IOError as original_ioerror:
763 # There may be junk add the end of the file
764 # See http://stackoverflow.com/q/4928560/35070 for details
765 for i in range(1, 1024):
767 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
768 uncompressed = io.BytesIO(gz.read())
773 raise original_ioerror
774 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
775 resp.msg = old_resp.msg
777 if resp.headers.get('Content-encoding', '') == 'deflate':
778 gz = io.BytesIO(self.deflate(resp.read()))
779 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
780 resp.msg = old_resp.msg
781 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
782 # https://github.com/rg3/youtube-dl/issues/6457).
783 if 300 <= resp.code < 400:
784 location = resp.headers.get('Location')
786 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
787 if sys.version_info >= (3, 0):
788 location = location.encode('iso-8859-1').decode('utf-8')
789 location_escaped = escape_url(location)
790 if location != location_escaped:
791 del resp.headers['Location']
792 resp.headers['Location'] = location_escaped
795 https_request = http_request
796 https_response = http_response
799 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
800 def __init__(self, params, https_conn_class=None, *args, **kwargs):
801 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
802 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
803 self._params = params
805 def https_open(self, req):
807 if hasattr(self, '_context'): # python > 2.6
808 kwargs['context'] = self._context
809 if hasattr(self, '_check_hostname'): # python 3.x
810 kwargs['check_hostname'] = self._check_hostname
811 return self.do_open(functools.partial(
812 _create_http_connection, self, self._https_conn_class, True),
816 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
817 def __init__(self, cookiejar=None):
818 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
820 def http_response(self, request, response):
821 # Python 2 will choke on next HTTP request in row if there are non-ASCII
822 # characters in Set-Cookie HTTP header of last response (see
823 # https://github.com/rg3/youtube-dl/issues/6769).
824 # In order to at least prevent crashing we will percent encode Set-Cookie
825 # header before HTTPCookieProcessor starts processing it.
826 # if sys.version_info < (3, 0) and response.headers:
827 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
828 # set_cookie = response.headers.get(set_cookie_header)
830 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
831 # if set_cookie != set_cookie_escaped:
832 # del response.headers[set_cookie_header]
833 # response.headers[set_cookie_header] = set_cookie_escaped
834 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
836 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
837 https_response = http_response
840 def parse_iso8601(date_str, delimiter='T', timezone=None):
841 """ Return a UNIX timestamp from the given date """
846 date_str = re.sub(r'\.[0-9]+', '', date_str)
850 r'(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
853 timezone = datetime.timedelta()
855 date_str = date_str[:-len(m.group(0))]
856 if not m.group('sign'):
857 timezone = datetime.timedelta()
859 sign = 1 if m.group('sign') == '+' else -1
860 timezone = datetime.timedelta(
861 hours=sign * int(m.group('hours')),
862 minutes=sign * int(m.group('minutes')))
864 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
865 dt = datetime.datetime.strptime(date_str, date_format) - timezone
866 return calendar.timegm(dt.timetuple())
871 def unified_strdate(date_str, day_first=True):
872 """Return a string with the date in the format YYYYMMDD"""
878 date_str = date_str.replace(',', ' ')
879 # %z (UTC offset) is only supported in python>=3.2
880 if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
881 date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
882 # Remove AM/PM + timezone
883 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
885 format_expressions = [
890 '%b %dst %Y %I:%M%p',
891 '%b %dnd %Y %I:%M%p',
892 '%b %dth %Y %I:%M%p',
898 '%Y-%m-%d %H:%M:%S.%f',
901 '%Y-%m-%dT%H:%M:%SZ',
902 '%Y-%m-%dT%H:%M:%S.%fZ',
903 '%Y-%m-%dT%H:%M:%S.%f0Z',
905 '%Y-%m-%dT%H:%M:%S.%f',
909 format_expressions.extend([
917 format_expressions.extend([
924 for expression in format_expressions:
926 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
929 if upload_date is None:
930 timetuple = email.utils.parsedate_tz(date_str)
932 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
933 if upload_date is not None:
934 return compat_str(upload_date)
937 def determine_ext(url, default_ext='unknown_video'):
940 guess = url.partition('?')[0].rpartition('.')[2]
941 if re.match(r'^[A-Za-z0-9]+$', guess):
943 elif guess.rstrip('/') in (
944 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
945 'flv', 'f4v', 'f4a', 'f4b',
946 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
947 'mkv', 'mka', 'mk3d',
956 'f4f', 'f4m', 'm3u8', 'smil'):
957 return guess.rstrip('/')
962 def subtitles_filename(filename, sub_lang, sub_format):
963 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
966 def date_from_str(date_str):
968 Return a datetime object from a string in the format YYYYMMDD or
969 (now|today)[+-][0-9](day|week|month|year)(s)?"""
970 today = datetime.date.today()
971 if date_str in ('now', 'today'):
973 if date_str == 'yesterday':
974 return today - datetime.timedelta(days=1)
975 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
976 if match is not None:
977 sign = match.group('sign')
978 time = int(match.group('time'))
981 unit = match.group('unit')
982 # A bad aproximation?
990 delta = datetime.timedelta(**{unit: time})
992 return datetime.datetime.strptime(date_str, "%Y%m%d").date()
995 def hyphenate_date(date_str):
997 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
998 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
999 if match is not None:
1000 return '-'.join(match.groups())
1005 class DateRange(object):
1006 """Represents a time interval between two dates"""
1008 def __init__(self, start=None, end=None):
1009 """start and end must be strings in the format accepted by date"""
1010 if start is not None:
1011 self.start = date_from_str(start)
1013 self.start = datetime.datetime.min.date()
1015 self.end = date_from_str(end)
1017 self.end = datetime.datetime.max.date()
1018 if self.start > self.end:
1019 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
1023 """Returns a range that only contains the given day"""
1024 return cls(day, day)
1026 def __contains__(self, date):
1027 """Check if the date is in the range"""
1028 if not isinstance(date, datetime.date):
1029 date = date_from_str(date)
1030 return self.start <= date <= self.end
1033 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
1036 def platform_name():
1037 """ Returns the platform name as a compat_str """
1038 res = platform.platform()
1039 if isinstance(res, bytes):
1040 res = res.decode(preferredencoding())
1042 assert isinstance(res, compat_str)
1046 def _windows_write_string(s, out):
1047 """ Returns True if the string was written using special methods,
1048 False if it has yet to be written out."""
1049 # Adapted from http://stackoverflow.com/a/3259271/35070
1052 import ctypes.wintypes
1060 fileno = out.fileno()
1061 except AttributeError:
1062 # If the output stream doesn't have a fileno, it's virtual
1064 except io.UnsupportedOperation:
1065 # Some strange Windows pseudo files?
1067 if fileno not in WIN_OUTPUT_IDS:
1070 GetStdHandle = ctypes.WINFUNCTYPE(
1071 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1072 (b"GetStdHandle", ctypes.windll.kernel32))
1073 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1075 WriteConsoleW = ctypes.WINFUNCTYPE(
1076 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1077 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1078 ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
1079 written = ctypes.wintypes.DWORD(0)
1081 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
1082 FILE_TYPE_CHAR = 0x0002
1083 FILE_TYPE_REMOTE = 0x8000
1084 GetConsoleMode = ctypes.WINFUNCTYPE(
1085 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1086 ctypes.POINTER(ctypes.wintypes.DWORD))(
1087 (b"GetConsoleMode", ctypes.windll.kernel32))
1088 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1090 def not_a_console(handle):
1091 if handle == INVALID_HANDLE_VALUE or handle is None:
1093 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1094 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1096 if not_a_console(h):
1099 def next_nonbmp_pos(s):
1101 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1102 except StopIteration:
1106 count = min(next_nonbmp_pos(s), 1024)
1108 ret = WriteConsoleW(
1109 h, s, count if count else 2, ctypes.byref(written), None)
1111 raise OSError('Failed to write string')
1112 if not count: # We just wrote a non-BMP character
1113 assert written.value == 2
1116 assert written.value > 0
1117 s = s[written.value:]
1121 def write_string(s, out=None, encoding=None):
1124 assert type(s) == compat_str
1126 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1127 if _windows_write_string(s, out):
1130 if ('b' in getattr(out, 'mode', '') or
1131 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1132 byt = s.encode(encoding or preferredencoding(), 'ignore')
1134 elif hasattr(out, 'buffer'):
1135 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1136 byt = s.encode(enc, 'ignore')
1137 out.buffer.write(byt)
1143 def bytes_to_intlist(bs):
1146 if isinstance(bs[0], int): # Python 3
1149 return [ord(c) for c in bs]
1152 def intlist_to_bytes(xs):
1155 return struct_pack('%dB' % len(xs), *xs)
1158 # Cross-platform file locking
1159 if sys.platform == 'win32':
1160 import ctypes.wintypes
1163 class OVERLAPPED(ctypes.Structure):
1165 ('Internal', ctypes.wintypes.LPVOID),
1166 ('InternalHigh', ctypes.wintypes.LPVOID),
1167 ('Offset', ctypes.wintypes.DWORD),
1168 ('OffsetHigh', ctypes.wintypes.DWORD),
1169 ('hEvent', ctypes.wintypes.HANDLE),
1172 kernel32 = ctypes.windll.kernel32
1173 LockFileEx = kernel32.LockFileEx
1174 LockFileEx.argtypes = [
1175 ctypes.wintypes.HANDLE, # hFile
1176 ctypes.wintypes.DWORD, # dwFlags
1177 ctypes.wintypes.DWORD, # dwReserved
1178 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1179 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1180 ctypes.POINTER(OVERLAPPED) # Overlapped
1182 LockFileEx.restype = ctypes.wintypes.BOOL
1183 UnlockFileEx = kernel32.UnlockFileEx
1184 UnlockFileEx.argtypes = [
1185 ctypes.wintypes.HANDLE, # hFile
1186 ctypes.wintypes.DWORD, # dwReserved
1187 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1188 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1189 ctypes.POINTER(OVERLAPPED) # Overlapped
1191 UnlockFileEx.restype = ctypes.wintypes.BOOL
1192 whole_low = 0xffffffff
1193 whole_high = 0x7fffffff
1195 def _lock_file(f, exclusive):
1196 overlapped = OVERLAPPED()
1197 overlapped.Offset = 0
1198 overlapped.OffsetHigh = 0
1199 overlapped.hEvent = 0
1200 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1201 handle = msvcrt.get_osfhandle(f.fileno())
1202 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1203 whole_low, whole_high, f._lock_file_overlapped_p):
1204 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1206 def _unlock_file(f):
1207 assert f._lock_file_overlapped_p
1208 handle = msvcrt.get_osfhandle(f.fileno())
1209 if not UnlockFileEx(handle, 0,
1210 whole_low, whole_high, f._lock_file_overlapped_p):
1211 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1216 def _lock_file(f, exclusive):
1217 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1219 def _unlock_file(f):
1220 fcntl.flock(f, fcntl.LOCK_UN)
1223 class locked_file(object):
1224 def __init__(self, filename, mode, encoding=None):
1225 assert mode in ['r', 'a', 'w']
1226 self.f = io.open(filename, mode, encoding=encoding)
1229 def __enter__(self):
1230 exclusive = self.mode != 'r'
1232 _lock_file(self.f, exclusive)
1238 def __exit__(self, etype, value, traceback):
1240 _unlock_file(self.f)
1247 def write(self, *args):
1248 return self.f.write(*args)
1250 def read(self, *args):
1251 return self.f.read(*args)
1254 def get_filesystem_encoding():
1255 encoding = sys.getfilesystemencoding()
1256 return encoding if encoding is not None else 'utf-8'
1259 def shell_quote(args):
1261 encoding = get_filesystem_encoding()
1263 if isinstance(a, bytes):
1264 # We may get a filename encoded with 'encodeFilename'
1265 a = a.decode(encoding)
1266 quoted_args.append(pipes.quote(a))
1267 return ' '.join(quoted_args)
1270 def smuggle_url(url, data):
1271 """ Pass additional data in a URL for internal use. """
1273 sdata = compat_urllib_parse.urlencode(
1274 {'__youtubedl_smuggle': json.dumps(data)})
1275 return url + '#' + sdata
1278 def unsmuggle_url(smug_url, default=None):
1279 if '#__youtubedl_smuggle' not in smug_url:
1280 return smug_url, default
1281 url, _, sdata = smug_url.rpartition('#')
1282 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1283 data = json.loads(jsond)
1287 def format_bytes(bytes):
1290 if type(bytes) is str:
1291 bytes = float(bytes)
1295 exponent = int(math.log(bytes, 1024.0))
1296 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1297 converted = float(bytes) / float(1024 ** exponent)
1298 return '%.2f%s' % (converted, suffix)
1301 def parse_filesize(s):
1305 # The lower-case forms are of course incorrect and inofficial,
1306 # but we support those too
1344 units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
1346 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
1350 num_str = m.group('num').replace(',', '.')
1351 mult = _UNIT_TABLE[m.group('unit')]
1352 return int(float(num_str) * mult)
1355 def month_by_name(name):
1356 """ Return the number of a month by (locale-independently) English name """
1359 return ENGLISH_MONTH_NAMES.index(name) + 1
1364 def month_by_abbreviation(abbrev):
1365 """ Return the number of a month by (locale-independently) English
1369 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1374 def fix_xml_ampersands(xml_str):
1375 """Replace all the '&' by '&' in XML"""
1377 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1382 def setproctitle(title):
1383 assert isinstance(title, compat_str)
1385 libc = ctypes.cdll.LoadLibrary("libc.so.6")
1388 title_bytes = title.encode('utf-8')
1389 buf = ctypes.create_string_buffer(len(title_bytes))
1390 buf.value = title_bytes
1392 libc.prctl(15, buf, 0, 0, 0)
1393 except AttributeError:
1394 return # Strange libc, just skip this
1397 def remove_start(s, start):
1398 if s.startswith(start):
1399 return s[len(start):]
1403 def remove_end(s, end):
1405 return s[:-len(end)]
1409 def remove_quotes(s):
1410 if s is None or len(s) < 2:
1412 for quote in ('"', "'", ):
1413 if s[0] == quote and s[-1] == quote:
1418 def url_basename(url):
1419 path = compat_urlparse.urlparse(url).path
1420 return path.strip('/').split('/')[-1]
1423 class HEADRequest(compat_urllib_request.Request):
1424 def get_method(self):
1428 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1431 v = getattr(v, get_attr, None)
1437 return int(v) * invscale // scale
1442 def str_or_none(v, default=None):
1443 return default if v is None else compat_str(v)
1446 def str_to_int(int_str):
1447 """ A more relaxed version of int_or_none """
1450 int_str = re.sub(r'[,\.\+]', '', int_str)
1454 def float_or_none(v, scale=1, invscale=1, default=None):
1458 return float(v) * invscale / scale
1463 def parse_duration(s):
1464 if not isinstance(s, compat_basestring):
1472 (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
1473 (?P<only_hours>[0-9.]+)\s*(?:hours?)|
1475 \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
1478 (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
1479 (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
1481 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
1483 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
1488 if m.group('only_mins'):
1489 return float_or_none(m.group('only_mins'), invscale=60)
1490 if m.group('only_hours'):
1491 return float_or_none(m.group('only_hours'), invscale=60 * 60)
1493 res += int(m.group('secs'))
1494 if m.group('mins_reversed'):
1495 res += int(m.group('mins_reversed')) * 60
1497 res += int(m.group('mins')) * 60
1498 if m.group('hours'):
1499 res += int(m.group('hours')) * 60 * 60
1500 if m.group('hours_reversed'):
1501 res += int(m.group('hours_reversed')) * 60 * 60
1503 res += int(m.group('days')) * 24 * 60 * 60
1505 res += float(m.group('ms'))
1509 def prepend_extension(filename, ext, expected_real_ext=None):
1510 name, real_ext = os.path.splitext(filename)
1512 '{0}.{1}{2}'.format(name, ext, real_ext)
1513 if not expected_real_ext or real_ext[1:] == expected_real_ext
1514 else '{0}.{1}'.format(filename, ext))
1517 def replace_extension(filename, ext, expected_real_ext=None):
1518 name, real_ext = os.path.splitext(filename)
1519 return '{0}.{1}'.format(
1520 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1524 def check_executable(exe, args=[]):
1525 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1526 args can be a list of arguments for a short output (like -version) """
1528 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1534 def get_exe_version(exe, args=['--version'],
1535 version_re=None, unrecognized='present'):
1536 """ Returns the version of the specified executable,
1537 or False if the executable is not present """
1539 out, _ = subprocess.Popen(
1540 [encodeArgument(exe)] + args,
1541 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1544 if isinstance(out, bytes): # Python 2.x
1545 out = out.decode('ascii', 'ignore')
1546 return detect_exe_version(out, version_re, unrecognized)
1549 def detect_exe_version(output, version_re=None, unrecognized='present'):
1550 assert isinstance(output, compat_str)
1551 if version_re is None:
1552 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1553 m = re.search(version_re, output)
1560 class PagedList(object):
1562 # This is only useful for tests
1563 return len(self.getslice())
1566 class OnDemandPagedList(PagedList):
1567 def __init__(self, pagefunc, pagesize):
1568 self._pagefunc = pagefunc
1569 self._pagesize = pagesize
1571 def getslice(self, start=0, end=None):
1573 for pagenum in itertools.count(start // self._pagesize):
1574 firstid = pagenum * self._pagesize
1575 nextfirstid = pagenum * self._pagesize + self._pagesize
1576 if start >= nextfirstid:
1579 page_results = list(self._pagefunc(pagenum))
1582 start % self._pagesize
1583 if firstid <= start < nextfirstid
1587 ((end - 1) % self._pagesize) + 1
1588 if (end is not None and firstid <= end <= nextfirstid)
1591 if startv != 0 or endv is not None:
1592 page_results = page_results[startv:endv]
1593 res.extend(page_results)
1595 # A little optimization - if current page is not "full", ie. does
1596 # not contain page_size videos then we can assume that this page
1597 # is the last one - there are no more ids on further pages -
1598 # i.e. no need to query again.
1599 if len(page_results) + startv < self._pagesize:
1602 # If we got the whole page, but the next page is not interesting,
1603 # break out early as well
1604 if end == nextfirstid:
1609 class InAdvancePagedList(PagedList):
1610 def __init__(self, pagefunc, pagecount, pagesize):
1611 self._pagefunc = pagefunc
1612 self._pagecount = pagecount
1613 self._pagesize = pagesize
1615 def getslice(self, start=0, end=None):
1617 start_page = start // self._pagesize
1619 self._pagecount if end is None else (end // self._pagesize + 1))
1620 skip_elems = start - start_page * self._pagesize
1621 only_more = None if end is None else end - start
1622 for pagenum in range(start_page, end_page):
1623 page = list(self._pagefunc(pagenum))
1625 page = page[skip_elems:]
1627 if only_more is not None:
1628 if len(page) < only_more:
1629 only_more -= len(page)
1631 page = page[:only_more]
1638 def uppercase_escape(s):
1639 unicode_escape = codecs.getdecoder('unicode_escape')
1641 r'\\U[0-9a-fA-F]{8}',
1642 lambda m: unicode_escape(m.group(0))[0],
1646 def lowercase_escape(s):
1647 unicode_escape = codecs.getdecoder('unicode_escape')
1649 r'\\u[0-9a-fA-F]{4}',
1650 lambda m: unicode_escape(m.group(0))[0],
1654 def escape_rfc3986(s):
1655 """Escape non-ASCII characters as suggested by RFC 3986"""
1656 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1657 s = s.encode('utf-8')
1658 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1661 def escape_url(url):
1662 """Escape URL as suggested by RFC 3986"""
1663 url_parsed = compat_urllib_parse_urlparse(url)
1664 return url_parsed._replace(
1665 path=escape_rfc3986(url_parsed.path),
1666 params=escape_rfc3986(url_parsed.params),
1667 query=escape_rfc3986(url_parsed.query),
1668 fragment=escape_rfc3986(url_parsed.fragment)
1672 struct.pack('!I', 0)
1674 # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1675 def struct_pack(spec, *args):
1676 if isinstance(spec, compat_str):
1677 spec = spec.encode('ascii')
1678 return struct.pack(spec, *args)
1680 def struct_unpack(spec, *args):
1681 if isinstance(spec, compat_str):
1682 spec = spec.encode('ascii')
1683 return struct.unpack(spec, *args)
1685 struct_pack = struct.pack
1686 struct_unpack = struct.unpack
1689 def read_batch_urls(batch_fd):
1691 if not isinstance(url, compat_str):
1692 url = url.decode('utf-8', 'replace')
1693 BOM_UTF8 = '\xef\xbb\xbf'
1694 if url.startswith(BOM_UTF8):
1695 url = url[len(BOM_UTF8):]
1697 if url.startswith(('#', ';', ']')):
1701 with contextlib.closing(batch_fd) as fd:
1702 return [url for url in map(fixup, fd) if url]
1705 def urlencode_postdata(*args, **kargs):
1706 return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
1709 def encode_dict(d, encoding='utf-8'):
1711 return v.encode(encoding) if isinstance(v, compat_basestring) else v
1712 return dict((encode(k), encode(v)) for k, v in d.items())
1724 def parse_age_limit(s):
1727 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1728 return int(m.group('age')) if m else US_RATINGS.get(s, None)
1731 def strip_jsonp(code):
1733 r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1736 def js_to_json(code):
1739 if v in ('true', 'false', 'null'):
1741 if v.startswith('"'):
1742 v = re.sub(r"\\'", "'", v[1:-1])
1743 elif v.startswith("'"):
1745 v = re.sub(r"\\\\|\\'|\"", lambda m: {
1752 res = re.sub(r'''(?x)
1753 "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
1754 '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
1755 [a-zA-Z_][.a-zA-Z_0-9]*
1757 res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
1761 def qualities(quality_ids):
1762 """ Get a numeric quality value out of a list of possible values """
1765 return quality_ids.index(qid)
1771 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1774 def limit_length(s, length):
1775 """ Add ellipses to overly long strings """
1780 return s[:length - len(ELLIPSES)] + ELLIPSES
1784 def version_tuple(v):
1785 return tuple(int(e) for e in re.split(r'[-.]', v))
1788 def is_outdated_version(version, limit, assume_new=True):
1790 return not assume_new
1792 return version_tuple(version) < version_tuple(limit)
1794 return not assume_new
1797 def ytdl_is_updateable():
1798 """ Returns if youtube-dl can be updated with -U """
1799 from zipimport import zipimporter
1801 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
1804 def args_to_str(args):
1805 # Get a short string representation for a subprocess command
1806 return ' '.join(shlex_quote(a) for a in args)
1809 def mimetype2ext(mt):
1810 _, _, res = mt.rpartition('/')
1814 'x-mp4-fragmented': 'mp4',
1819 def urlhandle_detect_ext(url_handle):
1822 getheader = lambda h: url_handle.headers[h]
1823 except AttributeError: # Python < 3
1824 getheader = url_handle.info().getheader
1826 cd = getheader('Content-Disposition')
1828 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
1830 e = determine_ext(m.group('filename'), default_ext=None)
1834 return mimetype2ext(getheader('Content-Type'))
1837 def encode_data_uri(data, mime_type):
1838 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
1841 def age_restricted(content_limit, age_limit):
1842 """ Returns True iff the content should be blocked """
1844 if age_limit is None: # No limit set
1846 if content_limit is None:
1847 return False # Content available for everyone
1848 return age_limit < content_limit
1851 def is_html(first_bytes):
1852 """ Detect whether a file contains HTML by examining its first bytes. """
1855 (b'\xef\xbb\xbf', 'utf-8'),
1856 (b'\x00\x00\xfe\xff', 'utf-32-be'),
1857 (b'\xff\xfe\x00\x00', 'utf-32-le'),
1858 (b'\xff\xfe', 'utf-16-le'),
1859 (b'\xfe\xff', 'utf-16-be'),
1861 for bom, enc in BOMS:
1862 if first_bytes.startswith(bom):
1863 s = first_bytes[len(bom):].decode(enc, 'replace')
1866 s = first_bytes.decode('utf-8', 'replace')
1868 return re.match(r'^\s*<', s)
1871 def determine_protocol(info_dict):
1872 protocol = info_dict.get('protocol')
1873 if protocol is not None:
1876 url = info_dict['url']
1877 if url.startswith('rtmp'):
1879 elif url.startswith('mms'):
1881 elif url.startswith('rtsp'):
1884 ext = determine_ext(url)
1890 return compat_urllib_parse_urlparse(url).scheme
1893 def render_table(header_row, data):
1894 """ Render a list of rows, each as a list of values """
1895 table = [header_row] + data
1896 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
1897 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
1898 return '\n'.join(format_str % tuple(row) for row in table)
1901 def _match_one(filter_part, dct):
1902 COMPARISON_OPERATORS = {
1910 operator_rex = re.compile(r'''(?x)\s*
1912 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1914 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
1915 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
1918 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
1919 m = operator_rex.search(filter_part)
1921 op = COMPARISON_OPERATORS[m.group('op')]
1922 if m.group('strval') is not None:
1923 if m.group('op') not in ('=', '!='):
1925 'Operator %s does not support string values!' % m.group('op'))
1926 comparison_value = m.group('strval')
1929 comparison_value = int(m.group('intval'))
1931 comparison_value = parse_filesize(m.group('intval'))
1932 if comparison_value is None:
1933 comparison_value = parse_filesize(m.group('intval') + 'B')
1934 if comparison_value is None:
1936 'Invalid integer value %r in filter part %r' % (
1937 m.group('intval'), filter_part))
1938 actual_value = dct.get(m.group('key'))
1939 if actual_value is None:
1940 return m.group('none_inclusive')
1941 return op(actual_value, comparison_value)
1944 '': lambda v: v is not None,
1945 '!': lambda v: v is None,
1947 operator_rex = re.compile(r'''(?x)\s*
1948 (?P<op>%s)\s*(?P<key>[a-z_]+)
1950 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
1951 m = operator_rex.search(filter_part)
1953 op = UNARY_OPERATORS[m.group('op')]
1954 actual_value = dct.get(m.group('key'))
1955 return op(actual_value)
1957 raise ValueError('Invalid filter part %r' % filter_part)
1960 def match_str(filter_str, dct):
1961 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
1964 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
1967 def match_filter_func(filter_str):
1968 def _match_func(info_dict):
1969 if match_str(filter_str, info_dict):
1972 video_title = info_dict.get('title', info_dict.get('id', 'video'))
1973 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
1977 def parse_dfxp_time_expr(time_expr):
1981 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
1983 return float(mobj.group('time_offset'))
1985 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:\.\d+)?)$', time_expr)
1987 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
1990 def srt_subtitles_timecode(seconds):
1991 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
1994 def dfxp2srt(dfxp_data):
1995 _x = functools.partial(xpath_with_ns, ns_map={
1996 'ttml': 'http://www.w3.org/ns/ttml',
1997 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
2000 def parse_node(node):
2001 str_or_empty = functools.partial(str_or_none, default='')
2003 out = str_or_empty(node.text)
2006 if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
2007 out += '\n' + str_or_empty(child.tail)
2008 elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
2009 out += str_or_empty(parse_node(child))
2011 out += str_or_empty(xml.etree.ElementTree.tostring(child))
2015 dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
2017 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
2020 raise ValueError('Invalid dfxp/TTML subtitle')
2022 for para, index in zip(paras, itertools.count(1)):
2023 begin_time = parse_dfxp_time_expr(para.attrib['begin'])
2024 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
2026 end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
2027 out.append('%d\n%s --> %s\n%s\n\n' % (
2029 srt_subtitles_timecode(begin_time),
2030 srt_subtitles_timecode(end_time),
2036 def cli_option(params, command_option, param):
2037 param = params.get(param)
2038 return [command_option, param] if param is not None else []
2041 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
2042 param = params.get(param)
2043 assert isinstance(param, bool)
2045 return [command_option + separator + (true_value if param else false_value)]
2046 return [command_option, true_value if param else false_value]
2049 def cli_valueless_option(params, command_option, param, expected_value=True):
2050 param = params.get(param)
2051 return [command_option] if param == expected_value else []
2054 def cli_configuration_args(params, param, default=[]):
2055 ex_args = params.get(param)
2058 assert isinstance(ex_args, list)
2062 class ISO639Utils(object):
2063 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2252 def short2long(cls, code):
2253 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2254 return cls._lang_map.get(code[:2])
2257 def long2short(cls, code):
2258 """Convert language code from ISO 639-2/T to ISO 639-1"""
2259 for short_name, long_name in cls._lang_map.items():
2260 if long_name == code:
2264 class ISO3166Utils(object):
2265 # From http://data.okfn.org/data/core/country-list
2267 'AF': 'Afghanistan',
2268 'AX': 'Åland Islands',
2271 'AS': 'American Samoa',
2276 'AG': 'Antigua and Barbuda',
2293 'BO': 'Bolivia, Plurinational State of',
2294 'BQ': 'Bonaire, Sint Eustatius and Saba',
2295 'BA': 'Bosnia and Herzegovina',
2297 'BV': 'Bouvet Island',
2299 'IO': 'British Indian Ocean Territory',
2300 'BN': 'Brunei Darussalam',
2302 'BF': 'Burkina Faso',
2308 'KY': 'Cayman Islands',
2309 'CF': 'Central African Republic',
2313 'CX': 'Christmas Island',
2314 'CC': 'Cocos (Keeling) Islands',
2318 'CD': 'Congo, the Democratic Republic of the',
2319 'CK': 'Cook Islands',
2321 'CI': 'Côte d\'Ivoire',
2326 'CZ': 'Czech Republic',
2330 'DO': 'Dominican Republic',
2333 'SV': 'El Salvador',
2334 'GQ': 'Equatorial Guinea',
2338 'FK': 'Falkland Islands (Malvinas)',
2339 'FO': 'Faroe Islands',
2343 'GF': 'French Guiana',
2344 'PF': 'French Polynesia',
2345 'TF': 'French Southern Territories',
2360 'GW': 'Guinea-Bissau',
2363 'HM': 'Heard Island and McDonald Islands',
2364 'VA': 'Holy See (Vatican City State)',
2371 'IR': 'Iran, Islamic Republic of',
2374 'IM': 'Isle of Man',
2384 'KP': 'Korea, Democratic People\'s Republic of',
2385 'KR': 'Korea, Republic of',
2388 'LA': 'Lao People\'s Democratic Republic',
2394 'LI': 'Liechtenstein',
2398 'MK': 'Macedonia, the Former Yugoslav Republic of',
2405 'MH': 'Marshall Islands',
2411 'FM': 'Micronesia, Federated States of',
2412 'MD': 'Moldova, Republic of',
2423 'NL': 'Netherlands',
2424 'NC': 'New Caledonia',
2425 'NZ': 'New Zealand',
2430 'NF': 'Norfolk Island',
2431 'MP': 'Northern Mariana Islands',
2436 'PS': 'Palestine, State of',
2438 'PG': 'Papua New Guinea',
2441 'PH': 'Philippines',
2445 'PR': 'Puerto Rico',
2449 'RU': 'Russian Federation',
2451 'BL': 'Saint Barthélemy',
2452 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2453 'KN': 'Saint Kitts and Nevis',
2454 'LC': 'Saint Lucia',
2455 'MF': 'Saint Martin (French part)',
2456 'PM': 'Saint Pierre and Miquelon',
2457 'VC': 'Saint Vincent and the Grenadines',
2460 'ST': 'Sao Tome and Principe',
2461 'SA': 'Saudi Arabia',
2465 'SL': 'Sierra Leone',
2467 'SX': 'Sint Maarten (Dutch part)',
2470 'SB': 'Solomon Islands',
2472 'ZA': 'South Africa',
2473 'GS': 'South Georgia and the South Sandwich Islands',
2474 'SS': 'South Sudan',
2479 'SJ': 'Svalbard and Jan Mayen',
2482 'CH': 'Switzerland',
2483 'SY': 'Syrian Arab Republic',
2484 'TW': 'Taiwan, Province of China',
2486 'TZ': 'Tanzania, United Republic of',
2488 'TL': 'Timor-Leste',
2492 'TT': 'Trinidad and Tobago',
2495 'TM': 'Turkmenistan',
2496 'TC': 'Turks and Caicos Islands',
2500 'AE': 'United Arab Emirates',
2501 'GB': 'United Kingdom',
2502 'US': 'United States',
2503 'UM': 'United States Minor Outlying Islands',
2507 'VE': 'Venezuela, Bolivarian Republic of',
2509 'VG': 'Virgin Islands, British',
2510 'VI': 'Virgin Islands, U.S.',
2511 'WF': 'Wallis and Futuna',
2512 'EH': 'Western Sahara',
2519 def short2full(cls, code):
2520 """Convert an ISO 3166-2 country code to the corresponding full name"""
2521 return cls._country_map.get(code.upper())
2524 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2525 def __init__(self, proxies=None):
2526 # Set default handlers
2527 for type in ('http', 'https'):
2528 setattr(self, '%s_open' % type,
2529 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2530 meth(r, proxy, type))
2531 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2533 def proxy_open(self, req, proxy, type):
2534 req_proxy = req.headers.get('Ytdl-request-proxy')
2535 if req_proxy is not None:
2537 del req.headers['Ytdl-request-proxy']
2539 if proxy == '__noproxy__':
2540 return None # No Proxy
2541 return compat_urllib_request.ProxyHandler.proxy_open(
2542 self, req, proxy, type)