2 # -*- coding: utf-8 -*-
4 from __future__ import unicode_literals
34 import xml.etree.ElementTree
40 compat_etree_fromstring,
45 compat_socket_create_connection,
49 compat_urllib_parse_urlparse,
50 compat_urllib_request,
56 # This is not clearly defined otherwise
57 compiled_regex_type = type(re.compile(''))
60 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/44.0 (Chrome)',
61 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
62 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
63 'Accept-Encoding': 'gzip, deflate',
64 'Accept-Language': 'en-us,en;q=0.5',
70 ENGLISH_MONTH_NAMES = [
71 'January', 'February', 'March', 'April', 'May', 'June',
72 'July', 'August', 'September', 'October', 'November', 'December']
75 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
76 'flv', 'f4v', 'f4a', 'f4b',
77 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
87 'f4f', 'f4m', 'm3u8', 'smil')
90 def preferredencoding():
91 """Get preferred encoding.
93 Returns the best encoding scheme for the system, based on
94 locale.getpreferredencoding() and some further tweaks.
97 pref = locale.getpreferredencoding()
105 def write_json_file(obj, fn):
106 """ Encode obj as JSON and write it to fn, atomically if possible """
108 fn = encodeFilename(fn)
109 if sys.version_info < (3, 0) and sys.platform != 'win32':
110 encoding = get_filesystem_encoding()
111 # os.path.basename returns a bytes object, but NamedTemporaryFile
112 # will fail if the filename contains non ascii characters unless we
113 # use a unicode object
114 path_basename = lambda f: os.path.basename(fn).decode(encoding)
115 # the same for os.path.dirname
116 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
118 path_basename = os.path.basename
119 path_dirname = os.path.dirname
123 'prefix': path_basename(fn) + '.',
124 'dir': path_dirname(fn),
128 # In Python 2.x, json.dump expects a bytestream.
129 # In Python 3.x, it writes to a character stream
130 if sys.version_info < (3, 0):
138 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
143 if sys.platform == 'win32':
144 # Need to remove existing file on Windows, else os.rename raises
145 # WindowsError or FileExistsError.
150 os.rename(tf.name, fn)
159 if sys.version_info >= (2, 7):
160 def find_xpath_attr(node, xpath, key, val=None):
161 """ Find the xpath xpath[@key=val] """
162 assert re.match(r'^[a-zA-Z_-]+$', key)
163 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
164 return node.find(expr)
166 def find_xpath_attr(node, xpath, key, val=None):
167 # Here comes the crazy part: In 2.6, if the xpath is a unicode,
168 # .//node does not match if a node is a direct child of . !
169 if isinstance(xpath, compat_str):
170 xpath = xpath.encode('ascii')
172 for f in node.findall(xpath):
173 if key not in f.attrib:
175 if val is None or f.attrib.get(key) == val:
179 # On python2.6 the xml.etree.ElementTree.Element methods don't support
180 # the namespace parameter
183 def xpath_with_ns(path, ns_map):
184 components = [c.split(':') for c in path.split('/')]
188 replaced.append(c[0])
191 replaced.append('{%s}%s' % (ns_map[ns], tag))
192 return '/'.join(replaced)
195 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
196 def _find_xpath(xpath):
197 if sys.version_info < (2, 7): # Crazy 2.6
198 xpath = xpath.encode('ascii')
199 return node.find(xpath)
201 if isinstance(xpath, (str, compat_str)):
202 n = _find_xpath(xpath)
210 if default is not NO_DEFAULT:
213 name = xpath if name is None else name
214 raise ExtractorError('Could not find XML element %s' % name)
220 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
221 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
222 if n is None or n == default:
225 if default is not NO_DEFAULT:
228 name = xpath if name is None else name
229 raise ExtractorError('Could not find XML element\'s text %s' % name)
235 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
236 n = find_xpath_attr(node, xpath, key)
238 if default is not NO_DEFAULT:
241 name = '%s[@%s]' % (xpath, key) if name is None else name
242 raise ExtractorError('Could not find XML attribute %s' % name)
248 def get_element_by_id(id, html):
249 """Return the content of the tag with the specified ID in the passed HTML document"""
250 return get_element_by_attribute('id', id, html)
253 def get_element_by_attribute(attribute, value, html):
254 """Return the content of the tag with the specified attribute in the passed HTML document"""
256 m = re.search(r'''(?xs)
258 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
260 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
264 ''' % (re.escape(attribute), re.escape(value)), html)
268 res = m.group('content')
270 if res.startswith('"') or res.startswith("'"):
273 return unescapeHTML(res)
276 def clean_html(html):
277 """Clean an HTML snippet into a readable string"""
279 if html is None: # Convenience for sanitizing descriptions etc.
283 html = html.replace('\n', ' ')
284 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
285 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
287 html = re.sub('<.*?>', '', html)
288 # Replace html entities
289 html = unescapeHTML(html)
293 def sanitize_open(filename, open_mode):
294 """Try to open the given filename, and slightly tweak it if this fails.
296 Attempts to open the given filename. If this fails, it tries to change
297 the filename slightly, step by step, until it's either able to open it
298 or it fails and raises a final exception, like the standard open()
301 It returns the tuple (stream, definitive_file_name).
305 if sys.platform == 'win32':
307 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
308 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
309 stream = open(encodeFilename(filename), open_mode)
310 return (stream, filename)
311 except (IOError, OSError) as err:
312 if err.errno in (errno.EACCES,):
315 # In case of error, try to remove win32 forbidden chars
316 alt_filename = sanitize_path(filename)
317 if alt_filename == filename:
320 # An exception here should be caught in the caller
321 stream = open(encodeFilename(alt_filename), open_mode)
322 return (stream, alt_filename)
325 def timeconvert(timestr):
326 """Convert RFC 2822 defined time string into system timestamp"""
328 timetuple = email.utils.parsedate_tz(timestr)
329 if timetuple is not None:
330 timestamp = email.utils.mktime_tz(timetuple)
334 def sanitize_filename(s, restricted=False, is_id=False):
335 """Sanitizes a string so it could be used as part of a filename.
336 If restricted is set, use a stricter subset of allowed characters.
337 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
339 def replace_insane(char):
340 if char == '?' or ord(char) < 32 or ord(char) == 127:
343 return '' if restricted else '\''
345 return '_-' if restricted else ' -'
346 elif char in '\\/|*<>':
348 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
350 if restricted and ord(char) > 127:
355 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
356 result = ''.join(map(replace_insane, s))
358 while '__' in result:
359 result = result.replace('__', '_')
360 result = result.strip('_')
361 # Common case of "Foreign band name - English song title"
362 if restricted and result.startswith('-_'):
364 if result.startswith('-'):
365 result = '_' + result[len('-'):]
366 result = result.lstrip('.')
372 def sanitize_path(s):
373 """Sanitizes and normalizes path on Windows"""
374 if sys.platform != 'win32':
376 drive_or_unc, _ = os.path.splitdrive(s)
377 if sys.version_info < (2, 7) and not drive_or_unc:
378 drive_or_unc, _ = os.path.splitunc(s)
379 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
383 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part)
384 for path_part in norm_path]
386 sanitized_path.insert(0, drive_or_unc + os.path.sep)
387 return os.path.join(*sanitized_path)
390 # Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
391 # unwanted failures due to missing protocol
392 def sanitized_Request(url, *args, **kwargs):
393 return compat_urllib_request.Request(
394 'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
397 def orderedSet(iterable):
398 """ Remove all duplicates from the input iterable """
406 def _htmlentity_transform(entity):
407 """Transforms an HTML entity to a character."""
408 # Known non-numeric HTML entity
409 if entity in compat_html_entities.name2codepoint:
410 return compat_chr(compat_html_entities.name2codepoint[entity])
412 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
414 numstr = mobj.group(1)
415 if numstr.startswith('x'):
417 numstr = '0%s' % numstr
420 # See https://github.com/rg3/youtube-dl/issues/7518
422 return compat_chr(int(numstr, base))
426 # Unknown entity in name, return its literal representation
427 return '&%s;' % entity
433 assert type(s) == compat_str
436 r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
439 def get_subprocess_encoding():
440 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
441 # For subprocess calls, encode with locale encoding
442 # Refer to http://stackoverflow.com/a/9951851/35070
443 encoding = preferredencoding()
445 encoding = sys.getfilesystemencoding()
451 def encodeFilename(s, for_subprocess=False):
453 @param s The name of the file
456 assert type(s) == compat_str
458 # Python 3 has a Unicode API
459 if sys.version_info >= (3, 0):
462 # Pass '' directly to use Unicode APIs on Windows 2000 and up
463 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
464 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
465 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
468 # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
469 if sys.platform.startswith('java'):
472 return s.encode(get_subprocess_encoding(), 'ignore')
475 def decodeFilename(b, for_subprocess=False):
477 if sys.version_info >= (3, 0):
480 if not isinstance(b, bytes):
483 return b.decode(get_subprocess_encoding(), 'ignore')
486 def encodeArgument(s):
487 if not isinstance(s, compat_str):
488 # Legacy code that uses byte strings
489 # Uncomment the following line after fixing all post processors
490 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
491 s = s.decode('ascii')
492 return encodeFilename(s, True)
495 def decodeArgument(b):
496 return decodeFilename(b, True)
499 def decodeOption(optval):
502 if isinstance(optval, bytes):
503 optval = optval.decode(preferredencoding())
505 assert isinstance(optval, compat_str)
509 def formatSeconds(secs):
511 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
513 return '%d:%02d' % (secs // 60, secs % 60)
518 def make_HTTPS_handler(params, **kwargs):
519 opts_no_check_certificate = params.get('nocheckcertificate', False)
520 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
521 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
522 if opts_no_check_certificate:
523 context.check_hostname = False
524 context.verify_mode = ssl.CERT_NONE
526 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
529 # (create_default_context present but HTTPSHandler has no context=)
532 if sys.version_info < (3, 2):
533 return YoutubeDLHTTPSHandler(params, **kwargs)
535 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
536 context.verify_mode = (ssl.CERT_NONE
537 if opts_no_check_certificate
538 else ssl.CERT_REQUIRED)
539 context.set_default_verify_paths()
540 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
543 def bug_reports_message():
544 if ytdl_is_updateable():
545 update_cmd = 'type youtube-dl -U to update'
547 update_cmd = 'see https://yt-dl.org/update on how to update'
548 msg = '; please report this issue on https://yt-dl.org/bug .'
549 msg += ' Make sure you are using the latest version; %s.' % update_cmd
550 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
554 class ExtractorError(Exception):
555 """Error during info extraction."""
557 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
558 """ tb, if given, is the original traceback (so that it can be printed out).
559 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
562 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
564 if video_id is not None:
565 msg = video_id + ': ' + msg
567 msg += ' (caused by %r)' % cause
569 msg += bug_reports_message()
570 super(ExtractorError, self).__init__(msg)
573 self.exc_info = sys.exc_info() # preserve original exception
575 self.video_id = video_id
577 def format_traceback(self):
578 if self.traceback is None:
580 return ''.join(traceback.format_tb(self.traceback))
583 class UnsupportedError(ExtractorError):
584 def __init__(self, url):
585 super(UnsupportedError, self).__init__(
586 'Unsupported URL: %s' % url, expected=True)
590 class RegexNotFoundError(ExtractorError):
591 """Error when a regex didn't match"""
595 class DownloadError(Exception):
596 """Download Error exception.
598 This exception may be thrown by FileDownloader objects if they are not
599 configured to continue on errors. They will contain the appropriate
603 def __init__(self, msg, exc_info=None):
604 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
605 super(DownloadError, self).__init__(msg)
606 self.exc_info = exc_info
609 class SameFileError(Exception):
610 """Same File exception.
612 This exception will be thrown by FileDownloader objects if they detect
613 multiple files would have to be downloaded to the same file on disk.
618 class PostProcessingError(Exception):
619 """Post Processing exception.
621 This exception may be raised by PostProcessor's .run() method to
622 indicate an error in the postprocessing task.
625 def __init__(self, msg):
629 class MaxDownloadsReached(Exception):
630 """ --max-downloads limit has been reached. """
634 class UnavailableVideoError(Exception):
635 """Unavailable Format exception.
637 This exception will be thrown when a video is requested
638 in a format that is not available for that video.
643 class ContentTooShortError(Exception):
644 """Content Too Short exception.
646 This exception may be raised by FileDownloader objects when a file they
647 download is too small for what the server announced first, indicating
648 the connection was probably interrupted.
651 def __init__(self, downloaded, expected):
653 self.downloaded = downloaded
654 self.expected = expected
657 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
658 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
659 # expected HTTP responses to meet HTTP/1.0 or later (see also
660 # https://github.com/rg3/youtube-dl/issues/6727)
661 if sys.version_info < (3, 0):
662 kwargs[b'strict'] = True
663 hc = http_class(*args, **kwargs)
664 source_address = ydl_handler._params.get('source_address')
665 if source_address is not None:
666 sa = (source_address, 0)
667 if hasattr(hc, 'source_address'): # Python 2.7+
668 hc.source_address = sa
670 def _hc_connect(self, *args, **kwargs):
671 sock = compat_socket_create_connection(
672 (self.host, self.port), self.timeout, sa)
674 self.sock = ssl.wrap_socket(
675 sock, self.key_file, self.cert_file,
676 ssl_version=ssl.PROTOCOL_TLSv1)
679 hc.connect = functools.partial(_hc_connect, hc)
684 def handle_youtubedl_headers(headers):
685 filtered_headers = headers
687 if 'Youtubedl-no-compression' in filtered_headers:
688 filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
689 del filtered_headers['Youtubedl-no-compression']
691 return filtered_headers
694 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
695 """Handler for HTTP requests and responses.
697 This class, when installed with an OpenerDirector, automatically adds
698 the standard headers to every HTTP request and handles gzipped and
699 deflated responses from web servers. If compression is to be avoided in
700 a particular request, the original request in the program code only has
701 to include the HTTP header "Youtubedl-no-compression", which will be
702 removed before making the real request.
704 Part of this code was copied from:
706 http://techknack.net/python-urllib2-handlers/
708 Andrew Rowls, the author of that code, agreed to release it to the
712 def __init__(self, params, *args, **kwargs):
713 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
714 self._params = params
716 def http_open(self, req):
717 return self.do_open(functools.partial(
718 _create_http_connection, self, compat_http_client.HTTPConnection, False),
724 return zlib.decompress(data, -zlib.MAX_WBITS)
726 return zlib.decompress(data)
729 def addinfourl_wrapper(stream, headers, url, code):
730 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
731 return compat_urllib_request.addinfourl(stream, headers, url, code)
732 ret = compat_urllib_request.addinfourl(stream, headers, url)
736 def http_request(self, req):
737 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
738 # always respected by websites, some tend to give out URLs with non percent-encoded
739 # non-ASCII characters (see telemb.py, ard.py [#3412])
740 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
741 # To work around aforementioned issue we will replace request's original URL with
742 # percent-encoded one
743 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
744 # the code of this workaround has been moved here from YoutubeDL.urlopen()
745 url = req.get_full_url()
746 url_escaped = escape_url(url)
748 # Substitute URL if any change after escaping
749 if url != url_escaped:
750 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
752 url_escaped, data=req.data, headers=req.headers,
753 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
754 new_req.timeout = req.timeout
757 for h, v in std_headers.items():
758 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
759 # The dict keys are capitalized because of this bug by urllib
760 if h.capitalize() not in req.headers:
763 req.headers = handle_youtubedl_headers(req.headers)
765 if sys.version_info < (2, 7) and '#' in req.get_full_url():
766 # Python 2.6 is brain-dead when it comes to fragments
767 req._Request__original = req._Request__original.partition('#')[0]
768 req._Request__r_type = req._Request__r_type.partition('#')[0]
772 def http_response(self, req, resp):
775 if resp.headers.get('Content-encoding', '') == 'gzip':
776 content = resp.read()
777 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
779 uncompressed = io.BytesIO(gz.read())
780 except IOError as original_ioerror:
781 # There may be junk add the end of the file
782 # See http://stackoverflow.com/q/4928560/35070 for details
783 for i in range(1, 1024):
785 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
786 uncompressed = io.BytesIO(gz.read())
791 raise original_ioerror
792 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
793 resp.msg = old_resp.msg
794 del resp.headers['Content-encoding']
796 if resp.headers.get('Content-encoding', '') == 'deflate':
797 gz = io.BytesIO(self.deflate(resp.read()))
798 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
799 resp.msg = old_resp.msg
800 del resp.headers['Content-encoding']
801 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
802 # https://github.com/rg3/youtube-dl/issues/6457).
803 if 300 <= resp.code < 400:
804 location = resp.headers.get('Location')
806 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
807 if sys.version_info >= (3, 0):
808 location = location.encode('iso-8859-1').decode('utf-8')
809 location_escaped = escape_url(location)
810 if location != location_escaped:
811 del resp.headers['Location']
812 resp.headers['Location'] = location_escaped
815 https_request = http_request
816 https_response = http_response
819 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
820 def __init__(self, params, https_conn_class=None, *args, **kwargs):
821 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
822 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
823 self._params = params
825 def https_open(self, req):
827 if hasattr(self, '_context'): # python > 2.6
828 kwargs['context'] = self._context
829 if hasattr(self, '_check_hostname'): # python 3.x
830 kwargs['check_hostname'] = self._check_hostname
831 return self.do_open(functools.partial(
832 _create_http_connection, self, self._https_conn_class, True),
836 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
837 def __init__(self, cookiejar=None):
838 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
840 def http_response(self, request, response):
841 # Python 2 will choke on next HTTP request in row if there are non-ASCII
842 # characters in Set-Cookie HTTP header of last response (see
843 # https://github.com/rg3/youtube-dl/issues/6769).
844 # In order to at least prevent crashing we will percent encode Set-Cookie
845 # header before HTTPCookieProcessor starts processing it.
846 # if sys.version_info < (3, 0) and response.headers:
847 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
848 # set_cookie = response.headers.get(set_cookie_header)
850 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
851 # if set_cookie != set_cookie_escaped:
852 # del response.headers[set_cookie_header]
853 # response.headers[set_cookie_header] = set_cookie_escaped
854 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
856 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
857 https_response = http_response
860 def parse_iso8601(date_str, delimiter='T', timezone=None):
861 """ Return a UNIX timestamp from the given date """
866 date_str = re.sub(r'\.[0-9]+', '', date_str)
870 r'(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
873 timezone = datetime.timedelta()
875 date_str = date_str[:-len(m.group(0))]
876 if not m.group('sign'):
877 timezone = datetime.timedelta()
879 sign = 1 if m.group('sign') == '+' else -1
880 timezone = datetime.timedelta(
881 hours=sign * int(m.group('hours')),
882 minutes=sign * int(m.group('minutes')))
884 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
885 dt = datetime.datetime.strptime(date_str, date_format) - timezone
886 return calendar.timegm(dt.timetuple())
891 def unified_strdate(date_str, day_first=True):
892 """Return a string with the date in the format YYYYMMDD"""
898 date_str = date_str.replace(',', ' ')
899 # %z (UTC offset) is only supported in python>=3.2
900 if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
901 date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
902 # Remove AM/PM + timezone
903 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
905 format_expressions = [
918 '%Y-%m-%d %H:%M:%S.%f',
921 '%Y-%m-%dT%H:%M:%SZ',
922 '%Y-%m-%dT%H:%M:%S.%fZ',
923 '%Y-%m-%dT%H:%M:%S.%f0Z',
925 '%Y-%m-%dT%H:%M:%S.%f',
929 format_expressions.extend([
937 format_expressions.extend([
944 for expression in format_expressions:
946 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
949 if upload_date is None:
950 timetuple = email.utils.parsedate_tz(date_str)
952 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
953 if upload_date is not None:
954 return compat_str(upload_date)
957 def determine_ext(url, default_ext='unknown_video'):
960 guess = url.partition('?')[0].rpartition('.')[2]
961 if re.match(r'^[A-Za-z0-9]+$', guess):
963 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
964 elif guess.rstrip('/') in KNOWN_EXTENSIONS:
965 return guess.rstrip('/')
970 def subtitles_filename(filename, sub_lang, sub_format):
971 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
974 def date_from_str(date_str):
976 Return a datetime object from a string in the format YYYYMMDD or
977 (now|today)[+-][0-9](day|week|month|year)(s)?"""
978 today = datetime.date.today()
979 if date_str in ('now', 'today'):
981 if date_str == 'yesterday':
982 return today - datetime.timedelta(days=1)
983 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
984 if match is not None:
985 sign = match.group('sign')
986 time = int(match.group('time'))
989 unit = match.group('unit')
990 # A bad approximation?
998 delta = datetime.timedelta(**{unit: time})
1000 return datetime.datetime.strptime(date_str, '%Y%m%d').date()
1003 def hyphenate_date(date_str):
1005 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
1006 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
1007 if match is not None:
1008 return '-'.join(match.groups())
1013 class DateRange(object):
1014 """Represents a time interval between two dates"""
1016 def __init__(self, start=None, end=None):
1017 """start and end must be strings in the format accepted by date"""
1018 if start is not None:
1019 self.start = date_from_str(start)
1021 self.start = datetime.datetime.min.date()
1023 self.end = date_from_str(end)
1025 self.end = datetime.datetime.max.date()
1026 if self.start > self.end:
1027 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
1031 """Returns a range that only contains the given day"""
1032 return cls(day, day)
1034 def __contains__(self, date):
1035 """Check if the date is in the range"""
1036 if not isinstance(date, datetime.date):
1037 date = date_from_str(date)
1038 return self.start <= date <= self.end
1041 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
1044 def platform_name():
1045 """ Returns the platform name as a compat_str """
1046 res = platform.platform()
1047 if isinstance(res, bytes):
1048 res = res.decode(preferredencoding())
1050 assert isinstance(res, compat_str)
1054 def _windows_write_string(s, out):
1055 """ Returns True if the string was written using special methods,
1056 False if it has yet to be written out."""
1057 # Adapted from http://stackoverflow.com/a/3259271/35070
1060 import ctypes.wintypes
1068 fileno = out.fileno()
1069 except AttributeError:
1070 # If the output stream doesn't have a fileno, it's virtual
1072 except io.UnsupportedOperation:
1073 # Some strange Windows pseudo files?
1075 if fileno not in WIN_OUTPUT_IDS:
1078 GetStdHandle = ctypes.WINFUNCTYPE(
1079 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1080 (b'GetStdHandle', ctypes.windll.kernel32))
1081 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1083 WriteConsoleW = ctypes.WINFUNCTYPE(
1084 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1085 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1086 ctypes.wintypes.LPVOID)((b'WriteConsoleW', ctypes.windll.kernel32))
1087 written = ctypes.wintypes.DWORD(0)
1089 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b'GetFileType', ctypes.windll.kernel32))
1090 FILE_TYPE_CHAR = 0x0002
1091 FILE_TYPE_REMOTE = 0x8000
1092 GetConsoleMode = ctypes.WINFUNCTYPE(
1093 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1094 ctypes.POINTER(ctypes.wintypes.DWORD))(
1095 (b'GetConsoleMode', ctypes.windll.kernel32))
1096 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1098 def not_a_console(handle):
1099 if handle == INVALID_HANDLE_VALUE or handle is None:
1101 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1102 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1104 if not_a_console(h):
1107 def next_nonbmp_pos(s):
1109 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1110 except StopIteration:
1114 count = min(next_nonbmp_pos(s), 1024)
1116 ret = WriteConsoleW(
1117 h, s, count if count else 2, ctypes.byref(written), None)
1119 raise OSError('Failed to write string')
1120 if not count: # We just wrote a non-BMP character
1121 assert written.value == 2
1124 assert written.value > 0
1125 s = s[written.value:]
1129 def write_string(s, out=None, encoding=None):
1132 assert type(s) == compat_str
1134 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1135 if _windows_write_string(s, out):
1138 if ('b' in getattr(out, 'mode', '') or
1139 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1140 byt = s.encode(encoding or preferredencoding(), 'ignore')
1142 elif hasattr(out, 'buffer'):
1143 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1144 byt = s.encode(enc, 'ignore')
1145 out.buffer.write(byt)
1151 def bytes_to_intlist(bs):
1154 if isinstance(bs[0], int): # Python 3
1157 return [ord(c) for c in bs]
1160 def intlist_to_bytes(xs):
1163 return struct_pack('%dB' % len(xs), *xs)
1166 # Cross-platform file locking
1167 if sys.platform == 'win32':
1168 import ctypes.wintypes
1171 class OVERLAPPED(ctypes.Structure):
1173 ('Internal', ctypes.wintypes.LPVOID),
1174 ('InternalHigh', ctypes.wintypes.LPVOID),
1175 ('Offset', ctypes.wintypes.DWORD),
1176 ('OffsetHigh', ctypes.wintypes.DWORD),
1177 ('hEvent', ctypes.wintypes.HANDLE),
1180 kernel32 = ctypes.windll.kernel32
1181 LockFileEx = kernel32.LockFileEx
1182 LockFileEx.argtypes = [
1183 ctypes.wintypes.HANDLE, # hFile
1184 ctypes.wintypes.DWORD, # dwFlags
1185 ctypes.wintypes.DWORD, # dwReserved
1186 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1187 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1188 ctypes.POINTER(OVERLAPPED) # Overlapped
1190 LockFileEx.restype = ctypes.wintypes.BOOL
1191 UnlockFileEx = kernel32.UnlockFileEx
1192 UnlockFileEx.argtypes = [
1193 ctypes.wintypes.HANDLE, # hFile
1194 ctypes.wintypes.DWORD, # dwReserved
1195 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1196 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1197 ctypes.POINTER(OVERLAPPED) # Overlapped
1199 UnlockFileEx.restype = ctypes.wintypes.BOOL
1200 whole_low = 0xffffffff
1201 whole_high = 0x7fffffff
1203 def _lock_file(f, exclusive):
1204 overlapped = OVERLAPPED()
1205 overlapped.Offset = 0
1206 overlapped.OffsetHigh = 0
1207 overlapped.hEvent = 0
1208 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1209 handle = msvcrt.get_osfhandle(f.fileno())
1210 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1211 whole_low, whole_high, f._lock_file_overlapped_p):
1212 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1214 def _unlock_file(f):
1215 assert f._lock_file_overlapped_p
1216 handle = msvcrt.get_osfhandle(f.fileno())
1217 if not UnlockFileEx(handle, 0,
1218 whole_low, whole_high, f._lock_file_overlapped_p):
1219 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1222 # Some platforms, such as Jython, is missing fcntl
1226 def _lock_file(f, exclusive):
1227 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1229 def _unlock_file(f):
1230 fcntl.flock(f, fcntl.LOCK_UN)
1232 UNSUPPORTED_MSG = 'file locking is not supported on this platform'
1234 def _lock_file(f, exclusive):
1235 raise IOError(UNSUPPORTED_MSG)
1237 def _unlock_file(f):
1238 raise IOError(UNSUPPORTED_MSG)
1241 class locked_file(object):
1242 def __init__(self, filename, mode, encoding=None):
1243 assert mode in ['r', 'a', 'w']
1244 self.f = io.open(filename, mode, encoding=encoding)
1247 def __enter__(self):
1248 exclusive = self.mode != 'r'
1250 _lock_file(self.f, exclusive)
1256 def __exit__(self, etype, value, traceback):
1258 _unlock_file(self.f)
1265 def write(self, *args):
1266 return self.f.write(*args)
1268 def read(self, *args):
1269 return self.f.read(*args)
1272 def get_filesystem_encoding():
1273 encoding = sys.getfilesystemencoding()
1274 return encoding if encoding is not None else 'utf-8'
1277 def shell_quote(args):
1279 encoding = get_filesystem_encoding()
1281 if isinstance(a, bytes):
1282 # We may get a filename encoded with 'encodeFilename'
1283 a = a.decode(encoding)
1284 quoted_args.append(pipes.quote(a))
1285 return ' '.join(quoted_args)
1288 def smuggle_url(url, data):
1289 """ Pass additional data in a URL for internal use. """
1291 sdata = compat_urllib_parse.urlencode(
1292 {'__youtubedl_smuggle': json.dumps(data)})
1293 return url + '#' + sdata
1296 def unsmuggle_url(smug_url, default=None):
1297 if '#__youtubedl_smuggle' not in smug_url:
1298 return smug_url, default
1299 url, _, sdata = smug_url.rpartition('#')
1300 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1301 data = json.loads(jsond)
1305 def format_bytes(bytes):
1308 if type(bytes) is str:
1309 bytes = float(bytes)
1313 exponent = int(math.log(bytes, 1024.0))
1314 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1315 converted = float(bytes) / float(1024 ** exponent)
1316 return '%.2f%s' % (converted, suffix)
1319 def lookup_unit_table(unit_table, s):
1320 units_re = '|'.join(re.escape(u) for u in unit_table)
1322 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
1325 num_str = m.group('num').replace(',', '.')
1326 mult = unit_table[m.group('unit')]
1327 return int(float(num_str) * mult)
1330 def parse_filesize(s):
1334 # The lower-case forms are of course incorrect and unofficial,
1335 # but we support those too
1373 return lookup_unit_table(_UNIT_TABLE, s)
1382 if re.match(r'^[\d,.]+$', s):
1383 return str_to_int(s)
1394 return lookup_unit_table(_UNIT_TABLE, s)
1396 def month_by_name(name):
1397 """ Return the number of a month by (locale-independently) English name """
1400 return ENGLISH_MONTH_NAMES.index(name) + 1
1405 def month_by_abbreviation(abbrev):
1406 """ Return the number of a month by (locale-independently) English
1410 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1415 def fix_xml_ampersands(xml_str):
1416 """Replace all the '&' by '&' in XML"""
1418 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1423 def setproctitle(title):
1424 assert isinstance(title, compat_str)
1426 # ctypes in Jython is not complete
1427 # http://bugs.jython.org/issue2148
1428 if sys.platform.startswith('java'):
1432 libc = ctypes.cdll.LoadLibrary('libc.so.6')
1435 title_bytes = title.encode('utf-8')
1436 buf = ctypes.create_string_buffer(len(title_bytes))
1437 buf.value = title_bytes
1439 libc.prctl(15, buf, 0, 0, 0)
1440 except AttributeError:
1441 return # Strange libc, just skip this
1444 def remove_start(s, start):
1445 if s.startswith(start):
1446 return s[len(start):]
1450 def remove_end(s, end):
1452 return s[:-len(end)]
1456 def remove_quotes(s):
1457 if s is None or len(s) < 2:
1459 for quote in ('"', "'", ):
1460 if s[0] == quote and s[-1] == quote:
1465 def url_basename(url):
1466 path = compat_urlparse.urlparse(url).path
1467 return path.strip('/').split('/')[-1]
1470 class HEADRequest(compat_urllib_request.Request):
1471 def get_method(self):
1475 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1478 v = getattr(v, get_attr, None)
1484 return int(v) * invscale // scale
1489 def str_or_none(v, default=None):
1490 return default if v is None else compat_str(v)
1493 def str_to_int(int_str):
1494 """ A more relaxed version of int_or_none """
1497 int_str = re.sub(r'[,\.\+]', '', int_str)
1501 def float_or_none(v, scale=1, invscale=1, default=None):
1505 return float(v) * invscale / scale
1510 def parse_duration(s):
1511 if not isinstance(s, compat_basestring):
1519 (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
1520 (?P<only_hours>[0-9.]+)\s*(?:hours?)|
1522 \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
1525 (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
1526 (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
1528 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
1530 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
1535 if m.group('only_mins'):
1536 return float_or_none(m.group('only_mins'), invscale=60)
1537 if m.group('only_hours'):
1538 return float_or_none(m.group('only_hours'), invscale=60 * 60)
1540 res += int(m.group('secs'))
1541 if m.group('mins_reversed'):
1542 res += int(m.group('mins_reversed')) * 60
1544 res += int(m.group('mins')) * 60
1545 if m.group('hours'):
1546 res += int(m.group('hours')) * 60 * 60
1547 if m.group('hours_reversed'):
1548 res += int(m.group('hours_reversed')) * 60 * 60
1550 res += int(m.group('days')) * 24 * 60 * 60
1552 res += float(m.group('ms'))
1556 def prepend_extension(filename, ext, expected_real_ext=None):
1557 name, real_ext = os.path.splitext(filename)
1559 '{0}.{1}{2}'.format(name, ext, real_ext)
1560 if not expected_real_ext or real_ext[1:] == expected_real_ext
1561 else '{0}.{1}'.format(filename, ext))
1564 def replace_extension(filename, ext, expected_real_ext=None):
1565 name, real_ext = os.path.splitext(filename)
1566 return '{0}.{1}'.format(
1567 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1571 def check_executable(exe, args=[]):
1572 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1573 args can be a list of arguments for a short output (like -version) """
1575 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1581 def get_exe_version(exe, args=['--version'],
1582 version_re=None, unrecognized='present'):
1583 """ Returns the version of the specified executable,
1584 or False if the executable is not present """
1586 out, _ = subprocess.Popen(
1587 [encodeArgument(exe)] + args,
1588 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1591 if isinstance(out, bytes): # Python 2.x
1592 out = out.decode('ascii', 'ignore')
1593 return detect_exe_version(out, version_re, unrecognized)
1596 def detect_exe_version(output, version_re=None, unrecognized='present'):
1597 assert isinstance(output, compat_str)
1598 if version_re is None:
1599 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1600 m = re.search(version_re, output)
1607 class PagedList(object):
1609 # This is only useful for tests
1610 return len(self.getslice())
1613 class OnDemandPagedList(PagedList):
1614 def __init__(self, pagefunc, pagesize, use_cache=False):
1615 self._pagefunc = pagefunc
1616 self._pagesize = pagesize
1617 self._use_cache = use_cache
1621 def getslice(self, start=0, end=None):
1623 for pagenum in itertools.count(start // self._pagesize):
1624 firstid = pagenum * self._pagesize
1625 nextfirstid = pagenum * self._pagesize + self._pagesize
1626 if start >= nextfirstid:
1631 page_results = self._cache.get(pagenum)
1632 if page_results is None:
1633 page_results = list(self._pagefunc(pagenum))
1635 self._cache[pagenum] = page_results
1638 start % self._pagesize
1639 if firstid <= start < nextfirstid
1643 ((end - 1) % self._pagesize) + 1
1644 if (end is not None and firstid <= end <= nextfirstid)
1647 if startv != 0 or endv is not None:
1648 page_results = page_results[startv:endv]
1649 res.extend(page_results)
1651 # A little optimization - if current page is not "full", ie. does
1652 # not contain page_size videos then we can assume that this page
1653 # is the last one - there are no more ids on further pages -
1654 # i.e. no need to query again.
1655 if len(page_results) + startv < self._pagesize:
1658 # If we got the whole page, but the next page is not interesting,
1659 # break out early as well
1660 if end == nextfirstid:
1665 class InAdvancePagedList(PagedList):
1666 def __init__(self, pagefunc, pagecount, pagesize):
1667 self._pagefunc = pagefunc
1668 self._pagecount = pagecount
1669 self._pagesize = pagesize
1671 def getslice(self, start=0, end=None):
1673 start_page = start // self._pagesize
1675 self._pagecount if end is None else (end // self._pagesize + 1))
1676 skip_elems = start - start_page * self._pagesize
1677 only_more = None if end is None else end - start
1678 for pagenum in range(start_page, end_page):
1679 page = list(self._pagefunc(pagenum))
1681 page = page[skip_elems:]
1683 if only_more is not None:
1684 if len(page) < only_more:
1685 only_more -= len(page)
1687 page = page[:only_more]
1694 def uppercase_escape(s):
1695 unicode_escape = codecs.getdecoder('unicode_escape')
1697 r'\\U[0-9a-fA-F]{8}',
1698 lambda m: unicode_escape(m.group(0))[0],
1702 def lowercase_escape(s):
1703 unicode_escape = codecs.getdecoder('unicode_escape')
1705 r'\\u[0-9a-fA-F]{4}',
1706 lambda m: unicode_escape(m.group(0))[0],
1710 def escape_rfc3986(s):
1711 """Escape non-ASCII characters as suggested by RFC 3986"""
1712 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1713 s = s.encode('utf-8')
1714 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1717 def escape_url(url):
1718 """Escape URL as suggested by RFC 3986"""
1719 url_parsed = compat_urllib_parse_urlparse(url)
1720 return url_parsed._replace(
1721 path=escape_rfc3986(url_parsed.path),
1722 params=escape_rfc3986(url_parsed.params),
1723 query=escape_rfc3986(url_parsed.query),
1724 fragment=escape_rfc3986(url_parsed.fragment)
1728 struct.pack('!I', 0)
1730 # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1731 def struct_pack(spec, *args):
1732 if isinstance(spec, compat_str):
1733 spec = spec.encode('ascii')
1734 return struct.pack(spec, *args)
1736 def struct_unpack(spec, *args):
1737 if isinstance(spec, compat_str):
1738 spec = spec.encode('ascii')
1739 return struct.unpack(spec, *args)
1741 struct_pack = struct.pack
1742 struct_unpack = struct.unpack
1745 def read_batch_urls(batch_fd):
1747 if not isinstance(url, compat_str):
1748 url = url.decode('utf-8', 'replace')
1749 BOM_UTF8 = '\xef\xbb\xbf'
1750 if url.startswith(BOM_UTF8):
1751 url = url[len(BOM_UTF8):]
1753 if url.startswith(('#', ';', ']')):
1757 with contextlib.closing(batch_fd) as fd:
1758 return [url for url in map(fixup, fd) if url]
1761 def urlencode_postdata(*args, **kargs):
1762 return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
1765 def update_url_query(url, query):
1766 parsed_url = compat_urlparse.urlparse(url)
1767 qs = compat_parse_qs(parsed_url.query)
1769 qs = encode_dict(qs)
1770 return compat_urlparse.urlunparse(parsed_url._replace(
1771 query=compat_urllib_parse.urlencode(qs, True)))
1774 def encode_dict(d, encoding='utf-8'):
1776 return v.encode(encoding) if isinstance(v, compat_basestring) else v
1777 return dict((encode(k), encode(v)) for k, v in d.items())
1780 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
1781 if isinstance(key_or_keys, (list, tuple)):
1782 for key in key_or_keys:
1783 if key not in d or d[key] is None or skip_false_values and not d[key]:
1787 return d.get(key_or_keys, default)
1790 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
1791 return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
1803 def parse_age_limit(s):
1806 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1807 return int(m.group('age')) if m else US_RATINGS.get(s)
1810 def strip_jsonp(code):
1812 r'(?s)^[a-zA-Z0-9_.]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1815 def js_to_json(code):
1818 if v in ('true', 'false', 'null'):
1820 if v.startswith('"'):
1821 v = re.sub(r"\\'", "'", v[1:-1])
1822 elif v.startswith("'"):
1824 v = re.sub(r"\\\\|\\'|\"", lambda m: {
1831 res = re.sub(r'''(?x)
1832 "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
1833 '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
1834 [a-zA-Z_][.a-zA-Z_0-9]*
1836 res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
1840 def qualities(quality_ids):
1841 """ Get a numeric quality value out of a list of possible values """
1844 return quality_ids.index(qid)
1850 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1853 def limit_length(s, length):
1854 """ Add ellipses to overly long strings """
1859 return s[:length - len(ELLIPSES)] + ELLIPSES
1863 def version_tuple(v):
1864 return tuple(int(e) for e in re.split(r'[-.]', v))
1867 def is_outdated_version(version, limit, assume_new=True):
1869 return not assume_new
1871 return version_tuple(version) < version_tuple(limit)
1873 return not assume_new
1876 def ytdl_is_updateable():
1877 """ Returns if youtube-dl can be updated with -U """
1878 from zipimport import zipimporter
1880 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
1883 def args_to_str(args):
1884 # Get a short string representation for a subprocess command
1885 return ' '.join(shlex_quote(a) for a in args)
1888 def error_to_compat_str(err):
1890 # On python 2 error byte string must be decoded with proper
1891 # encoding rather than ascii
1892 if sys.version_info[0] < 3:
1893 err_str = err_str.decode(preferredencoding())
1897 def mimetype2ext(mt):
1904 _, _, res = mt.rpartition('/')
1908 'smptett+xml': 'tt',
1914 'x-mp4-fragmented': 'mp4',
1919 def urlhandle_detect_ext(url_handle):
1922 getheader = lambda h: url_handle.headers[h]
1923 except AttributeError: # Python < 3
1924 getheader = url_handle.info().getheader
1926 cd = getheader('Content-Disposition')
1928 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
1930 e = determine_ext(m.group('filename'), default_ext=None)
1934 return mimetype2ext(getheader('Content-Type'))
1937 def encode_data_uri(data, mime_type):
1938 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
1941 def age_restricted(content_limit, age_limit):
1942 """ Returns True iff the content should be blocked """
1944 if age_limit is None: # No limit set
1946 if content_limit is None:
1947 return False # Content available for everyone
1948 return age_limit < content_limit
1951 def is_html(first_bytes):
1952 """ Detect whether a file contains HTML by examining its first bytes. """
1955 (b'\xef\xbb\xbf', 'utf-8'),
1956 (b'\x00\x00\xfe\xff', 'utf-32-be'),
1957 (b'\xff\xfe\x00\x00', 'utf-32-le'),
1958 (b'\xff\xfe', 'utf-16-le'),
1959 (b'\xfe\xff', 'utf-16-be'),
1961 for bom, enc in BOMS:
1962 if first_bytes.startswith(bom):
1963 s = first_bytes[len(bom):].decode(enc, 'replace')
1966 s = first_bytes.decode('utf-8', 'replace')
1968 return re.match(r'^\s*<', s)
1971 def determine_protocol(info_dict):
1972 protocol = info_dict.get('protocol')
1973 if protocol is not None:
1976 url = info_dict['url']
1977 if url.startswith('rtmp'):
1979 elif url.startswith('mms'):
1981 elif url.startswith('rtsp'):
1984 ext = determine_ext(url)
1990 return compat_urllib_parse_urlparse(url).scheme
1993 def render_table(header_row, data):
1994 """ Render a list of rows, each as a list of values """
1995 table = [header_row] + data
1996 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
1997 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
1998 return '\n'.join(format_str % tuple(row) for row in table)
2001 def _match_one(filter_part, dct):
2002 COMPARISON_OPERATORS = {
2010 operator_rex = re.compile(r'''(?x)\s*
2012 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
2014 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
2015 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
2018 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
2019 m = operator_rex.search(filter_part)
2021 op = COMPARISON_OPERATORS[m.group('op')]
2022 if m.group('strval') is not None:
2023 if m.group('op') not in ('=', '!='):
2025 'Operator %s does not support string values!' % m.group('op'))
2026 comparison_value = m.group('strval')
2029 comparison_value = int(m.group('intval'))
2031 comparison_value = parse_filesize(m.group('intval'))
2032 if comparison_value is None:
2033 comparison_value = parse_filesize(m.group('intval') + 'B')
2034 if comparison_value is None:
2036 'Invalid integer value %r in filter part %r' % (
2037 m.group('intval'), filter_part))
2038 actual_value = dct.get(m.group('key'))
2039 if actual_value is None:
2040 return m.group('none_inclusive')
2041 return op(actual_value, comparison_value)
2044 '': lambda v: v is not None,
2045 '!': lambda v: v is None,
2047 operator_rex = re.compile(r'''(?x)\s*
2048 (?P<op>%s)\s*(?P<key>[a-z_]+)
2050 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
2051 m = operator_rex.search(filter_part)
2053 op = UNARY_OPERATORS[m.group('op')]
2054 actual_value = dct.get(m.group('key'))
2055 return op(actual_value)
2057 raise ValueError('Invalid filter part %r' % filter_part)
2060 def match_str(filter_str, dct):
2061 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
2064 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
2067 def match_filter_func(filter_str):
2068 def _match_func(info_dict):
2069 if match_str(filter_str, info_dict):
2072 video_title = info_dict.get('title', info_dict.get('id', 'video'))
2073 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
2077 def parse_dfxp_time_expr(time_expr):
2081 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
2083 return float(mobj.group('time_offset'))
2085 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
2087 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
2090 def srt_subtitles_timecode(seconds):
2091 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
2094 def dfxp2srt(dfxp_data):
2095 _x = functools.partial(xpath_with_ns, ns_map={
2096 'ttml': 'http://www.w3.org/ns/ttml',
2097 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
2100 class TTMLPElementParser(object):
2103 def start(self, tag, attrib):
2104 if tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
2110 def data(self, data):
2114 return self.out.strip()
2116 def parse_node(node):
2117 target = TTMLPElementParser()
2118 parser = xml.etree.ElementTree.XMLParser(target=target)
2119 parser.feed(xml.etree.ElementTree.tostring(node))
2120 return parser.close()
2122 dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
2124 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
2127 raise ValueError('Invalid dfxp/TTML subtitle')
2129 for para, index in zip(paras, itertools.count(1)):
2130 begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
2131 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
2132 dur = parse_dfxp_time_expr(para.attrib.get('dur'))
2133 if begin_time is None:
2138 end_time = begin_time + dur
2139 out.append('%d\n%s --> %s\n%s\n\n' % (
2141 srt_subtitles_timecode(begin_time),
2142 srt_subtitles_timecode(end_time),
2148 def cli_option(params, command_option, param):
2149 param = params.get(param)
2150 return [command_option, param] if param is not None else []
2153 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
2154 param = params.get(param)
2155 assert isinstance(param, bool)
2157 return [command_option + separator + (true_value if param else false_value)]
2158 return [command_option, true_value if param else false_value]
2161 def cli_valueless_option(params, command_option, param, expected_value=True):
2162 param = params.get(param)
2163 return [command_option] if param == expected_value else []
2166 def cli_configuration_args(params, param, default=[]):
2167 ex_args = params.get(param)
2170 assert isinstance(ex_args, list)
2174 class ISO639Utils(object):
2175 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2364 def short2long(cls, code):
2365 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2366 return cls._lang_map.get(code[:2])
2369 def long2short(cls, code):
2370 """Convert language code from ISO 639-2/T to ISO 639-1"""
2371 for short_name, long_name in cls._lang_map.items():
2372 if long_name == code:
2376 class ISO3166Utils(object):
2377 # From http://data.okfn.org/data/core/country-list
2379 'AF': 'Afghanistan',
2380 'AX': 'Ã…land Islands',
2383 'AS': 'American Samoa',
2388 'AG': 'Antigua and Barbuda',
2405 'BO': 'Bolivia, Plurinational State of',
2406 'BQ': 'Bonaire, Sint Eustatius and Saba',
2407 'BA': 'Bosnia and Herzegovina',
2409 'BV': 'Bouvet Island',
2411 'IO': 'British Indian Ocean Territory',
2412 'BN': 'Brunei Darussalam',
2414 'BF': 'Burkina Faso',
2420 'KY': 'Cayman Islands',
2421 'CF': 'Central African Republic',
2425 'CX': 'Christmas Island',
2426 'CC': 'Cocos (Keeling) Islands',
2430 'CD': 'Congo, the Democratic Republic of the',
2431 'CK': 'Cook Islands',
2433 'CI': 'Côte d\'Ivoire',
2438 'CZ': 'Czech Republic',
2442 'DO': 'Dominican Republic',
2445 'SV': 'El Salvador',
2446 'GQ': 'Equatorial Guinea',
2450 'FK': 'Falkland Islands (Malvinas)',
2451 'FO': 'Faroe Islands',
2455 'GF': 'French Guiana',
2456 'PF': 'French Polynesia',
2457 'TF': 'French Southern Territories',
2472 'GW': 'Guinea-Bissau',
2475 'HM': 'Heard Island and McDonald Islands',
2476 'VA': 'Holy See (Vatican City State)',
2483 'IR': 'Iran, Islamic Republic of',
2486 'IM': 'Isle of Man',
2496 'KP': 'Korea, Democratic People\'s Republic of',
2497 'KR': 'Korea, Republic of',
2500 'LA': 'Lao People\'s Democratic Republic',
2506 'LI': 'Liechtenstein',
2510 'MK': 'Macedonia, the Former Yugoslav Republic of',
2517 'MH': 'Marshall Islands',
2523 'FM': 'Micronesia, Federated States of',
2524 'MD': 'Moldova, Republic of',
2535 'NL': 'Netherlands',
2536 'NC': 'New Caledonia',
2537 'NZ': 'New Zealand',
2542 'NF': 'Norfolk Island',
2543 'MP': 'Northern Mariana Islands',
2548 'PS': 'Palestine, State of',
2550 'PG': 'Papua New Guinea',
2553 'PH': 'Philippines',
2557 'PR': 'Puerto Rico',
2561 'RU': 'Russian Federation',
2563 'BL': 'Saint Barthélemy',
2564 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2565 'KN': 'Saint Kitts and Nevis',
2566 'LC': 'Saint Lucia',
2567 'MF': 'Saint Martin (French part)',
2568 'PM': 'Saint Pierre and Miquelon',
2569 'VC': 'Saint Vincent and the Grenadines',
2572 'ST': 'Sao Tome and Principe',
2573 'SA': 'Saudi Arabia',
2577 'SL': 'Sierra Leone',
2579 'SX': 'Sint Maarten (Dutch part)',
2582 'SB': 'Solomon Islands',
2584 'ZA': 'South Africa',
2585 'GS': 'South Georgia and the South Sandwich Islands',
2586 'SS': 'South Sudan',
2591 'SJ': 'Svalbard and Jan Mayen',
2594 'CH': 'Switzerland',
2595 'SY': 'Syrian Arab Republic',
2596 'TW': 'Taiwan, Province of China',
2598 'TZ': 'Tanzania, United Republic of',
2600 'TL': 'Timor-Leste',
2604 'TT': 'Trinidad and Tobago',
2607 'TM': 'Turkmenistan',
2608 'TC': 'Turks and Caicos Islands',
2612 'AE': 'United Arab Emirates',
2613 'GB': 'United Kingdom',
2614 'US': 'United States',
2615 'UM': 'United States Minor Outlying Islands',
2619 'VE': 'Venezuela, Bolivarian Republic of',
2621 'VG': 'Virgin Islands, British',
2622 'VI': 'Virgin Islands, U.S.',
2623 'WF': 'Wallis and Futuna',
2624 'EH': 'Western Sahara',
2631 def short2full(cls, code):
2632 """Convert an ISO 3166-2 country code to the corresponding full name"""
2633 return cls._country_map.get(code.upper())
2636 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2637 def __init__(self, proxies=None):
2638 # Set default handlers
2639 for type in ('http', 'https'):
2640 setattr(self, '%s_open' % type,
2641 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2642 meth(r, proxy, type))
2643 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2645 def proxy_open(self, req, proxy, type):
2646 req_proxy = req.headers.get('Ytdl-request-proxy')
2647 if req_proxy is not None:
2649 del req.headers['Ytdl-request-proxy']
2651 if proxy == '__noproxy__':
2652 return None # No Proxy
2653 return compat_urllib_request.ProxyHandler.proxy_open(
2654 self, req, proxy, type)
2657 def ohdave_rsa_encrypt(data, exponent, modulus):
2659 Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
2662 data: data to encrypt, bytes-like object
2663 exponent, modulus: parameter e and N of RSA algorithm, both integer
2664 Output: hex string of encrypted data
2666 Limitation: supports one block encryption only
2669 payload = int(binascii.hexlify(data[::-1]), 16)
2670 encrypted = pow(payload, exponent, modulus)
2671 return '%x' % encrypted
2674 def encode_base_n(num, n, table=None):
2675 FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
2677 table = FULL_TABLE[:n]
2680 raise ValueError('base %d exceeds table length %d' % (n, len(table)))
2687 ret = table[num % n] + ret
2692 def decode_packed_codes(code):
2694 r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)",
2696 obfucasted_code, base, count, symbols = mobj.groups()
2699 symbols = symbols.split('|')
2704 base_n_count = encode_base_n(count, base)
2705 symbol_table[base_n_count] = symbols[count] or base_n_count
2708 r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],