2 # -*- coding: utf-8 -*-
4 from __future__ import unicode_literals
32 import xml.etree.ElementTree
42 compat_socket_create_connection,
46 compat_urllib_parse_urlparse,
47 compat_urllib_request,
53 # This is not clearly defined otherwise
54 compiled_regex_type = type(re.compile(''))
57 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
58 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
59 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
60 'Accept-Encoding': 'gzip, deflate',
61 'Accept-Language': 'en-us,en;q=0.5',
67 ENGLISH_MONTH_NAMES = [
68 'January', 'February', 'March', 'April', 'May', 'June',
69 'July', 'August', 'September', 'October', 'November', 'December']
72 def preferredencoding():
73 """Get preferred encoding.
75 Returns the best encoding scheme for the system, based on
76 locale.getpreferredencoding() and some further tweaks.
79 pref = locale.getpreferredencoding()
87 def write_json_file(obj, fn):
88 """ Encode obj as JSON and write it to fn, atomically if possible """
90 fn = encodeFilename(fn)
91 if sys.version_info < (3, 0) and sys.platform != 'win32':
92 encoding = get_filesystem_encoding()
93 # os.path.basename returns a bytes object, but NamedTemporaryFile
94 # will fail if the filename contains non ascii characters unless we
95 # use a unicode object
96 path_basename = lambda f: os.path.basename(fn).decode(encoding)
97 # the same for os.path.dirname
98 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
100 path_basename = os.path.basename
101 path_dirname = os.path.dirname
105 'prefix': path_basename(fn) + '.',
106 'dir': path_dirname(fn),
110 # In Python 2.x, json.dump expects a bytestream.
111 # In Python 3.x, it writes to a character stream
112 if sys.version_info < (3, 0):
120 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
125 if sys.platform == 'win32':
126 # Need to remove existing file on Windows, else os.rename raises
127 # WindowsError or FileExistsError.
132 os.rename(tf.name, fn)
141 if sys.version_info >= (2, 7):
142 def find_xpath_attr(node, xpath, key, val=None):
143 """ Find the xpath xpath[@key=val] """
144 assert re.match(r'^[a-zA-Z-]+$', key)
146 assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
147 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
148 return node.find(expr)
150 def find_xpath_attr(node, xpath, key, val=None):
151 # Here comes the crazy part: In 2.6, if the xpath is a unicode,
152 # .//node does not match if a node is a direct child of . !
153 if isinstance(xpath, compat_str):
154 xpath = xpath.encode('ascii')
156 for f in node.findall(xpath):
157 if key not in f.attrib:
159 if val is None or f.attrib.get(key) == val:
163 # On python2.6 the xml.etree.ElementTree.Element methods don't support
164 # the namespace parameter
167 def xpath_with_ns(path, ns_map):
168 components = [c.split(':') for c in path.split('/')]
172 replaced.append(c[0])
175 replaced.append('{%s}%s' % (ns_map[ns], tag))
176 return '/'.join(replaced)
179 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
180 if sys.version_info < (2, 7): # Crazy 2.6
181 xpath = xpath.encode('ascii')
184 if n is None or n.text is None:
185 if default is not NO_DEFAULT:
188 name = xpath if name is None else name
189 raise ExtractorError('Could not find XML element %s' % name)
195 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
196 return xpath_element(node, xpath, name, fatal=fatal, default=default).text
199 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
200 n = find_xpath_attr(node, xpath, key)
202 if default is not NO_DEFAULT:
205 name = '%s[@%s]' % (xpath, key) if name is None else name
206 raise ExtractorError('Could not find XML attribute %s' % name)
212 def get_element_by_id(id, html):
213 """Return the content of the tag with the specified ID in the passed HTML document"""
214 return get_element_by_attribute("id", id, html)
217 def get_element_by_attribute(attribute, value, html):
218 """Return the content of the tag with the specified attribute in the passed HTML document"""
220 m = re.search(r'''(?xs)
222 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
224 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
228 ''' % (re.escape(attribute), re.escape(value)), html)
232 res = m.group('content')
234 if res.startswith('"') or res.startswith("'"):
237 return unescapeHTML(res)
240 def clean_html(html):
241 """Clean an HTML snippet into a readable string"""
243 if html is None: # Convenience for sanitizing descriptions etc.
247 html = html.replace('\n', ' ')
248 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
249 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
251 html = re.sub('<.*?>', '', html)
252 # Replace html entities
253 html = unescapeHTML(html)
257 def sanitize_open(filename, open_mode):
258 """Try to open the given filename, and slightly tweak it if this fails.
260 Attempts to open the given filename. If this fails, it tries to change
261 the filename slightly, step by step, until it's either able to open it
262 or it fails and raises a final exception, like the standard open()
265 It returns the tuple (stream, definitive_file_name).
269 if sys.platform == 'win32':
271 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
272 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
273 stream = open(encodeFilename(filename), open_mode)
274 return (stream, filename)
275 except (IOError, OSError) as err:
276 if err.errno in (errno.EACCES,):
279 # In case of error, try to remove win32 forbidden chars
280 alt_filename = sanitize_path(filename)
281 if alt_filename == filename:
284 # An exception here should be caught in the caller
285 stream = open(encodeFilename(alt_filename), open_mode)
286 return (stream, alt_filename)
289 def timeconvert(timestr):
290 """Convert RFC 2822 defined time string into system timestamp"""
292 timetuple = email.utils.parsedate_tz(timestr)
293 if timetuple is not None:
294 timestamp = email.utils.mktime_tz(timetuple)
298 def sanitize_filename(s, restricted=False, is_id=False):
299 """Sanitizes a string so it could be used as part of a filename.
300 If restricted is set, use a stricter subset of allowed characters.
301 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
303 def replace_insane(char):
304 if char == '?' or ord(char) < 32 or ord(char) == 127:
307 return '' if restricted else '\''
309 return '_-' if restricted else ' -'
310 elif char in '\\/|*<>':
312 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
314 if restricted and ord(char) > 127:
319 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
320 result = ''.join(map(replace_insane, s))
322 while '__' in result:
323 result = result.replace('__', '_')
324 result = result.strip('_')
325 # Common case of "Foreign band name - English song title"
326 if restricted and result.startswith('-_'):
328 if result.startswith('-'):
329 result = '_' + result[len('-'):]
330 result = result.lstrip('.')
336 def sanitize_path(s):
337 """Sanitizes and normalizes path on Windows"""
338 if sys.platform != 'win32':
340 drive_or_unc, _ = os.path.splitdrive(s)
341 if sys.version_info < (2, 7) and not drive_or_unc:
342 drive_or_unc, _ = os.path.splitunc(s)
343 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
347 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
348 for path_part in norm_path]
350 sanitized_path.insert(0, drive_or_unc + os.path.sep)
351 return os.path.join(*sanitized_path)
354 def orderedSet(iterable):
355 """ Remove all duplicates from the input iterable """
363 def _htmlentity_transform(entity):
364 """Transforms an HTML entity to a character."""
365 # Known non-numeric HTML entity
366 if entity in compat_html_entities.name2codepoint:
367 return compat_chr(compat_html_entities.name2codepoint[entity])
369 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
371 numstr = mobj.group(1)
372 if numstr.startswith('x'):
374 numstr = '0%s' % numstr
377 return compat_chr(int(numstr, base))
379 # Unknown entity in name, return its literal representation
380 return ('&%s;' % entity)
386 assert type(s) == compat_str
389 r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
392 def get_subprocess_encoding():
393 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
394 # For subprocess calls, encode with locale encoding
395 # Refer to http://stackoverflow.com/a/9951851/35070
396 encoding = preferredencoding()
398 encoding = sys.getfilesystemencoding()
404 def encodeFilename(s, for_subprocess=False):
406 @param s The name of the file
409 assert type(s) == compat_str
411 # Python 3 has a Unicode API
412 if sys.version_info >= (3, 0):
415 # Pass '' directly to use Unicode APIs on Windows 2000 and up
416 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
417 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
418 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
421 return s.encode(get_subprocess_encoding(), 'ignore')
424 def decodeFilename(b, for_subprocess=False):
426 if sys.version_info >= (3, 0):
429 if not isinstance(b, bytes):
432 return b.decode(get_subprocess_encoding(), 'ignore')
435 def encodeArgument(s):
436 if not isinstance(s, compat_str):
437 # Legacy code that uses byte strings
438 # Uncomment the following line after fixing all post processors
439 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
440 s = s.decode('ascii')
441 return encodeFilename(s, True)
444 def decodeArgument(b):
445 return decodeFilename(b, True)
448 def decodeOption(optval):
451 if isinstance(optval, bytes):
452 optval = optval.decode(preferredencoding())
454 assert isinstance(optval, compat_str)
458 def formatSeconds(secs):
460 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
462 return '%d:%02d' % (secs // 60, secs % 60)
467 def make_HTTPS_handler(params, **kwargs):
468 opts_no_check_certificate = params.get('nocheckcertificate', False)
469 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
470 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
471 if opts_no_check_certificate:
472 context.check_hostname = False
473 context.verify_mode = ssl.CERT_NONE
475 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
478 # (create_default_context present but HTTPSHandler has no context=)
481 if sys.version_info < (3, 2):
482 return YoutubeDLHTTPSHandler(params, **kwargs)
484 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
485 context.verify_mode = (ssl.CERT_NONE
486 if opts_no_check_certificate
487 else ssl.CERT_REQUIRED)
488 context.set_default_verify_paths()
489 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
492 def bug_reports_message():
493 if ytdl_is_updateable():
494 update_cmd = 'type youtube-dl -U to update'
496 update_cmd = 'see https://yt-dl.org/update on how to update'
497 msg = '; please report this issue on https://yt-dl.org/bug .'
498 msg += ' Make sure you are using the latest version; %s.' % update_cmd
499 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
503 class ExtractorError(Exception):
504 """Error during info extraction."""
506 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
507 """ tb, if given, is the original traceback (so that it can be printed out).
508 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
511 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
513 if video_id is not None:
514 msg = video_id + ': ' + msg
516 msg += ' (caused by %r)' % cause
518 msg += bug_reports_message()
519 super(ExtractorError, self).__init__(msg)
522 self.exc_info = sys.exc_info() # preserve original exception
524 self.video_id = video_id
526 def format_traceback(self):
527 if self.traceback is None:
529 return ''.join(traceback.format_tb(self.traceback))
532 class UnsupportedError(ExtractorError):
533 def __init__(self, url):
534 super(UnsupportedError, self).__init__(
535 'Unsupported URL: %s' % url, expected=True)
539 class RegexNotFoundError(ExtractorError):
540 """Error when a regex didn't match"""
544 class DownloadError(Exception):
545 """Download Error exception.
547 This exception may be thrown by FileDownloader objects if they are not
548 configured to continue on errors. They will contain the appropriate
552 def __init__(self, msg, exc_info=None):
553 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
554 super(DownloadError, self).__init__(msg)
555 self.exc_info = exc_info
558 class SameFileError(Exception):
559 """Same File exception.
561 This exception will be thrown by FileDownloader objects if they detect
562 multiple files would have to be downloaded to the same file on disk.
567 class PostProcessingError(Exception):
568 """Post Processing exception.
570 This exception may be raised by PostProcessor's .run() method to
571 indicate an error in the postprocessing task.
574 def __init__(self, msg):
578 class MaxDownloadsReached(Exception):
579 """ --max-downloads limit has been reached. """
583 class UnavailableVideoError(Exception):
584 """Unavailable Format exception.
586 This exception will be thrown when a video is requested
587 in a format that is not available for that video.
592 class ContentTooShortError(Exception):
593 """Content Too Short exception.
595 This exception may be raised by FileDownloader objects when a file they
596 download is too small for what the server announced first, indicating
597 the connection was probably interrupted.
600 def __init__(self, downloaded, expected):
602 self.downloaded = downloaded
603 self.expected = expected
606 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
607 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
608 # expected HTTP responses to meet HTTP/1.0 or later (see also
609 # https://github.com/rg3/youtube-dl/issues/6727)
610 if sys.version_info < (3, 0):
611 kwargs['strict'] = True
612 hc = http_class(*args, **kwargs)
613 source_address = ydl_handler._params.get('source_address')
614 if source_address is not None:
615 sa = (source_address, 0)
616 if hasattr(hc, 'source_address'): # Python 2.7+
617 hc.source_address = sa
619 def _hc_connect(self, *args, **kwargs):
620 sock = compat_socket_create_connection(
621 (self.host, self.port), self.timeout, sa)
623 self.sock = ssl.wrap_socket(
624 sock, self.key_file, self.cert_file,
625 ssl_version=ssl.PROTOCOL_TLSv1)
628 hc.connect = functools.partial(_hc_connect, hc)
633 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
634 """Handler for HTTP requests and responses.
636 This class, when installed with an OpenerDirector, automatically adds
637 the standard headers to every HTTP request and handles gzipped and
638 deflated responses from web servers. If compression is to be avoided in
639 a particular request, the original request in the program code only has
640 to include the HTTP header "Youtubedl-No-Compression", which will be
641 removed before making the real request.
643 Part of this code was copied from:
645 http://techknack.net/python-urllib2-handlers/
647 Andrew Rowls, the author of that code, agreed to release it to the
651 def __init__(self, params, *args, **kwargs):
652 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
653 self._params = params
655 def http_open(self, req):
656 return self.do_open(functools.partial(
657 _create_http_connection, self, compat_http_client.HTTPConnection, False),
663 return zlib.decompress(data, -zlib.MAX_WBITS)
665 return zlib.decompress(data)
668 def addinfourl_wrapper(stream, headers, url, code):
669 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
670 return compat_urllib_request.addinfourl(stream, headers, url, code)
671 ret = compat_urllib_request.addinfourl(stream, headers, url)
675 def http_request(self, req):
676 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
677 # always respected by websites, some tend to give out URLs with non percent-encoded
678 # non-ASCII characters (see telemb.py, ard.py [#3412])
679 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
680 # To work around aforementioned issue we will replace request's original URL with
681 # percent-encoded one
682 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
683 # the code of this workaround has been moved here from YoutubeDL.urlopen()
684 url = req.get_full_url()
685 url_escaped = escape_url(url)
687 # Substitute URL if any change after escaping
688 if url != url_escaped:
689 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
691 url_escaped, data=req.data, headers=req.headers,
692 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
693 new_req.timeout = req.timeout
696 for h, v in std_headers.items():
697 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
698 # The dict keys are capitalized because of this bug by urllib
699 if h.capitalize() not in req.headers:
701 if 'Youtubedl-no-compression' in req.headers:
702 if 'Accept-encoding' in req.headers:
703 del req.headers['Accept-encoding']
704 del req.headers['Youtubedl-no-compression']
706 if sys.version_info < (2, 7) and '#' in req.get_full_url():
707 # Python 2.6 is brain-dead when it comes to fragments
708 req._Request__original = req._Request__original.partition('#')[0]
709 req._Request__r_type = req._Request__r_type.partition('#')[0]
713 def http_response(self, req, resp):
716 if resp.headers.get('Content-encoding', '') == 'gzip':
717 content = resp.read()
718 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
720 uncompressed = io.BytesIO(gz.read())
721 except IOError as original_ioerror:
722 # There may be junk add the end of the file
723 # See http://stackoverflow.com/q/4928560/35070 for details
724 for i in range(1, 1024):
726 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
727 uncompressed = io.BytesIO(gz.read())
732 raise original_ioerror
733 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
734 resp.msg = old_resp.msg
736 if resp.headers.get('Content-encoding', '') == 'deflate':
737 gz = io.BytesIO(self.deflate(resp.read()))
738 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
739 resp.msg = old_resp.msg
740 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986
741 if 300 <= resp.code < 400:
742 location = resp.headers.get('Location')
744 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
745 if sys.version_info >= (3, 0):
746 location = location.encode('iso-8859-1').decode('utf-8')
747 location_escaped = escape_url(location)
748 if location != location_escaped:
749 del resp.headers['Location']
750 resp.headers['Location'] = location_escaped
753 https_request = http_request
754 https_response = http_response
757 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
758 def __init__(self, params, https_conn_class=None, *args, **kwargs):
759 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
760 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
761 self._params = params
763 def https_open(self, req):
765 if hasattr(self, '_context'): # python > 2.6
766 kwargs['context'] = self._context
767 if hasattr(self, '_check_hostname'): # python 3.x
768 kwargs['check_hostname'] = self._check_hostname
769 return self.do_open(functools.partial(
770 _create_http_connection, self, self._https_conn_class, True),
774 def parse_iso8601(date_str, delimiter='T', timezone=None):
775 """ Return a UNIX timestamp from the given date """
782 r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
785 timezone = datetime.timedelta()
787 date_str = date_str[:-len(m.group(0))]
788 if not m.group('sign'):
789 timezone = datetime.timedelta()
791 sign = 1 if m.group('sign') == '+' else -1
792 timezone = datetime.timedelta(
793 hours=sign * int(m.group('hours')),
794 minutes=sign * int(m.group('minutes')))
795 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
796 dt = datetime.datetime.strptime(date_str, date_format) - timezone
797 return calendar.timegm(dt.timetuple())
800 def unified_strdate(date_str, day_first=True):
801 """Return a string with the date in the format YYYYMMDD"""
807 date_str = date_str.replace(',', ' ')
808 # %z (UTC offset) is only supported in python>=3.2
809 if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
810 date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
811 # Remove AM/PM + timezone
812 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
814 format_expressions = [
819 '%b %dst %Y %I:%M%p',
820 '%b %dnd %Y %I:%M%p',
821 '%b %dth %Y %I:%M%p',
827 '%Y-%m-%d %H:%M:%S.%f',
830 '%Y-%m-%dT%H:%M:%SZ',
831 '%Y-%m-%dT%H:%M:%S.%fZ',
832 '%Y-%m-%dT%H:%M:%S.%f0Z',
834 '%Y-%m-%dT%H:%M:%S.%f',
838 format_expressions.extend([
846 format_expressions.extend([
853 for expression in format_expressions:
855 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
858 if upload_date is None:
859 timetuple = email.utils.parsedate_tz(date_str)
861 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
865 def determine_ext(url, default_ext='unknown_video'):
868 guess = url.partition('?')[0].rpartition('.')[2]
869 if re.match(r'^[A-Za-z0-9]+$', guess):
875 def subtitles_filename(filename, sub_lang, sub_format):
876 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
879 def date_from_str(date_str):
881 Return a datetime object from a string in the format YYYYMMDD or
882 (now|today)[+-][0-9](day|week|month|year)(s)?"""
883 today = datetime.date.today()
884 if date_str in ('now', 'today'):
886 if date_str == 'yesterday':
887 return today - datetime.timedelta(days=1)
888 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
889 if match is not None:
890 sign = match.group('sign')
891 time = int(match.group('time'))
894 unit = match.group('unit')
895 # A bad aproximation?
903 delta = datetime.timedelta(**{unit: time})
905 return datetime.datetime.strptime(date_str, "%Y%m%d").date()
908 def hyphenate_date(date_str):
910 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
911 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
912 if match is not None:
913 return '-'.join(match.groups())
918 class DateRange(object):
919 """Represents a time interval between two dates"""
921 def __init__(self, start=None, end=None):
922 """start and end must be strings in the format accepted by date"""
923 if start is not None:
924 self.start = date_from_str(start)
926 self.start = datetime.datetime.min.date()
928 self.end = date_from_str(end)
930 self.end = datetime.datetime.max.date()
931 if self.start > self.end:
932 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
936 """Returns a range that only contains the given day"""
939 def __contains__(self, date):
940 """Check if the date is in the range"""
941 if not isinstance(date, datetime.date):
942 date = date_from_str(date)
943 return self.start <= date <= self.end
946 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
950 """ Returns the platform name as a compat_str """
951 res = platform.platform()
952 if isinstance(res, bytes):
953 res = res.decode(preferredencoding())
955 assert isinstance(res, compat_str)
959 def _windows_write_string(s, out):
960 """ Returns True if the string was written using special methods,
961 False if it has yet to be written out."""
962 # Adapted from http://stackoverflow.com/a/3259271/35070
965 import ctypes.wintypes
973 fileno = out.fileno()
974 except AttributeError:
975 # If the output stream doesn't have a fileno, it's virtual
977 except io.UnsupportedOperation:
978 # Some strange Windows pseudo files?
980 if fileno not in WIN_OUTPUT_IDS:
983 GetStdHandle = ctypes.WINFUNCTYPE(
984 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
985 (b"GetStdHandle", ctypes.windll.kernel32))
986 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
988 WriteConsoleW = ctypes.WINFUNCTYPE(
989 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
990 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
991 ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
992 written = ctypes.wintypes.DWORD(0)
994 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
995 FILE_TYPE_CHAR = 0x0002
996 FILE_TYPE_REMOTE = 0x8000
997 GetConsoleMode = ctypes.WINFUNCTYPE(
998 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
999 ctypes.POINTER(ctypes.wintypes.DWORD))(
1000 (b"GetConsoleMode", ctypes.windll.kernel32))
1001 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1003 def not_a_console(handle):
1004 if handle == INVALID_HANDLE_VALUE or handle is None:
1006 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1007 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1009 if not_a_console(h):
1012 def next_nonbmp_pos(s):
1014 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1015 except StopIteration:
1019 count = min(next_nonbmp_pos(s), 1024)
1021 ret = WriteConsoleW(
1022 h, s, count if count else 2, ctypes.byref(written), None)
1024 raise OSError('Failed to write string')
1025 if not count: # We just wrote a non-BMP character
1026 assert written.value == 2
1029 assert written.value > 0
1030 s = s[written.value:]
1034 def write_string(s, out=None, encoding=None):
1037 assert type(s) == compat_str
1039 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1040 if _windows_write_string(s, out):
1043 if ('b' in getattr(out, 'mode', '') or
1044 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1045 byt = s.encode(encoding or preferredencoding(), 'ignore')
1047 elif hasattr(out, 'buffer'):
1048 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1049 byt = s.encode(enc, 'ignore')
1050 out.buffer.write(byt)
1056 def bytes_to_intlist(bs):
1059 if isinstance(bs[0], int): # Python 3
1062 return [ord(c) for c in bs]
1065 def intlist_to_bytes(xs):
1068 return struct_pack('%dB' % len(xs), *xs)
1071 # Cross-platform file locking
1072 if sys.platform == 'win32':
1073 import ctypes.wintypes
1076 class OVERLAPPED(ctypes.Structure):
1078 ('Internal', ctypes.wintypes.LPVOID),
1079 ('InternalHigh', ctypes.wintypes.LPVOID),
1080 ('Offset', ctypes.wintypes.DWORD),
1081 ('OffsetHigh', ctypes.wintypes.DWORD),
1082 ('hEvent', ctypes.wintypes.HANDLE),
1085 kernel32 = ctypes.windll.kernel32
1086 LockFileEx = kernel32.LockFileEx
1087 LockFileEx.argtypes = [
1088 ctypes.wintypes.HANDLE, # hFile
1089 ctypes.wintypes.DWORD, # dwFlags
1090 ctypes.wintypes.DWORD, # dwReserved
1091 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1092 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1093 ctypes.POINTER(OVERLAPPED) # Overlapped
1095 LockFileEx.restype = ctypes.wintypes.BOOL
1096 UnlockFileEx = kernel32.UnlockFileEx
1097 UnlockFileEx.argtypes = [
1098 ctypes.wintypes.HANDLE, # hFile
1099 ctypes.wintypes.DWORD, # dwReserved
1100 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1101 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1102 ctypes.POINTER(OVERLAPPED) # Overlapped
1104 UnlockFileEx.restype = ctypes.wintypes.BOOL
1105 whole_low = 0xffffffff
1106 whole_high = 0x7fffffff
1108 def _lock_file(f, exclusive):
1109 overlapped = OVERLAPPED()
1110 overlapped.Offset = 0
1111 overlapped.OffsetHigh = 0
1112 overlapped.hEvent = 0
1113 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1114 handle = msvcrt.get_osfhandle(f.fileno())
1115 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1116 whole_low, whole_high, f._lock_file_overlapped_p):
1117 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1119 def _unlock_file(f):
1120 assert f._lock_file_overlapped_p
1121 handle = msvcrt.get_osfhandle(f.fileno())
1122 if not UnlockFileEx(handle, 0,
1123 whole_low, whole_high, f._lock_file_overlapped_p):
1124 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1129 def _lock_file(f, exclusive):
1130 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1132 def _unlock_file(f):
1133 fcntl.flock(f, fcntl.LOCK_UN)
1136 class locked_file(object):
1137 def __init__(self, filename, mode, encoding=None):
1138 assert mode in ['r', 'a', 'w']
1139 self.f = io.open(filename, mode, encoding=encoding)
1142 def __enter__(self):
1143 exclusive = self.mode != 'r'
1145 _lock_file(self.f, exclusive)
1151 def __exit__(self, etype, value, traceback):
1153 _unlock_file(self.f)
1160 def write(self, *args):
1161 return self.f.write(*args)
1163 def read(self, *args):
1164 return self.f.read(*args)
1167 def get_filesystem_encoding():
1168 encoding = sys.getfilesystemencoding()
1169 return encoding if encoding is not None else 'utf-8'
1172 def shell_quote(args):
1174 encoding = get_filesystem_encoding()
1176 if isinstance(a, bytes):
1177 # We may get a filename encoded with 'encodeFilename'
1178 a = a.decode(encoding)
1179 quoted_args.append(pipes.quote(a))
1180 return ' '.join(quoted_args)
1183 def smuggle_url(url, data):
1184 """ Pass additional data in a URL for internal use. """
1186 sdata = compat_urllib_parse.urlencode(
1187 {'__youtubedl_smuggle': json.dumps(data)})
1188 return url + '#' + sdata
1191 def unsmuggle_url(smug_url, default=None):
1192 if '#__youtubedl_smuggle' not in smug_url:
1193 return smug_url, default
1194 url, _, sdata = smug_url.rpartition('#')
1195 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1196 data = json.loads(jsond)
1200 def format_bytes(bytes):
1203 if type(bytes) is str:
1204 bytes = float(bytes)
1208 exponent = int(math.log(bytes, 1024.0))
1209 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1210 converted = float(bytes) / float(1024 ** exponent)
1211 return '%.2f%s' % (converted, suffix)
1214 def parse_filesize(s):
1218 # The lower-case forms are of course incorrect and inofficial,
1219 # but we support those too
1257 units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
1259 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
1263 num_str = m.group('num').replace(',', '.')
1264 mult = _UNIT_TABLE[m.group('unit')]
1265 return int(float(num_str) * mult)
1268 def month_by_name(name):
1269 """ Return the number of a month by (locale-independently) English name """
1272 return ENGLISH_MONTH_NAMES.index(name) + 1
1277 def month_by_abbreviation(abbrev):
1278 """ Return the number of a month by (locale-independently) English
1282 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1287 def fix_xml_ampersands(xml_str):
1288 """Replace all the '&' by '&' in XML"""
1290 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1295 def setproctitle(title):
1296 assert isinstance(title, compat_str)
1298 libc = ctypes.cdll.LoadLibrary("libc.so.6")
1301 title_bytes = title.encode('utf-8')
1302 buf = ctypes.create_string_buffer(len(title_bytes))
1303 buf.value = title_bytes
1305 libc.prctl(15, buf, 0, 0, 0)
1306 except AttributeError:
1307 return # Strange libc, just skip this
1310 def remove_start(s, start):
1311 if s.startswith(start):
1312 return s[len(start):]
1316 def remove_end(s, end):
1318 return s[:-len(end)]
1322 def url_basename(url):
1323 path = compat_urlparse.urlparse(url).path
1324 return path.strip('/').split('/')[-1]
1327 class HEADRequest(compat_urllib_request.Request):
1328 def get_method(self):
1332 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1335 v = getattr(v, get_attr, None)
1338 return default if v is None else (int(v) * invscale // scale)
1341 def str_or_none(v, default=None):
1342 return default if v is None else compat_str(v)
1345 def str_to_int(int_str):
1346 """ A more relaxed version of int_or_none """
1349 int_str = re.sub(r'[,\.\+]', '', int_str)
1353 def float_or_none(v, scale=1, invscale=1, default=None):
1354 return default if v is None else (float(v) * invscale / scale)
1357 def parse_duration(s):
1358 if not isinstance(s, compat_basestring):
1366 (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
1367 (?P<only_hours>[0-9.]+)\s*(?:hours?)|
1369 \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
1372 (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
1373 (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
1375 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
1377 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
1382 if m.group('only_mins'):
1383 return float_or_none(m.group('only_mins'), invscale=60)
1384 if m.group('only_hours'):
1385 return float_or_none(m.group('only_hours'), invscale=60 * 60)
1387 res += int(m.group('secs'))
1388 if m.group('mins_reversed'):
1389 res += int(m.group('mins_reversed')) * 60
1391 res += int(m.group('mins')) * 60
1392 if m.group('hours'):
1393 res += int(m.group('hours')) * 60 * 60
1394 if m.group('hours_reversed'):
1395 res += int(m.group('hours_reversed')) * 60 * 60
1397 res += int(m.group('days')) * 24 * 60 * 60
1399 res += float(m.group('ms'))
1403 def prepend_extension(filename, ext, expected_real_ext=None):
1404 name, real_ext = os.path.splitext(filename)
1406 '{0}.{1}{2}'.format(name, ext, real_ext)
1407 if not expected_real_ext or real_ext[1:] == expected_real_ext
1408 else '{0}.{1}'.format(filename, ext))
1411 def replace_extension(filename, ext, expected_real_ext=None):
1412 name, real_ext = os.path.splitext(filename)
1413 return '{0}.{1}'.format(
1414 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1418 def check_executable(exe, args=[]):
1419 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1420 args can be a list of arguments for a short output (like -version) """
1422 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1428 def get_exe_version(exe, args=['--version'],
1429 version_re=None, unrecognized='present'):
1430 """ Returns the version of the specified executable,
1431 or False if the executable is not present """
1433 out, _ = subprocess.Popen(
1434 [encodeArgument(exe)] + args,
1435 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1438 if isinstance(out, bytes): # Python 2.x
1439 out = out.decode('ascii', 'ignore')
1440 return detect_exe_version(out, version_re, unrecognized)
1443 def detect_exe_version(output, version_re=None, unrecognized='present'):
1444 assert isinstance(output, compat_str)
1445 if version_re is None:
1446 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1447 m = re.search(version_re, output)
1454 class PagedList(object):
1456 # This is only useful for tests
1457 return len(self.getslice())
1460 class OnDemandPagedList(PagedList):
1461 def __init__(self, pagefunc, pagesize):
1462 self._pagefunc = pagefunc
1463 self._pagesize = pagesize
1465 def getslice(self, start=0, end=None):
1467 for pagenum in itertools.count(start // self._pagesize):
1468 firstid = pagenum * self._pagesize
1469 nextfirstid = pagenum * self._pagesize + self._pagesize
1470 if start >= nextfirstid:
1473 page_results = list(self._pagefunc(pagenum))
1476 start % self._pagesize
1477 if firstid <= start < nextfirstid
1481 ((end - 1) % self._pagesize) + 1
1482 if (end is not None and firstid <= end <= nextfirstid)
1485 if startv != 0 or endv is not None:
1486 page_results = page_results[startv:endv]
1487 res.extend(page_results)
1489 # A little optimization - if current page is not "full", ie. does
1490 # not contain page_size videos then we can assume that this page
1491 # is the last one - there are no more ids on further pages -
1492 # i.e. no need to query again.
1493 if len(page_results) + startv < self._pagesize:
1496 # If we got the whole page, but the next page is not interesting,
1497 # break out early as well
1498 if end == nextfirstid:
1503 class InAdvancePagedList(PagedList):
1504 def __init__(self, pagefunc, pagecount, pagesize):
1505 self._pagefunc = pagefunc
1506 self._pagecount = pagecount
1507 self._pagesize = pagesize
1509 def getslice(self, start=0, end=None):
1511 start_page = start // self._pagesize
1513 self._pagecount if end is None else (end // self._pagesize + 1))
1514 skip_elems = start - start_page * self._pagesize
1515 only_more = None if end is None else end - start
1516 for pagenum in range(start_page, end_page):
1517 page = list(self._pagefunc(pagenum))
1519 page = page[skip_elems:]
1521 if only_more is not None:
1522 if len(page) < only_more:
1523 only_more -= len(page)
1525 page = page[:only_more]
1532 def uppercase_escape(s):
1533 unicode_escape = codecs.getdecoder('unicode_escape')
1535 r'\\U[0-9a-fA-F]{8}',
1536 lambda m: unicode_escape(m.group(0))[0],
1540 def lowercase_escape(s):
1541 unicode_escape = codecs.getdecoder('unicode_escape')
1543 r'\\u[0-9a-fA-F]{4}',
1544 lambda m: unicode_escape(m.group(0))[0],
1548 def escape_rfc3986(s):
1549 """Escape non-ASCII characters as suggested by RFC 3986"""
1550 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1551 s = s.encode('utf-8')
1552 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1555 def escape_url(url):
1556 """Escape URL as suggested by RFC 3986"""
1557 url_parsed = compat_urllib_parse_urlparse(url)
1558 return url_parsed._replace(
1559 path=escape_rfc3986(url_parsed.path),
1560 params=escape_rfc3986(url_parsed.params),
1561 query=escape_rfc3986(url_parsed.query),
1562 fragment=escape_rfc3986(url_parsed.fragment)
1566 struct.pack('!I', 0)
1568 # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1569 def struct_pack(spec, *args):
1570 if isinstance(spec, compat_str):
1571 spec = spec.encode('ascii')
1572 return struct.pack(spec, *args)
1574 def struct_unpack(spec, *args):
1575 if isinstance(spec, compat_str):
1576 spec = spec.encode('ascii')
1577 return struct.unpack(spec, *args)
1579 struct_pack = struct.pack
1580 struct_unpack = struct.unpack
1583 def read_batch_urls(batch_fd):
1585 if not isinstance(url, compat_str):
1586 url = url.decode('utf-8', 'replace')
1587 BOM_UTF8 = '\xef\xbb\xbf'
1588 if url.startswith(BOM_UTF8):
1589 url = url[len(BOM_UTF8):]
1591 if url.startswith(('#', ';', ']')):
1595 with contextlib.closing(batch_fd) as fd:
1596 return [url for url in map(fixup, fd) if url]
1599 def urlencode_postdata(*args, **kargs):
1600 return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
1604 etree_iter = xml.etree.ElementTree.Element.iter
1605 except AttributeError: # Python <=2.6
1606 etree_iter = lambda n: n.findall('.//*')
1610 class TreeBuilder(xml.etree.ElementTree.TreeBuilder):
1611 def doctype(self, name, pubid, system):
1612 pass # Ignore doctypes
1614 parser = xml.etree.ElementTree.XMLParser(target=TreeBuilder())
1615 kwargs = {'parser': parser} if sys.version_info >= (2, 7) else {}
1616 tree = xml.etree.ElementTree.XML(s.encode('utf-8'), **kwargs)
1617 # Fix up XML parser in Python 2.x
1618 if sys.version_info < (3, 0):
1619 for n in etree_iter(tree):
1620 if n.text is not None:
1621 if not isinstance(n.text, compat_str):
1622 n.text = n.text.decode('utf-8')
1635 def parse_age_limit(s):
1638 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1639 return int(m.group('age')) if m else US_RATINGS.get(s, None)
1642 def strip_jsonp(code):
1644 r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1647 def js_to_json(code):
1650 if v in ('true', 'false', 'null'):
1652 if v.startswith('"'):
1654 if v.startswith("'"):
1656 v = re.sub(r"\\\\|\\'|\"", lambda m: {
1663 res = re.sub(r'''(?x)
1664 "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
1665 '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
1666 [a-zA-Z_][.a-zA-Z_0-9]*
1668 res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
1672 def qualities(quality_ids):
1673 """ Get a numeric quality value out of a list of possible values """
1676 return quality_ids.index(qid)
1682 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1685 def limit_length(s, length):
1686 """ Add ellipses to overly long strings """
1691 return s[:length - len(ELLIPSES)] + ELLIPSES
1695 def version_tuple(v):
1696 return tuple(int(e) for e in re.split(r'[-.]', v))
1699 def is_outdated_version(version, limit, assume_new=True):
1701 return not assume_new
1703 return version_tuple(version) < version_tuple(limit)
1705 return not assume_new
1708 def ytdl_is_updateable():
1709 """ Returns if youtube-dl can be updated with -U """
1710 from zipimport import zipimporter
1712 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
1715 def args_to_str(args):
1716 # Get a short string representation for a subprocess command
1717 return ' '.join(shlex_quote(a) for a in args)
1720 def mimetype2ext(mt):
1721 _, _, res = mt.rpartition('/')
1725 'x-mp4-fragmented': 'mp4',
1730 def urlhandle_detect_ext(url_handle):
1733 getheader = lambda h: url_handle.headers[h]
1734 except AttributeError: # Python < 3
1735 getheader = url_handle.info().getheader
1737 cd = getheader('Content-Disposition')
1739 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
1741 e = determine_ext(m.group('filename'), default_ext=None)
1745 return mimetype2ext(getheader('Content-Type'))
1748 def age_restricted(content_limit, age_limit):
1749 """ Returns True iff the content should be blocked """
1751 if age_limit is None: # No limit set
1753 if content_limit is None:
1754 return False # Content available for everyone
1755 return age_limit < content_limit
1758 def is_html(first_bytes):
1759 """ Detect whether a file contains HTML by examining its first bytes. """
1762 (b'\xef\xbb\xbf', 'utf-8'),
1763 (b'\x00\x00\xfe\xff', 'utf-32-be'),
1764 (b'\xff\xfe\x00\x00', 'utf-32-le'),
1765 (b'\xff\xfe', 'utf-16-le'),
1766 (b'\xfe\xff', 'utf-16-be'),
1768 for bom, enc in BOMS:
1769 if first_bytes.startswith(bom):
1770 s = first_bytes[len(bom):].decode(enc, 'replace')
1773 s = first_bytes.decode('utf-8', 'replace')
1775 return re.match(r'^\s*<', s)
1778 def determine_protocol(info_dict):
1779 protocol = info_dict.get('protocol')
1780 if protocol is not None:
1783 url = info_dict['url']
1784 if url.startswith('rtmp'):
1786 elif url.startswith('mms'):
1788 elif url.startswith('rtsp'):
1791 ext = determine_ext(url)
1797 return compat_urllib_parse_urlparse(url).scheme
1800 def render_table(header_row, data):
1801 """ Render a list of rows, each as a list of values """
1802 table = [header_row] + data
1803 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
1804 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
1805 return '\n'.join(format_str % tuple(row) for row in table)
1808 def _match_one(filter_part, dct):
1809 COMPARISON_OPERATORS = {
1817 operator_rex = re.compile(r'''(?x)\s*
1819 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1821 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
1822 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
1825 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
1826 m = operator_rex.search(filter_part)
1828 op = COMPARISON_OPERATORS[m.group('op')]
1829 if m.group('strval') is not None:
1830 if m.group('op') not in ('=', '!='):
1832 'Operator %s does not support string values!' % m.group('op'))
1833 comparison_value = m.group('strval')
1836 comparison_value = int(m.group('intval'))
1838 comparison_value = parse_filesize(m.group('intval'))
1839 if comparison_value is None:
1840 comparison_value = parse_filesize(m.group('intval') + 'B')
1841 if comparison_value is None:
1843 'Invalid integer value %r in filter part %r' % (
1844 m.group('intval'), filter_part))
1845 actual_value = dct.get(m.group('key'))
1846 if actual_value is None:
1847 return m.group('none_inclusive')
1848 return op(actual_value, comparison_value)
1851 '': lambda v: v is not None,
1852 '!': lambda v: v is None,
1854 operator_rex = re.compile(r'''(?x)\s*
1855 (?P<op>%s)\s*(?P<key>[a-z_]+)
1857 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
1858 m = operator_rex.search(filter_part)
1860 op = UNARY_OPERATORS[m.group('op')]
1861 actual_value = dct.get(m.group('key'))
1862 return op(actual_value)
1864 raise ValueError('Invalid filter part %r' % filter_part)
1867 def match_str(filter_str, dct):
1868 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
1871 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
1874 def match_filter_func(filter_str):
1875 def _match_func(info_dict):
1876 if match_str(filter_str, info_dict):
1879 video_title = info_dict.get('title', info_dict.get('id', 'video'))
1880 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
1884 def parse_dfxp_time_expr(time_expr):
1888 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
1890 return float(mobj.group('time_offset'))
1892 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:\.\d+)?)$', time_expr)
1894 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
1897 def srt_subtitles_timecode(seconds):
1898 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
1901 def dfxp2srt(dfxp_data):
1902 _x = functools.partial(xpath_with_ns, ns_map={
1903 'ttml': 'http://www.w3.org/ns/ttml',
1904 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
1907 def parse_node(node):
1908 str_or_empty = functools.partial(str_or_none, default='')
1910 out = str_or_empty(node.text)
1913 if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
1914 out += '\n' + str_or_empty(child.tail)
1915 elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
1916 out += str_or_empty(parse_node(child))
1918 out += str_or_empty(xml.etree.ElementTree.tostring(child))
1922 dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
1924 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
1927 raise ValueError('Invalid dfxp/TTML subtitle')
1929 for para, index in zip(paras, itertools.count(1)):
1930 begin_time = parse_dfxp_time_expr(para.attrib['begin'])
1931 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
1933 end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
1934 out.append('%d\n%s --> %s\n%s\n\n' % (
1936 srt_subtitles_timecode(begin_time),
1937 srt_subtitles_timecode(end_time),
1943 class ISO639Utils(object):
1944 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2133 def short2long(cls, code):
2134 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2135 return cls._lang_map.get(code[:2])
2138 def long2short(cls, code):
2139 """Convert language code from ISO 639-2/T to ISO 639-1"""
2140 for short_name, long_name in cls._lang_map.items():
2141 if long_name == code:
2145 class ISO3166Utils(object):
2146 # From http://data.okfn.org/data/core/country-list
2148 'AF': 'Afghanistan',
2149 'AX': 'Ã…land Islands',
2152 'AS': 'American Samoa',
2157 'AG': 'Antigua and Barbuda',
2174 'BO': 'Bolivia, Plurinational State of',
2175 'BQ': 'Bonaire, Sint Eustatius and Saba',
2176 'BA': 'Bosnia and Herzegovina',
2178 'BV': 'Bouvet Island',
2180 'IO': 'British Indian Ocean Territory',
2181 'BN': 'Brunei Darussalam',
2183 'BF': 'Burkina Faso',
2189 'KY': 'Cayman Islands',
2190 'CF': 'Central African Republic',
2194 'CX': 'Christmas Island',
2195 'CC': 'Cocos (Keeling) Islands',
2199 'CD': 'Congo, the Democratic Republic of the',
2200 'CK': 'Cook Islands',
2202 'CI': 'Côte d\'Ivoire',
2207 'CZ': 'Czech Republic',
2211 'DO': 'Dominican Republic',
2214 'SV': 'El Salvador',
2215 'GQ': 'Equatorial Guinea',
2219 'FK': 'Falkland Islands (Malvinas)',
2220 'FO': 'Faroe Islands',
2224 'GF': 'French Guiana',
2225 'PF': 'French Polynesia',
2226 'TF': 'French Southern Territories',
2241 'GW': 'Guinea-Bissau',
2244 'HM': 'Heard Island and McDonald Islands',
2245 'VA': 'Holy See (Vatican City State)',
2252 'IR': 'Iran, Islamic Republic of',
2255 'IM': 'Isle of Man',
2265 'KP': 'Korea, Democratic People\'s Republic of',
2266 'KR': 'Korea, Republic of',
2269 'LA': 'Lao People\'s Democratic Republic',
2275 'LI': 'Liechtenstein',
2279 'MK': 'Macedonia, the Former Yugoslav Republic of',
2286 'MH': 'Marshall Islands',
2292 'FM': 'Micronesia, Federated States of',
2293 'MD': 'Moldova, Republic of',
2304 'NL': 'Netherlands',
2305 'NC': 'New Caledonia',
2306 'NZ': 'New Zealand',
2311 'NF': 'Norfolk Island',
2312 'MP': 'Northern Mariana Islands',
2317 'PS': 'Palestine, State of',
2319 'PG': 'Papua New Guinea',
2322 'PH': 'Philippines',
2326 'PR': 'Puerto Rico',
2330 'RU': 'Russian Federation',
2332 'BL': 'Saint Barthélemy',
2333 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2334 'KN': 'Saint Kitts and Nevis',
2335 'LC': 'Saint Lucia',
2336 'MF': 'Saint Martin (French part)',
2337 'PM': 'Saint Pierre and Miquelon',
2338 'VC': 'Saint Vincent and the Grenadines',
2341 'ST': 'Sao Tome and Principe',
2342 'SA': 'Saudi Arabia',
2346 'SL': 'Sierra Leone',
2348 'SX': 'Sint Maarten (Dutch part)',
2351 'SB': 'Solomon Islands',
2353 'ZA': 'South Africa',
2354 'GS': 'South Georgia and the South Sandwich Islands',
2355 'SS': 'South Sudan',
2360 'SJ': 'Svalbard and Jan Mayen',
2363 'CH': 'Switzerland',
2364 'SY': 'Syrian Arab Republic',
2365 'TW': 'Taiwan, Province of China',
2367 'TZ': 'Tanzania, United Republic of',
2369 'TL': 'Timor-Leste',
2373 'TT': 'Trinidad and Tobago',
2376 'TM': 'Turkmenistan',
2377 'TC': 'Turks and Caicos Islands',
2381 'AE': 'United Arab Emirates',
2382 'GB': 'United Kingdom',
2383 'US': 'United States',
2384 'UM': 'United States Minor Outlying Islands',
2388 'VE': 'Venezuela, Bolivarian Republic of',
2390 'VG': 'Virgin Islands, British',
2391 'VI': 'Virgin Islands, U.S.',
2392 'WF': 'Wallis and Futuna',
2393 'EH': 'Western Sahara',
2400 def short2full(cls, code):
2401 """Convert an ISO 3166-2 country code to the corresponding full name"""
2402 return cls._country_map.get(code.upper())
2405 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2406 def __init__(self, proxies=None):
2407 # Set default handlers
2408 for type in ('http', 'https'):
2409 setattr(self, '%s_open' % type,
2410 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2411 meth(r, proxy, type))
2412 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2414 def proxy_open(self, req, proxy, type):
2415 req_proxy = req.headers.get('Ytdl-request-proxy')
2416 if req_proxy is not None:
2418 del req.headers['Ytdl-request-proxy']
2420 if proxy == '__noproxy__':
2421 return None # No Proxy
2422 return compat_urllib_request.ProxyHandler.proxy_open(
2423 self, req, proxy, type)