2 # -*- coding: utf-8 -*-
4 from __future__ import unicode_literals
32 import xml.etree.ElementTree
42 compat_socket_create_connection,
46 compat_urllib_parse_urlparse,
47 compat_urllib_request,
53 # This is not clearly defined otherwise
54 compiled_regex_type = type(re.compile(''))
57 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
58 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
59 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
60 'Accept-Encoding': 'gzip, deflate',
61 'Accept-Language': 'en-us,en;q=0.5',
67 ENGLISH_MONTH_NAMES = [
68 'January', 'February', 'March', 'April', 'May', 'June',
69 'July', 'August', 'September', 'October', 'November', 'December']
72 def preferredencoding():
73 """Get preferred encoding.
75 Returns the best encoding scheme for the system, based on
76 locale.getpreferredencoding() and some further tweaks.
79 pref = locale.getpreferredencoding()
87 def write_json_file(obj, fn):
88 """ Encode obj as JSON and write it to fn, atomically if possible """
90 fn = encodeFilename(fn)
91 if sys.version_info < (3, 0) and sys.platform != 'win32':
92 encoding = get_filesystem_encoding()
93 # os.path.basename returns a bytes object, but NamedTemporaryFile
94 # will fail if the filename contains non ascii characters unless we
95 # use a unicode object
96 path_basename = lambda f: os.path.basename(fn).decode(encoding)
97 # the same for os.path.dirname
98 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
100 path_basename = os.path.basename
101 path_dirname = os.path.dirname
105 'prefix': path_basename(fn) + '.',
106 'dir': path_dirname(fn),
110 # In Python 2.x, json.dump expects a bytestream.
111 # In Python 3.x, it writes to a character stream
112 if sys.version_info < (3, 0):
120 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
125 if sys.platform == 'win32':
126 # Need to remove existing file on Windows, else os.rename raises
127 # WindowsError or FileExistsError.
132 os.rename(tf.name, fn)
141 if sys.version_info >= (2, 7):
142 def find_xpath_attr(node, xpath, key, val=None):
143 """ Find the xpath xpath[@key=val] """
144 assert re.match(r'^[a-zA-Z-]+$', key)
146 assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
147 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
148 return node.find(expr)
150 def find_xpath_attr(node, xpath, key, val=None):
151 # Here comes the crazy part: In 2.6, if the xpath is a unicode,
152 # .//node does not match if a node is a direct child of . !
153 if isinstance(xpath, compat_str):
154 xpath = xpath.encode('ascii')
156 for f in node.findall(xpath):
157 if key not in f.attrib:
159 if val is None or f.attrib.get(key) == val:
163 # On python2.6 the xml.etree.ElementTree.Element methods don't support
164 # the namespace parameter
167 def xpath_with_ns(path, ns_map):
168 components = [c.split(':') for c in path.split('/')]
172 replaced.append(c[0])
175 replaced.append('{%s}%s' % (ns_map[ns], tag))
176 return '/'.join(replaced)
179 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
180 if sys.version_info < (2, 7): # Crazy 2.6
181 xpath = xpath.encode('ascii')
184 if n is None or n.text is None:
185 if default is not NO_DEFAULT:
188 name = xpath if name is None else name
189 raise ExtractorError('Could not find XML element %s' % name)
195 def get_element_by_id(id, html):
196 """Return the content of the tag with the specified ID in the passed HTML document"""
197 return get_element_by_attribute("id", id, html)
200 def get_element_by_attribute(attribute, value, html):
201 """Return the content of the tag with the specified attribute in the passed HTML document"""
203 m = re.search(r'''(?xs)
205 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
207 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
211 ''' % (re.escape(attribute), re.escape(value)), html)
215 res = m.group('content')
217 if res.startswith('"') or res.startswith("'"):
220 return unescapeHTML(res)
223 def clean_html(html):
224 """Clean an HTML snippet into a readable string"""
226 if html is None: # Convenience for sanitizing descriptions etc.
230 html = html.replace('\n', ' ')
231 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
232 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
234 html = re.sub('<.*?>', '', html)
235 # Replace html entities
236 html = unescapeHTML(html)
240 def sanitize_open(filename, open_mode):
241 """Try to open the given filename, and slightly tweak it if this fails.
243 Attempts to open the given filename. If this fails, it tries to change
244 the filename slightly, step by step, until it's either able to open it
245 or it fails and raises a final exception, like the standard open()
248 It returns the tuple (stream, definitive_file_name).
252 if sys.platform == 'win32':
254 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
255 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
256 stream = open(encodeFilename(filename), open_mode)
257 return (stream, filename)
258 except (IOError, OSError) as err:
259 if err.errno in (errno.EACCES,):
262 # In case of error, try to remove win32 forbidden chars
263 alt_filename = sanitize_path(filename)
264 if alt_filename == filename:
267 # An exception here should be caught in the caller
268 stream = open(encodeFilename(alt_filename), open_mode)
269 return (stream, alt_filename)
272 def timeconvert(timestr):
273 """Convert RFC 2822 defined time string into system timestamp"""
275 timetuple = email.utils.parsedate_tz(timestr)
276 if timetuple is not None:
277 timestamp = email.utils.mktime_tz(timetuple)
281 def sanitize_filename(s, restricted=False, is_id=False):
282 """Sanitizes a string so it could be used as part of a filename.
283 If restricted is set, use a stricter subset of allowed characters.
284 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
286 def replace_insane(char):
287 if char == '?' or ord(char) < 32 or ord(char) == 127:
290 return '' if restricted else '\''
292 return '_-' if restricted else ' -'
293 elif char in '\\/|*<>':
295 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
297 if restricted and ord(char) > 127:
302 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
303 result = ''.join(map(replace_insane, s))
305 while '__' in result:
306 result = result.replace('__', '_')
307 result = result.strip('_')
308 # Common case of "Foreign band name - English song title"
309 if restricted and result.startswith('-_'):
311 if result.startswith('-'):
312 result = '_' + result[len('-'):]
313 result = result.lstrip('.')
319 def sanitize_path(s):
320 """Sanitizes and normalizes path on Windows"""
321 if sys.platform != 'win32':
323 drive_or_unc, _ = os.path.splitdrive(s)
324 if sys.version_info < (2, 7) and not drive_or_unc:
325 drive_or_unc, _ = os.path.splitunc(s)
326 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
330 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
331 for path_part in norm_path]
333 sanitized_path.insert(0, drive_or_unc + os.path.sep)
334 return os.path.join(*sanitized_path)
337 def orderedSet(iterable):
338 """ Remove all duplicates from the input iterable """
346 def _htmlentity_transform(entity):
347 """Transforms an HTML entity to a character."""
348 # Known non-numeric HTML entity
349 if entity in compat_html_entities.name2codepoint:
350 return compat_chr(compat_html_entities.name2codepoint[entity])
352 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
354 numstr = mobj.group(1)
355 if numstr.startswith('x'):
357 numstr = '0%s' % numstr
360 return compat_chr(int(numstr, base))
362 # Unknown entity in name, return its literal representation
363 return ('&%s;' % entity)
369 assert type(s) == compat_str
372 r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
375 def get_subprocess_encoding():
376 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
377 # For subprocess calls, encode with locale encoding
378 # Refer to http://stackoverflow.com/a/9951851/35070
379 encoding = preferredencoding()
381 encoding = sys.getfilesystemencoding()
387 def encodeFilename(s, for_subprocess=False):
389 @param s The name of the file
392 assert type(s) == compat_str
394 # Python 3 has a Unicode API
395 if sys.version_info >= (3, 0):
398 # Pass '' directly to use Unicode APIs on Windows 2000 and up
399 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
400 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
401 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
404 return s.encode(get_subprocess_encoding(), 'ignore')
407 def decodeFilename(b, for_subprocess=False):
409 if sys.version_info >= (3, 0):
412 if not isinstance(b, bytes):
415 return b.decode(get_subprocess_encoding(), 'ignore')
418 def encodeArgument(s):
419 if not isinstance(s, compat_str):
420 # Legacy code that uses byte strings
421 # Uncomment the following line after fixing all post processors
422 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
423 s = s.decode('ascii')
424 return encodeFilename(s, True)
427 def decodeArgument(b):
428 return decodeFilename(b, True)
431 def decodeOption(optval):
434 if isinstance(optval, bytes):
435 optval = optval.decode(preferredencoding())
437 assert isinstance(optval, compat_str)
441 def formatSeconds(secs):
443 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
445 return '%d:%02d' % (secs // 60, secs % 60)
450 def make_HTTPS_handler(params, **kwargs):
451 opts_no_check_certificate = params.get('nocheckcertificate', False)
452 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
453 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
454 if opts_no_check_certificate:
455 context.check_hostname = False
456 context.verify_mode = ssl.CERT_NONE
458 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
461 # (create_default_context present but HTTPSHandler has no context=)
464 if sys.version_info < (3, 2):
465 return YoutubeDLHTTPSHandler(params, **kwargs)
467 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
468 context.verify_mode = (ssl.CERT_NONE
469 if opts_no_check_certificate
470 else ssl.CERT_REQUIRED)
471 context.set_default_verify_paths()
472 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
475 def bug_reports_message():
476 if ytdl_is_updateable():
477 update_cmd = 'type youtube-dl -U to update'
479 update_cmd = 'see https://yt-dl.org/update on how to update'
480 msg = '; please report this issue on https://yt-dl.org/bug .'
481 msg += ' Make sure you are using the latest version; %s.' % update_cmd
482 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
486 class ExtractorError(Exception):
487 """Error during info extraction."""
489 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
490 """ tb, if given, is the original traceback (so that it can be printed out).
491 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
494 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
496 if video_id is not None:
497 msg = video_id + ': ' + msg
499 msg += ' (caused by %r)' % cause
501 msg += bug_reports_message()
502 super(ExtractorError, self).__init__(msg)
505 self.exc_info = sys.exc_info() # preserve original exception
507 self.video_id = video_id
509 def format_traceback(self):
510 if self.traceback is None:
512 return ''.join(traceback.format_tb(self.traceback))
515 class UnsupportedError(ExtractorError):
516 def __init__(self, url):
517 super(UnsupportedError, self).__init__(
518 'Unsupported URL: %s' % url, expected=True)
522 class RegexNotFoundError(ExtractorError):
523 """Error when a regex didn't match"""
527 class DownloadError(Exception):
528 """Download Error exception.
530 This exception may be thrown by FileDownloader objects if they are not
531 configured to continue on errors. They will contain the appropriate
535 def __init__(self, msg, exc_info=None):
536 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
537 super(DownloadError, self).__init__(msg)
538 self.exc_info = exc_info
541 class SameFileError(Exception):
542 """Same File exception.
544 This exception will be thrown by FileDownloader objects if they detect
545 multiple files would have to be downloaded to the same file on disk.
550 class PostProcessingError(Exception):
551 """Post Processing exception.
553 This exception may be raised by PostProcessor's .run() method to
554 indicate an error in the postprocessing task.
557 def __init__(self, msg):
561 class MaxDownloadsReached(Exception):
562 """ --max-downloads limit has been reached. """
566 class UnavailableVideoError(Exception):
567 """Unavailable Format exception.
569 This exception will be thrown when a video is requested
570 in a format that is not available for that video.
575 class ContentTooShortError(Exception):
576 """Content Too Short exception.
578 This exception may be raised by FileDownloader objects when a file they
579 download is too small for what the server announced first, indicating
580 the connection was probably interrupted.
583 def __init__(self, downloaded, expected):
585 self.downloaded = downloaded
586 self.expected = expected
589 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
590 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
591 # expected HTTP responses to meet HTTP/1.0 or later (see also
592 # https://github.com/rg3/youtube-dl/issues/6727)
593 if sys.version_info < (3, 0):
594 kwargs['strict'] = True
595 hc = http_class(*args, **kwargs)
596 source_address = ydl_handler._params.get('source_address')
597 if source_address is not None:
598 sa = (source_address, 0)
599 if hasattr(hc, 'source_address'): # Python 2.7+
600 hc.source_address = sa
602 def _hc_connect(self, *args, **kwargs):
603 sock = compat_socket_create_connection(
604 (self.host, self.port), self.timeout, sa)
606 self.sock = ssl.wrap_socket(
607 sock, self.key_file, self.cert_file,
608 ssl_version=ssl.PROTOCOL_TLSv1)
611 hc.connect = functools.partial(_hc_connect, hc)
616 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
617 """Handler for HTTP requests and responses.
619 This class, when installed with an OpenerDirector, automatically adds
620 the standard headers to every HTTP request and handles gzipped and
621 deflated responses from web servers. If compression is to be avoided in
622 a particular request, the original request in the program code only has
623 to include the HTTP header "Youtubedl-No-Compression", which will be
624 removed before making the real request.
626 Part of this code was copied from:
628 http://techknack.net/python-urllib2-handlers/
630 Andrew Rowls, the author of that code, agreed to release it to the
634 def __init__(self, params, *args, **kwargs):
635 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
636 self._params = params
638 def http_open(self, req):
639 return self.do_open(functools.partial(
640 _create_http_connection, self, compat_http_client.HTTPConnection, False),
646 return zlib.decompress(data, -zlib.MAX_WBITS)
648 return zlib.decompress(data)
651 def addinfourl_wrapper(stream, headers, url, code):
652 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
653 return compat_urllib_request.addinfourl(stream, headers, url, code)
654 ret = compat_urllib_request.addinfourl(stream, headers, url)
658 def http_request(self, req):
659 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
660 # always respected by websites, some tend to give out URLs with non percent-encoded
661 # non-ASCII characters (see telemb.py, ard.py [#3412])
662 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
663 # To work around aforementioned issue we will replace request's original URL with
664 # percent-encoded one
665 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
666 # the code of this workaround has been moved here from YoutubeDL.urlopen()
667 url = req.get_full_url()
668 url_escaped = escape_url(url)
670 # Substitute URL if any change after escaping
671 if url != url_escaped:
672 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
674 url_escaped, data=req.data, headers=req.headers,
675 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
676 new_req.timeout = req.timeout
679 for h, v in std_headers.items():
680 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
681 # The dict keys are capitalized because of this bug by urllib
682 if h.capitalize() not in req.headers:
684 if 'Youtubedl-no-compression' in req.headers:
685 if 'Accept-encoding' in req.headers:
686 del req.headers['Accept-encoding']
687 del req.headers['Youtubedl-no-compression']
689 if sys.version_info < (2, 7) and '#' in req.get_full_url():
690 # Python 2.6 is brain-dead when it comes to fragments
691 req._Request__original = req._Request__original.partition('#')[0]
692 req._Request__r_type = req._Request__r_type.partition('#')[0]
696 def http_response(self, req, resp):
699 if resp.headers.get('Content-encoding', '') == 'gzip':
700 content = resp.read()
701 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
703 uncompressed = io.BytesIO(gz.read())
704 except IOError as original_ioerror:
705 # There may be junk add the end of the file
706 # See http://stackoverflow.com/q/4928560/35070 for details
707 for i in range(1, 1024):
709 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
710 uncompressed = io.BytesIO(gz.read())
715 raise original_ioerror
716 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
717 resp.msg = old_resp.msg
719 if resp.headers.get('Content-encoding', '') == 'deflate':
720 gz = io.BytesIO(self.deflate(resp.read()))
721 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
722 resp.msg = old_resp.msg
723 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986
724 if 300 <= resp.code < 400:
725 location = resp.headers.get('Location')
727 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
728 if sys.version_info >= (3, 0):
729 location = location.encode('iso-8859-1').decode('utf-8')
730 location_escaped = escape_url(location)
731 if location != location_escaped:
732 del resp.headers['Location']
733 resp.headers['Location'] = location_escaped
736 https_request = http_request
737 https_response = http_response
740 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
741 def __init__(self, params, https_conn_class=None, *args, **kwargs):
742 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
743 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
744 self._params = params
746 def https_open(self, req):
748 if hasattr(self, '_context'): # python > 2.6
749 kwargs['context'] = self._context
750 if hasattr(self, '_check_hostname'): # python 3.x
751 kwargs['check_hostname'] = self._check_hostname
752 return self.do_open(functools.partial(
753 _create_http_connection, self, self._https_conn_class, True),
757 def parse_iso8601(date_str, delimiter='T', timezone=None):
758 """ Return a UNIX timestamp from the given date """
765 r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
768 timezone = datetime.timedelta()
770 date_str = date_str[:-len(m.group(0))]
771 if not m.group('sign'):
772 timezone = datetime.timedelta()
774 sign = 1 if m.group('sign') == '+' else -1
775 timezone = datetime.timedelta(
776 hours=sign * int(m.group('hours')),
777 minutes=sign * int(m.group('minutes')))
778 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
779 dt = datetime.datetime.strptime(date_str, date_format) - timezone
780 return calendar.timegm(dt.timetuple())
783 def unified_strdate(date_str, day_first=True):
784 """Return a string with the date in the format YYYYMMDD"""
790 date_str = date_str.replace(',', ' ')
791 # %z (UTC offset) is only supported in python>=3.2
792 if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
793 date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
794 # Remove AM/PM + timezone
795 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
797 format_expressions = [
802 '%b %dst %Y %I:%M%p',
803 '%b %dnd %Y %I:%M%p',
804 '%b %dth %Y %I:%M%p',
810 '%Y-%m-%d %H:%M:%S.%f',
813 '%Y-%m-%dT%H:%M:%SZ',
814 '%Y-%m-%dT%H:%M:%S.%fZ',
815 '%Y-%m-%dT%H:%M:%S.%f0Z',
817 '%Y-%m-%dT%H:%M:%S.%f',
821 format_expressions.extend([
829 format_expressions.extend([
836 for expression in format_expressions:
838 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
841 if upload_date is None:
842 timetuple = email.utils.parsedate_tz(date_str)
844 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
848 def determine_ext(url, default_ext='unknown_video'):
851 guess = url.partition('?')[0].rpartition('.')[2]
852 if re.match(r'^[A-Za-z0-9]+$', guess):
858 def subtitles_filename(filename, sub_lang, sub_format):
859 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
862 def date_from_str(date_str):
864 Return a datetime object from a string in the format YYYYMMDD or
865 (now|today)[+-][0-9](day|week|month|year)(s)?"""
866 today = datetime.date.today()
867 if date_str in ('now', 'today'):
869 if date_str == 'yesterday':
870 return today - datetime.timedelta(days=1)
871 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
872 if match is not None:
873 sign = match.group('sign')
874 time = int(match.group('time'))
877 unit = match.group('unit')
878 # A bad aproximation?
886 delta = datetime.timedelta(**{unit: time})
888 return datetime.datetime.strptime(date_str, "%Y%m%d").date()
891 def hyphenate_date(date_str):
893 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
894 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
895 if match is not None:
896 return '-'.join(match.groups())
901 class DateRange(object):
902 """Represents a time interval between two dates"""
904 def __init__(self, start=None, end=None):
905 """start and end must be strings in the format accepted by date"""
906 if start is not None:
907 self.start = date_from_str(start)
909 self.start = datetime.datetime.min.date()
911 self.end = date_from_str(end)
913 self.end = datetime.datetime.max.date()
914 if self.start > self.end:
915 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
919 """Returns a range that only contains the given day"""
922 def __contains__(self, date):
923 """Check if the date is in the range"""
924 if not isinstance(date, datetime.date):
925 date = date_from_str(date)
926 return self.start <= date <= self.end
929 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
933 """ Returns the platform name as a compat_str """
934 res = platform.platform()
935 if isinstance(res, bytes):
936 res = res.decode(preferredencoding())
938 assert isinstance(res, compat_str)
942 def _windows_write_string(s, out):
943 """ Returns True if the string was written using special methods,
944 False if it has yet to be written out."""
945 # Adapted from http://stackoverflow.com/a/3259271/35070
948 import ctypes.wintypes
956 fileno = out.fileno()
957 except AttributeError:
958 # If the output stream doesn't have a fileno, it's virtual
960 except io.UnsupportedOperation:
961 # Some strange Windows pseudo files?
963 if fileno not in WIN_OUTPUT_IDS:
966 GetStdHandle = ctypes.WINFUNCTYPE(
967 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
968 (b"GetStdHandle", ctypes.windll.kernel32))
969 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
971 WriteConsoleW = ctypes.WINFUNCTYPE(
972 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
973 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
974 ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
975 written = ctypes.wintypes.DWORD(0)
977 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
978 FILE_TYPE_CHAR = 0x0002
979 FILE_TYPE_REMOTE = 0x8000
980 GetConsoleMode = ctypes.WINFUNCTYPE(
981 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
982 ctypes.POINTER(ctypes.wintypes.DWORD))(
983 (b"GetConsoleMode", ctypes.windll.kernel32))
984 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
986 def not_a_console(handle):
987 if handle == INVALID_HANDLE_VALUE or handle is None:
989 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
990 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
995 def next_nonbmp_pos(s):
997 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
998 except StopIteration:
1002 count = min(next_nonbmp_pos(s), 1024)
1004 ret = WriteConsoleW(
1005 h, s, count if count else 2, ctypes.byref(written), None)
1007 raise OSError('Failed to write string')
1008 if not count: # We just wrote a non-BMP character
1009 assert written.value == 2
1012 assert written.value > 0
1013 s = s[written.value:]
1017 def write_string(s, out=None, encoding=None):
1020 assert type(s) == compat_str
1022 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1023 if _windows_write_string(s, out):
1026 if ('b' in getattr(out, 'mode', '') or
1027 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1028 byt = s.encode(encoding or preferredencoding(), 'ignore')
1030 elif hasattr(out, 'buffer'):
1031 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1032 byt = s.encode(enc, 'ignore')
1033 out.buffer.write(byt)
1039 def bytes_to_intlist(bs):
1042 if isinstance(bs[0], int): # Python 3
1045 return [ord(c) for c in bs]
1048 def intlist_to_bytes(xs):
1051 return struct_pack('%dB' % len(xs), *xs)
1054 # Cross-platform file locking
1055 if sys.platform == 'win32':
1056 import ctypes.wintypes
1059 class OVERLAPPED(ctypes.Structure):
1061 ('Internal', ctypes.wintypes.LPVOID),
1062 ('InternalHigh', ctypes.wintypes.LPVOID),
1063 ('Offset', ctypes.wintypes.DWORD),
1064 ('OffsetHigh', ctypes.wintypes.DWORD),
1065 ('hEvent', ctypes.wintypes.HANDLE),
1068 kernel32 = ctypes.windll.kernel32
1069 LockFileEx = kernel32.LockFileEx
1070 LockFileEx.argtypes = [
1071 ctypes.wintypes.HANDLE, # hFile
1072 ctypes.wintypes.DWORD, # dwFlags
1073 ctypes.wintypes.DWORD, # dwReserved
1074 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1075 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1076 ctypes.POINTER(OVERLAPPED) # Overlapped
1078 LockFileEx.restype = ctypes.wintypes.BOOL
1079 UnlockFileEx = kernel32.UnlockFileEx
1080 UnlockFileEx.argtypes = [
1081 ctypes.wintypes.HANDLE, # hFile
1082 ctypes.wintypes.DWORD, # dwReserved
1083 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1084 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1085 ctypes.POINTER(OVERLAPPED) # Overlapped
1087 UnlockFileEx.restype = ctypes.wintypes.BOOL
1088 whole_low = 0xffffffff
1089 whole_high = 0x7fffffff
1091 def _lock_file(f, exclusive):
1092 overlapped = OVERLAPPED()
1093 overlapped.Offset = 0
1094 overlapped.OffsetHigh = 0
1095 overlapped.hEvent = 0
1096 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1097 handle = msvcrt.get_osfhandle(f.fileno())
1098 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1099 whole_low, whole_high, f._lock_file_overlapped_p):
1100 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1102 def _unlock_file(f):
1103 assert f._lock_file_overlapped_p
1104 handle = msvcrt.get_osfhandle(f.fileno())
1105 if not UnlockFileEx(handle, 0,
1106 whole_low, whole_high, f._lock_file_overlapped_p):
1107 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1112 def _lock_file(f, exclusive):
1113 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1115 def _unlock_file(f):
1116 fcntl.flock(f, fcntl.LOCK_UN)
1119 class locked_file(object):
1120 def __init__(self, filename, mode, encoding=None):
1121 assert mode in ['r', 'a', 'w']
1122 self.f = io.open(filename, mode, encoding=encoding)
1125 def __enter__(self):
1126 exclusive = self.mode != 'r'
1128 _lock_file(self.f, exclusive)
1134 def __exit__(self, etype, value, traceback):
1136 _unlock_file(self.f)
1143 def write(self, *args):
1144 return self.f.write(*args)
1146 def read(self, *args):
1147 return self.f.read(*args)
1150 def get_filesystem_encoding():
1151 encoding = sys.getfilesystemencoding()
1152 return encoding if encoding is not None else 'utf-8'
1155 def shell_quote(args):
1157 encoding = get_filesystem_encoding()
1159 if isinstance(a, bytes):
1160 # We may get a filename encoded with 'encodeFilename'
1161 a = a.decode(encoding)
1162 quoted_args.append(pipes.quote(a))
1163 return ' '.join(quoted_args)
1166 def smuggle_url(url, data):
1167 """ Pass additional data in a URL for internal use. """
1169 sdata = compat_urllib_parse.urlencode(
1170 {'__youtubedl_smuggle': json.dumps(data)})
1171 return url + '#' + sdata
1174 def unsmuggle_url(smug_url, default=None):
1175 if '#__youtubedl_smuggle' not in smug_url:
1176 return smug_url, default
1177 url, _, sdata = smug_url.rpartition('#')
1178 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1179 data = json.loads(jsond)
1183 def format_bytes(bytes):
1186 if type(bytes) is str:
1187 bytes = float(bytes)
1191 exponent = int(math.log(bytes, 1024.0))
1192 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1193 converted = float(bytes) / float(1024 ** exponent)
1194 return '%.2f%s' % (converted, suffix)
1197 def parse_filesize(s):
1201 # The lower-case forms are of course incorrect and inofficial,
1202 # but we support those too
1240 units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
1242 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
1246 num_str = m.group('num').replace(',', '.')
1247 mult = _UNIT_TABLE[m.group('unit')]
1248 return int(float(num_str) * mult)
1251 def month_by_name(name):
1252 """ Return the number of a month by (locale-independently) English name """
1255 return ENGLISH_MONTH_NAMES.index(name) + 1
1260 def month_by_abbreviation(abbrev):
1261 """ Return the number of a month by (locale-independently) English
1265 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1270 def fix_xml_ampersands(xml_str):
1271 """Replace all the '&' by '&' in XML"""
1273 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1278 def setproctitle(title):
1279 assert isinstance(title, compat_str)
1281 libc = ctypes.cdll.LoadLibrary("libc.so.6")
1284 title_bytes = title.encode('utf-8')
1285 buf = ctypes.create_string_buffer(len(title_bytes))
1286 buf.value = title_bytes
1288 libc.prctl(15, buf, 0, 0, 0)
1289 except AttributeError:
1290 return # Strange libc, just skip this
1293 def remove_start(s, start):
1294 if s.startswith(start):
1295 return s[len(start):]
1299 def remove_end(s, end):
1301 return s[:-len(end)]
1305 def url_basename(url):
1306 path = compat_urlparse.urlparse(url).path
1307 return path.strip('/').split('/')[-1]
1310 class HEADRequest(compat_urllib_request.Request):
1311 def get_method(self):
1315 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1318 v = getattr(v, get_attr, None)
1321 return default if v is None else (int(v) * invscale // scale)
1324 def str_or_none(v, default=None):
1325 return default if v is None else compat_str(v)
1328 def str_to_int(int_str):
1329 """ A more relaxed version of int_or_none """
1332 int_str = re.sub(r'[,\.\+]', '', int_str)
1336 def float_or_none(v, scale=1, invscale=1, default=None):
1337 return default if v is None else (float(v) * invscale / scale)
1340 def parse_duration(s):
1341 if not isinstance(s, compat_basestring):
1349 (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
1350 (?P<only_hours>[0-9.]+)\s*(?:hours?)|
1352 \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
1355 (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
1356 (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
1358 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
1360 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
1365 if m.group('only_mins'):
1366 return float_or_none(m.group('only_mins'), invscale=60)
1367 if m.group('only_hours'):
1368 return float_or_none(m.group('only_hours'), invscale=60 * 60)
1370 res += int(m.group('secs'))
1371 if m.group('mins_reversed'):
1372 res += int(m.group('mins_reversed')) * 60
1374 res += int(m.group('mins')) * 60
1375 if m.group('hours'):
1376 res += int(m.group('hours')) * 60 * 60
1377 if m.group('hours_reversed'):
1378 res += int(m.group('hours_reversed')) * 60 * 60
1380 res += int(m.group('days')) * 24 * 60 * 60
1382 res += float(m.group('ms'))
1386 def prepend_extension(filename, ext, expected_real_ext=None):
1387 name, real_ext = os.path.splitext(filename)
1389 '{0}.{1}{2}'.format(name, ext, real_ext)
1390 if not expected_real_ext or real_ext[1:] == expected_real_ext
1391 else '{0}.{1}'.format(filename, ext))
1394 def replace_extension(filename, ext, expected_real_ext=None):
1395 name, real_ext = os.path.splitext(filename)
1396 return '{0}.{1}'.format(
1397 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1401 def check_executable(exe, args=[]):
1402 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1403 args can be a list of arguments for a short output (like -version) """
1405 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1411 def get_exe_version(exe, args=['--version'],
1412 version_re=None, unrecognized='present'):
1413 """ Returns the version of the specified executable,
1414 or False if the executable is not present """
1416 out, _ = subprocess.Popen(
1417 [encodeArgument(exe)] + args,
1418 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1421 if isinstance(out, bytes): # Python 2.x
1422 out = out.decode('ascii', 'ignore')
1423 return detect_exe_version(out, version_re, unrecognized)
1426 def detect_exe_version(output, version_re=None, unrecognized='present'):
1427 assert isinstance(output, compat_str)
1428 if version_re is None:
1429 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1430 m = re.search(version_re, output)
1437 class PagedList(object):
1439 # This is only useful for tests
1440 return len(self.getslice())
1443 class OnDemandPagedList(PagedList):
1444 def __init__(self, pagefunc, pagesize):
1445 self._pagefunc = pagefunc
1446 self._pagesize = pagesize
1448 def getslice(self, start=0, end=None):
1450 for pagenum in itertools.count(start // self._pagesize):
1451 firstid = pagenum * self._pagesize
1452 nextfirstid = pagenum * self._pagesize + self._pagesize
1453 if start >= nextfirstid:
1456 page_results = list(self._pagefunc(pagenum))
1459 start % self._pagesize
1460 if firstid <= start < nextfirstid
1464 ((end - 1) % self._pagesize) + 1
1465 if (end is not None and firstid <= end <= nextfirstid)
1468 if startv != 0 or endv is not None:
1469 page_results = page_results[startv:endv]
1470 res.extend(page_results)
1472 # A little optimization - if current page is not "full", ie. does
1473 # not contain page_size videos then we can assume that this page
1474 # is the last one - there are no more ids on further pages -
1475 # i.e. no need to query again.
1476 if len(page_results) + startv < self._pagesize:
1479 # If we got the whole page, but the next page is not interesting,
1480 # break out early as well
1481 if end == nextfirstid:
1486 class InAdvancePagedList(PagedList):
1487 def __init__(self, pagefunc, pagecount, pagesize):
1488 self._pagefunc = pagefunc
1489 self._pagecount = pagecount
1490 self._pagesize = pagesize
1492 def getslice(self, start=0, end=None):
1494 start_page = start // self._pagesize
1496 self._pagecount if end is None else (end // self._pagesize + 1))
1497 skip_elems = start - start_page * self._pagesize
1498 only_more = None if end is None else end - start
1499 for pagenum in range(start_page, end_page):
1500 page = list(self._pagefunc(pagenum))
1502 page = page[skip_elems:]
1504 if only_more is not None:
1505 if len(page) < only_more:
1506 only_more -= len(page)
1508 page = page[:only_more]
1515 def uppercase_escape(s):
1516 unicode_escape = codecs.getdecoder('unicode_escape')
1518 r'\\U[0-9a-fA-F]{8}',
1519 lambda m: unicode_escape(m.group(0))[0],
1523 def lowercase_escape(s):
1524 unicode_escape = codecs.getdecoder('unicode_escape')
1526 r'\\u[0-9a-fA-F]{4}',
1527 lambda m: unicode_escape(m.group(0))[0],
1531 def escape_rfc3986(s):
1532 """Escape non-ASCII characters as suggested by RFC 3986"""
1533 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1534 s = s.encode('utf-8')
1535 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1538 def escape_url(url):
1539 """Escape URL as suggested by RFC 3986"""
1540 url_parsed = compat_urllib_parse_urlparse(url)
1541 return url_parsed._replace(
1542 path=escape_rfc3986(url_parsed.path),
1543 params=escape_rfc3986(url_parsed.params),
1544 query=escape_rfc3986(url_parsed.query),
1545 fragment=escape_rfc3986(url_parsed.fragment)
1549 struct.pack('!I', 0)
1551 # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1552 def struct_pack(spec, *args):
1553 if isinstance(spec, compat_str):
1554 spec = spec.encode('ascii')
1555 return struct.pack(spec, *args)
1557 def struct_unpack(spec, *args):
1558 if isinstance(spec, compat_str):
1559 spec = spec.encode('ascii')
1560 return struct.unpack(spec, *args)
1562 struct_pack = struct.pack
1563 struct_unpack = struct.unpack
1566 def read_batch_urls(batch_fd):
1568 if not isinstance(url, compat_str):
1569 url = url.decode('utf-8', 'replace')
1570 BOM_UTF8 = '\xef\xbb\xbf'
1571 if url.startswith(BOM_UTF8):
1572 url = url[len(BOM_UTF8):]
1574 if url.startswith(('#', ';', ']')):
1578 with contextlib.closing(batch_fd) as fd:
1579 return [url for url in map(fixup, fd) if url]
1582 def urlencode_postdata(*args, **kargs):
1583 return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
1587 etree_iter = xml.etree.ElementTree.Element.iter
1588 except AttributeError: # Python <=2.6
1589 etree_iter = lambda n: n.findall('.//*')
1593 class TreeBuilder(xml.etree.ElementTree.TreeBuilder):
1594 def doctype(self, name, pubid, system):
1595 pass # Ignore doctypes
1597 parser = xml.etree.ElementTree.XMLParser(target=TreeBuilder())
1598 kwargs = {'parser': parser} if sys.version_info >= (2, 7) else {}
1599 tree = xml.etree.ElementTree.XML(s.encode('utf-8'), **kwargs)
1600 # Fix up XML parser in Python 2.x
1601 if sys.version_info < (3, 0):
1602 for n in etree_iter(tree):
1603 if n.text is not None:
1604 if not isinstance(n.text, compat_str):
1605 n.text = n.text.decode('utf-8')
1618 def parse_age_limit(s):
1621 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1622 return int(m.group('age')) if m else US_RATINGS.get(s, None)
1625 def strip_jsonp(code):
1627 r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1630 def js_to_json(code):
1633 if v in ('true', 'false', 'null'):
1635 if v.startswith('"'):
1637 if v.startswith("'"):
1639 v = re.sub(r"\\\\|\\'|\"", lambda m: {
1646 res = re.sub(r'''(?x)
1647 "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
1648 '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
1649 [a-zA-Z_][.a-zA-Z_0-9]*
1651 res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
1655 def qualities(quality_ids):
1656 """ Get a numeric quality value out of a list of possible values """
1659 return quality_ids.index(qid)
1665 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1668 def limit_length(s, length):
1669 """ Add ellipses to overly long strings """
1674 return s[:length - len(ELLIPSES)] + ELLIPSES
1678 def version_tuple(v):
1679 return tuple(int(e) for e in re.split(r'[-.]', v))
1682 def is_outdated_version(version, limit, assume_new=True):
1684 return not assume_new
1686 return version_tuple(version) < version_tuple(limit)
1688 return not assume_new
1691 def ytdl_is_updateable():
1692 """ Returns if youtube-dl can be updated with -U """
1693 from zipimport import zipimporter
1695 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
1698 def args_to_str(args):
1699 # Get a short string representation for a subprocess command
1700 return ' '.join(shlex_quote(a) for a in args)
1703 def mimetype2ext(mt):
1704 _, _, res = mt.rpartition('/')
1708 'x-mp4-fragmented': 'mp4',
1713 def urlhandle_detect_ext(url_handle):
1716 getheader = lambda h: url_handle.headers[h]
1717 except AttributeError: # Python < 3
1718 getheader = url_handle.info().getheader
1720 cd = getheader('Content-Disposition')
1722 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
1724 e = determine_ext(m.group('filename'), default_ext=None)
1728 return mimetype2ext(getheader('Content-Type'))
1731 def age_restricted(content_limit, age_limit):
1732 """ Returns True iff the content should be blocked """
1734 if age_limit is None: # No limit set
1736 if content_limit is None:
1737 return False # Content available for everyone
1738 return age_limit < content_limit
1741 def is_html(first_bytes):
1742 """ Detect whether a file contains HTML by examining its first bytes. """
1745 (b'\xef\xbb\xbf', 'utf-8'),
1746 (b'\x00\x00\xfe\xff', 'utf-32-be'),
1747 (b'\xff\xfe\x00\x00', 'utf-32-le'),
1748 (b'\xff\xfe', 'utf-16-le'),
1749 (b'\xfe\xff', 'utf-16-be'),
1751 for bom, enc in BOMS:
1752 if first_bytes.startswith(bom):
1753 s = first_bytes[len(bom):].decode(enc, 'replace')
1756 s = first_bytes.decode('utf-8', 'replace')
1758 return re.match(r'^\s*<', s)
1761 def determine_protocol(info_dict):
1762 protocol = info_dict.get('protocol')
1763 if protocol is not None:
1766 url = info_dict['url']
1767 if url.startswith('rtmp'):
1769 elif url.startswith('mms'):
1771 elif url.startswith('rtsp'):
1774 ext = determine_ext(url)
1780 return compat_urllib_parse_urlparse(url).scheme
1783 def render_table(header_row, data):
1784 """ Render a list of rows, each as a list of values """
1785 table = [header_row] + data
1786 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
1787 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
1788 return '\n'.join(format_str % tuple(row) for row in table)
1791 def _match_one(filter_part, dct):
1792 COMPARISON_OPERATORS = {
1800 operator_rex = re.compile(r'''(?x)\s*
1802 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1804 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
1805 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
1808 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
1809 m = operator_rex.search(filter_part)
1811 op = COMPARISON_OPERATORS[m.group('op')]
1812 if m.group('strval') is not None:
1813 if m.group('op') not in ('=', '!='):
1815 'Operator %s does not support string values!' % m.group('op'))
1816 comparison_value = m.group('strval')
1819 comparison_value = int(m.group('intval'))
1821 comparison_value = parse_filesize(m.group('intval'))
1822 if comparison_value is None:
1823 comparison_value = parse_filesize(m.group('intval') + 'B')
1824 if comparison_value is None:
1826 'Invalid integer value %r in filter part %r' % (
1827 m.group('intval'), filter_part))
1828 actual_value = dct.get(m.group('key'))
1829 if actual_value is None:
1830 return m.group('none_inclusive')
1831 return op(actual_value, comparison_value)
1834 '': lambda v: v is not None,
1835 '!': lambda v: v is None,
1837 operator_rex = re.compile(r'''(?x)\s*
1838 (?P<op>%s)\s*(?P<key>[a-z_]+)
1840 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
1841 m = operator_rex.search(filter_part)
1843 op = UNARY_OPERATORS[m.group('op')]
1844 actual_value = dct.get(m.group('key'))
1845 return op(actual_value)
1847 raise ValueError('Invalid filter part %r' % filter_part)
1850 def match_str(filter_str, dct):
1851 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
1854 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
1857 def match_filter_func(filter_str):
1858 def _match_func(info_dict):
1859 if match_str(filter_str, info_dict):
1862 video_title = info_dict.get('title', info_dict.get('id', 'video'))
1863 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
1867 def parse_dfxp_time_expr(time_expr):
1871 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
1873 return float(mobj.group('time_offset'))
1875 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:\.\d+)?)$', time_expr)
1877 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
1880 def srt_subtitles_timecode(seconds):
1881 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
1884 def dfxp2srt(dfxp_data):
1885 _x = functools.partial(xpath_with_ns, ns_map={
1886 'ttml': 'http://www.w3.org/ns/ttml',
1887 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
1890 def parse_node(node):
1891 str_or_empty = functools.partial(str_or_none, default='')
1893 out = str_or_empty(node.text)
1896 if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
1897 out += '\n' + str_or_empty(child.tail)
1898 elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
1899 out += str_or_empty(parse_node(child))
1901 out += str_or_empty(xml.etree.ElementTree.tostring(child))
1905 dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
1907 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
1910 raise ValueError('Invalid dfxp/TTML subtitle')
1912 for para, index in zip(paras, itertools.count(1)):
1913 begin_time = parse_dfxp_time_expr(para.attrib['begin'])
1914 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
1916 end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
1917 out.append('%d\n%s --> %s\n%s\n\n' % (
1919 srt_subtitles_timecode(begin_time),
1920 srt_subtitles_timecode(end_time),
1926 class ISO639Utils(object):
1927 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2116 def short2long(cls, code):
2117 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2118 return cls._lang_map.get(code[:2])
2121 def long2short(cls, code):
2122 """Convert language code from ISO 639-2/T to ISO 639-1"""
2123 for short_name, long_name in cls._lang_map.items():
2124 if long_name == code:
2128 class ISO3166Utils(object):
2129 # From http://data.okfn.org/data/core/country-list
2131 'AF': 'Afghanistan',
2132 'AX': 'Ã…land Islands',
2135 'AS': 'American Samoa',
2140 'AG': 'Antigua and Barbuda',
2157 'BO': 'Bolivia, Plurinational State of',
2158 'BQ': 'Bonaire, Sint Eustatius and Saba',
2159 'BA': 'Bosnia and Herzegovina',
2161 'BV': 'Bouvet Island',
2163 'IO': 'British Indian Ocean Territory',
2164 'BN': 'Brunei Darussalam',
2166 'BF': 'Burkina Faso',
2172 'KY': 'Cayman Islands',
2173 'CF': 'Central African Republic',
2177 'CX': 'Christmas Island',
2178 'CC': 'Cocos (Keeling) Islands',
2182 'CD': 'Congo, the Democratic Republic of the',
2183 'CK': 'Cook Islands',
2185 'CI': 'Côte d\'Ivoire',
2190 'CZ': 'Czech Republic',
2194 'DO': 'Dominican Republic',
2197 'SV': 'El Salvador',
2198 'GQ': 'Equatorial Guinea',
2202 'FK': 'Falkland Islands (Malvinas)',
2203 'FO': 'Faroe Islands',
2207 'GF': 'French Guiana',
2208 'PF': 'French Polynesia',
2209 'TF': 'French Southern Territories',
2224 'GW': 'Guinea-Bissau',
2227 'HM': 'Heard Island and McDonald Islands',
2228 'VA': 'Holy See (Vatican City State)',
2235 'IR': 'Iran, Islamic Republic of',
2238 'IM': 'Isle of Man',
2248 'KP': 'Korea, Democratic People\'s Republic of',
2249 'KR': 'Korea, Republic of',
2252 'LA': 'Lao People\'s Democratic Republic',
2258 'LI': 'Liechtenstein',
2262 'MK': 'Macedonia, the Former Yugoslav Republic of',
2269 'MH': 'Marshall Islands',
2275 'FM': 'Micronesia, Federated States of',
2276 'MD': 'Moldova, Republic of',
2287 'NL': 'Netherlands',
2288 'NC': 'New Caledonia',
2289 'NZ': 'New Zealand',
2294 'NF': 'Norfolk Island',
2295 'MP': 'Northern Mariana Islands',
2300 'PS': 'Palestine, State of',
2302 'PG': 'Papua New Guinea',
2305 'PH': 'Philippines',
2309 'PR': 'Puerto Rico',
2313 'RU': 'Russian Federation',
2315 'BL': 'Saint Barthélemy',
2316 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2317 'KN': 'Saint Kitts and Nevis',
2318 'LC': 'Saint Lucia',
2319 'MF': 'Saint Martin (French part)',
2320 'PM': 'Saint Pierre and Miquelon',
2321 'VC': 'Saint Vincent and the Grenadines',
2324 'ST': 'Sao Tome and Principe',
2325 'SA': 'Saudi Arabia',
2329 'SL': 'Sierra Leone',
2331 'SX': 'Sint Maarten (Dutch part)',
2334 'SB': 'Solomon Islands',
2336 'ZA': 'South Africa',
2337 'GS': 'South Georgia and the South Sandwich Islands',
2338 'SS': 'South Sudan',
2343 'SJ': 'Svalbard and Jan Mayen',
2346 'CH': 'Switzerland',
2347 'SY': 'Syrian Arab Republic',
2348 'TW': 'Taiwan, Province of China',
2350 'TZ': 'Tanzania, United Republic of',
2352 'TL': 'Timor-Leste',
2356 'TT': 'Trinidad and Tobago',
2359 'TM': 'Turkmenistan',
2360 'TC': 'Turks and Caicos Islands',
2364 'AE': 'United Arab Emirates',
2365 'GB': 'United Kingdom',
2366 'US': 'United States',
2367 'UM': 'United States Minor Outlying Islands',
2371 'VE': 'Venezuela, Bolivarian Republic of',
2373 'VG': 'Virgin Islands, British',
2374 'VI': 'Virgin Islands, U.S.',
2375 'WF': 'Wallis and Futuna',
2376 'EH': 'Western Sahara',
2383 def short2full(cls, code):
2384 """Convert an ISO 3166-2 country code to the corresponding full name"""
2385 return cls._country_map.get(code.upper())
2388 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2389 def __init__(self, proxies=None):
2390 # Set default handlers
2391 for type in ('http', 'https'):
2392 setattr(self, '%s_open' % type,
2393 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2394 meth(r, proxy, type))
2395 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2397 def proxy_open(self, req, proxy, type):
2398 req_proxy = req.headers.get('Ytdl-request-proxy')
2399 if req_proxy is not None:
2401 del req.headers['Ytdl-request-proxy']
2403 if proxy == '__noproxy__':
2404 return None # No Proxy
2405 return compat_urllib_request.ProxyHandler.proxy_open(
2406 self, req, proxy, type)