2 # -*- coding: utf-8 -*-
18 import urllib.request as compat_urllib_request
19 except ImportError: # Python 2
20 import urllib2 as compat_urllib_request
23 import urllib.error as compat_urllib_error
24 except ImportError: # Python 2
25 import urllib2 as compat_urllib_error
28 import urllib.parse as compat_urllib_parse
29 except ImportError: # Python 2
30 import urllib as compat_urllib_parse
33 from urllib.parse import urlparse as compat_urllib_parse_urlparse
34 except ImportError: # Python 2
35 from urlparse import urlparse as compat_urllib_parse_urlparse
38 import http.cookiejar as compat_cookiejar
39 except ImportError: # Python 2
40 import cookielib as compat_cookiejar
43 import html.entities as compat_html_entities
44 except ImportError: # Python 2
45 import htmlentitydefs as compat_html_entities
48 import html.parser as compat_html_parser
49 except ImportError: # Python 2
50 import HTMLParser as compat_html_parser
53 import http.client as compat_http_client
54 except ImportError: # Python 2
55 import httplib as compat_http_client
58 from subprocess import DEVNULL
59 compat_subprocess_get_DEVNULL = lambda: DEVNULL
61 compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
64 from urllib.parse import parse_qs as compat_parse_qs
65 except ImportError: # Python 2
66 # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
67 # Python 2's version is apparently totally broken
68 def _unquote(string, encoding='utf-8', errors='replace'):
71 res = string.split('%')
78 # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
85 pct_sequence += item[:2].decode('hex')
88 # This segment was just a single percent-encoded character.
89 # May be part of a sequence of code units, so delay decoding.
90 # (Stored in pct_sequence).
94 # Encountered non-percent-encoded characters. Flush the current
96 string += pct_sequence.decode(encoding, errors) + rest
99 # Flush the final pct_sequence
100 string += pct_sequence.decode(encoding, errors)
103 def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
104 encoding='utf-8', errors='replace'):
105 qs, _coerce_result = qs, unicode
106 pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
108 for name_value in pairs:
109 if not name_value and not strict_parsing:
111 nv = name_value.split('=', 1)
114 raise ValueError("bad query field: %r" % (name_value,))
115 # Handle case of a control-name with no equal sign
116 if keep_blank_values:
120 if len(nv[1]) or keep_blank_values:
121 name = nv[0].replace('+', ' ')
122 name = _unquote(name, encoding=encoding, errors=errors)
123 name = _coerce_result(name)
124 value = nv[1].replace('+', ' ')
125 value = _unquote(value, encoding=encoding, errors=errors)
126 value = _coerce_result(value)
127 r.append((name, value))
130 def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
131 encoding='utf-8', errors='replace'):
133 pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
134 encoding=encoding, errors=errors)
135 for name, value in pairs:
136 if name in parsed_result:
137 parsed_result[name].append(value)
139 parsed_result[name] = [value]
143 compat_str = unicode # Python 2
148 compat_chr = unichr # Python 2
153 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
154 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
155 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
156 'Accept-Encoding': 'gzip, deflate',
157 'Accept-Language': 'en-us,en;q=0.5',
160 def preferredencoding():
161 """Get preferred encoding.
163 Returns the best encoding scheme for the system, based on
164 locale.getpreferredencoding() and some further tweaks.
167 pref = locale.getpreferredencoding()
174 if sys.version_info < (3,0):
176 print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
179 assert type(s) == type(u'')
182 # In Python 2.x, json.dump expects a bytestream.
183 # In Python 3.x, it writes to a character stream
184 if sys.version_info < (3,0):
185 def write_json_file(obj, fn):
186 with open(fn, 'wb') as f:
189 def write_json_file(obj, fn):
190 with open(fn, 'w', encoding='utf-8') as f:
193 def htmlentity_transform(matchobj):
194 """Transforms an HTML entity to a character.
196 This function receives a match object and is intended to be used with
197 the re.sub() function.
199 entity = matchobj.group(1)
201 # Known non-numeric HTML entity
202 if entity in compat_html_entities.name2codepoint:
203 return compat_chr(compat_html_entities.name2codepoint[entity])
205 mobj = re.match(u'(?u)#(x?\\d+)', entity)
207 numstr = mobj.group(1)
208 if numstr.startswith(u'x'):
210 numstr = u'0%s' % numstr
213 return compat_chr(int(numstr, base))
215 # Unknown entity in name, return its literal representation
216 return (u'&%s;' % entity)
218 compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
219 class AttrParser(compat_html_parser.HTMLParser):
220 """Modified HTMLParser that isolates a tag with the specified attribute"""
221 def __init__(self, attribute, value):
222 self.attribute = attribute
228 self.watch_startpos = False
230 compat_html_parser.HTMLParser.__init__(self)
232 def error(self, message):
233 if self.error_count > 10 or self.started:
234 raise compat_html_parser.HTMLParseError(message, self.getpos())
235 self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
236 self.error_count += 1
239 def loads(self, html):
244 def handle_starttag(self, tag, attrs):
247 self.find_startpos(None)
248 if self.attribute in attrs and attrs[self.attribute] == self.value:
251 self.watch_startpos = True
253 if not tag in self.depth: self.depth[tag] = 0
256 def handle_endtag(self, tag):
258 if tag in self.depth: self.depth[tag] -= 1
259 if self.depth[self.result[0]] == 0:
261 self.result.append(self.getpos())
263 def find_startpos(self, x):
264 """Needed to put the start position of the result (self.result[1])
265 after the opening tag with the requested id"""
266 if self.watch_startpos:
267 self.watch_startpos = False
268 self.result.append(self.getpos())
269 handle_entityref = handle_charref = handle_data = handle_comment = \
270 handle_decl = handle_pi = unknown_decl = find_startpos
272 def get_result(self):
273 if self.result is None:
275 if len(self.result) != 3:
277 lines = self.html.split('\n')
278 lines = lines[self.result[1][0]-1:self.result[2][0]]
279 lines[0] = lines[0][self.result[1][1]:]
281 lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
282 lines[-1] = lines[-1][:self.result[2][1]]
283 return '\n'.join(lines).strip()
284 # Hack for https://github.com/rg3/youtube-dl/issues/662
285 if sys.version_info < (2, 7, 3):
286 AttrParser.parse_endtag = (lambda self, i:
287 i + len("</scr'+'ipt>")
288 if self.rawdata[i:].startswith("</scr'+'ipt>")
289 else compat_html_parser.HTMLParser.parse_endtag(self, i))
291 def get_element_by_id(id, html):
292 """Return the content of the tag with the specified ID in the passed HTML document"""
293 return get_element_by_attribute("id", id, html)
295 def get_element_by_attribute(attribute, value, html):
296 """Return the content of the tag with the specified attribute in the passed HTML document"""
297 parser = AttrParser(attribute, value)
300 except compat_html_parser.HTMLParseError:
302 return parser.get_result()
305 def clean_html(html):
306 """Clean an HTML snippet into a readable string"""
308 html = html.replace('\n', ' ')
309 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
310 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
312 html = re.sub('<.*?>', '', html)
313 # Replace html entities
314 html = unescapeHTML(html)
318 def sanitize_open(filename, open_mode):
319 """Try to open the given filename, and slightly tweak it if this fails.
321 Attempts to open the given filename. If this fails, it tries to change
322 the filename slightly, step by step, until it's either able to open it
323 or it fails and raises a final exception, like the standard open()
326 It returns the tuple (stream, definitive_file_name).
330 if sys.platform == 'win32':
332 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
333 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
334 stream = open(encodeFilename(filename), open_mode)
335 return (stream, filename)
336 except (IOError, OSError) as err:
337 # In case of error, try to remove win32 forbidden chars
338 filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)
340 # An exception here should be caught in the caller
341 stream = open(encodeFilename(filename), open_mode)
342 return (stream, filename)
345 def timeconvert(timestr):
346 """Convert RFC 2822 defined time string into system timestamp"""
348 timetuple = email.utils.parsedate_tz(timestr)
349 if timetuple is not None:
350 timestamp = email.utils.mktime_tz(timetuple)
353 def sanitize_filename(s, restricted=False, is_id=False):
354 """Sanitizes a string so it could be used as part of a filename.
355 If restricted is set, use a stricter subset of allowed characters.
356 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
358 def replace_insane(char):
359 if char == '?' or ord(char) < 32 or ord(char) == 127:
362 return '' if restricted else '\''
364 return '_-' if restricted else ' -'
365 elif char in '\\/|*<>':
367 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
369 if restricted and ord(char) > 127:
373 result = u''.join(map(replace_insane, s))
375 while '__' in result:
376 result = result.replace('__', '_')
377 result = result.strip('_')
378 # Common case of "Foreign band name - English song title"
379 if restricted and result.startswith('-_'):
385 def orderedSet(iterable):
386 """ Remove all duplicates from the input iterable """
397 assert type(s) == type(u'')
399 result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
402 def encodeFilename(s):
404 @param s The name of the file
407 assert type(s) == type(u'')
409 # Python 3 has a Unicode API
410 if sys.version_info >= (3, 0):
413 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
414 # Pass u'' directly to use Unicode APIs on Windows 2000 and up
415 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
416 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
419 encoding = sys.getfilesystemencoding()
422 return s.encode(encoding, 'ignore')
424 def decodeOption(optval):
427 if isinstance(optval, bytes):
428 optval = optval.decode(preferredencoding())
430 assert isinstance(optval, compat_str)
433 class ExtractorError(Exception):
434 """Error during info extraction."""
435 def __init__(self, msg, tb=None):
436 """ tb, if given, is the original traceback (so that it can be printed out). """
437 super(ExtractorError, self).__init__(msg)
439 self.exc_info = sys.exc_info() # preserve original exception
441 def format_traceback(self):
442 if self.traceback is None:
444 return u''.join(traceback.format_tb(self.traceback))
447 class DownloadError(Exception):
448 """Download Error exception.
450 This exception may be thrown by FileDownloader objects if they are not
451 configured to continue on errors. They will contain the appropriate
454 def __init__(self, msg, exc_info=None):
455 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
456 super(DownloadError, self).__init__(msg)
457 self.exc_info = exc_info
460 class SameFileError(Exception):
461 """Same File exception.
463 This exception will be thrown by FileDownloader objects if they detect
464 multiple files would have to be downloaded to the same file on disk.
469 class PostProcessingError(Exception):
470 """Post Processing exception.
472 This exception may be raised by PostProcessor's .run() method to
473 indicate an error in the postprocessing task.
475 def __init__(self, msg):
478 class MaxDownloadsReached(Exception):
479 """ --max-downloads limit has been reached. """
483 class UnavailableVideoError(Exception):
484 """Unavailable Format exception.
486 This exception will be thrown when a video is requested
487 in a format that is not available for that video.
492 class ContentTooShortError(Exception):
493 """Content Too Short exception.
495 This exception may be raised by FileDownloader objects when a file they
496 download is too small for what the server announced first, indicating
497 the connection was probably interrupted.
503 def __init__(self, downloaded, expected):
504 self.downloaded = downloaded
505 self.expected = expected
507 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
508 """Handler for HTTP requests and responses.
510 This class, when installed with an OpenerDirector, automatically adds
511 the standard headers to every HTTP request and handles gzipped and
512 deflated responses from web servers. If compression is to be avoided in
513 a particular request, the original request in the program code only has
514 to include the HTTP header "Youtubedl-No-Compression", which will be
515 removed before making the real request.
517 Part of this code was copied from:
519 http://techknack.net/python-urllib2-handlers/
521 Andrew Rowls, the author of that code, agreed to release it to the
528 return zlib.decompress(data, -zlib.MAX_WBITS)
530 return zlib.decompress(data)
533 def addinfourl_wrapper(stream, headers, url, code):
534 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
535 return compat_urllib_request.addinfourl(stream, headers, url, code)
536 ret = compat_urllib_request.addinfourl(stream, headers, url)
540 def http_request(self, req):
541 for h,v in std_headers.items():
545 if 'Youtubedl-no-compression' in req.headers:
546 if 'Accept-encoding' in req.headers:
547 del req.headers['Accept-encoding']
548 del req.headers['Youtubedl-no-compression']
549 if 'Youtubedl-user-agent' in req.headers:
550 if 'User-agent' in req.headers:
551 del req.headers['User-agent']
552 req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
553 del req.headers['Youtubedl-user-agent']
556 def http_response(self, req, resp):
559 if resp.headers.get('Content-encoding', '') == 'gzip':
560 gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
561 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
562 resp.msg = old_resp.msg
564 if resp.headers.get('Content-encoding', '') == 'deflate':
565 gz = io.BytesIO(self.deflate(resp.read()))
566 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
567 resp.msg = old_resp.msg
570 https_request = http_request
571 https_response = http_response
573 def unified_strdate(date_str):
574 """Return a string with the date in the format YYYYMMDD"""
577 date_str = date_str.replace(',',' ')
578 # %z (UTC offset) is only supported in python>=3.2
579 date_str = re.sub(r' (\+|-)[\d]*$', '', date_str)
580 format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y', '%Y-%m-%d', '%d/%m/%Y', '%Y/%m/%d %H:%M:%S']
581 for expression in format_expressions:
583 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
588 def date_from_str(date_str):
590 Return a datetime object from a string in the format YYYYMMDD or
591 (now|today)[+-][0-9](day|week|month|year)(s)?"""
592 today = datetime.date.today()
593 if date_str == 'now'or date_str == 'today':
595 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
596 if match is not None:
597 sign = match.group('sign')
598 time = int(match.group('time'))
601 unit = match.group('unit')
610 delta = datetime.timedelta(**{unit: time})
612 return datetime.datetime.strptime(date_str, "%Y%m%d").date()
614 class DateRange(object):
615 """Represents a time interval between two dates"""
616 def __init__(self, start=None, end=None):
617 """start and end must be strings in the format accepted by date"""
618 if start is not None:
619 self.start = date_from_str(start)
621 self.start = datetime.datetime.min.date()
623 self.end = date_from_str(end)
625 self.end = datetime.datetime.max.date()
626 if self.start > self.end:
627 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
630 """Returns a range that only contains the given day"""
632 def __contains__(self, date):
633 """Check if the date is in the range"""
634 if not isinstance(date, datetime.date):
635 date = date_from_str(date)
636 return self.start <= date <= self.end
638 return '%s - %s' % ( self.start.isoformat(), self.end.isoformat())