2 # -*- coding: utf-8 -*-
17 import cStringIO as StringIO
22 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
23 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
24 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
25 'Accept-Encoding': 'gzip, deflate',
26 'Accept-Language': 'en-us,en;q=0.5',
30 compat_str = unicode # Python 2
34 def preferredencoding():
35 """Get preferred encoding.
37 Returns the best encoding scheme for the system, based on
38 locale.getpreferredencoding() and some further tweaks.
41 pref = locale.getpreferredencoding()
49 def htmlentity_transform(matchobj):
50 """Transforms an HTML entity to a Unicode character.
52 This function receives a match object and is intended to be used with
53 the re.sub() function.
55 entity = matchobj.group(1)
57 # Known non-numeric HTML entity
58 if entity in htmlentitydefs.name2codepoint:
59 return unichr(htmlentitydefs.name2codepoint[entity])
62 mobj = re.match(ur'(?u)#(x?\d+)', entity)
64 numstr = mobj.group(1)
65 if numstr.startswith(u'x'):
67 numstr = u'0%s' % numstr
70 return unichr(long(numstr, base))
72 # Unknown entity in name, return its literal representation
73 return (u'&%s;' % entity)
75 HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
76 class IDParser(HTMLParser.HTMLParser):
77 """Modified HTMLParser that isolates a tag with the specified id"""
78 def __init__(self, id):
84 self.watch_startpos = False
86 HTMLParser.HTMLParser.__init__(self)
88 def error(self, message):
89 if self.error_count > 10 or self.started:
90 raise HTMLParser.HTMLParseError(message, self.getpos())
91 self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
95 def loads(self, html):
100 def handle_starttag(self, tag, attrs):
103 self.find_startpos(None)
104 if 'id' in attrs and attrs['id'] == self.id:
107 self.watch_startpos = True
109 if not tag in self.depth: self.depth[tag] = 0
112 def handle_endtag(self, tag):
114 if tag in self.depth: self.depth[tag] -= 1
115 if self.depth[self.result[0]] == 0:
117 self.result.append(self.getpos())
119 def find_startpos(self, x):
120 """Needed to put the start position of the result (self.result[1])
121 after the opening tag with the requested id"""
122 if self.watch_startpos:
123 self.watch_startpos = False
124 self.result.append(self.getpos())
125 handle_entityref = handle_charref = handle_data = handle_comment = \
126 handle_decl = handle_pi = unknown_decl = find_startpos
128 def get_result(self):
129 if self.result is None:
131 if len(self.result) != 3:
133 lines = self.html.split('\n')
134 lines = lines[self.result[1][0]-1:self.result[2][0]]
135 lines[0] = lines[0][self.result[1][1]:]
137 lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
138 lines[-1] = lines[-1][:self.result[2][1]]
139 return '\n'.join(lines).strip()
141 def get_element_by_id(id, html):
142 """Return the content of the tag with the specified id in the passed HTML document"""
143 parser = IDParser(id)
146 except HTMLParser.HTMLParseError:
148 return parser.get_result()
151 def clean_html(html):
152 """Clean an HTML snippet into a readable string"""
154 html = html.replace('\n', ' ')
155 html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
157 html = re.sub('<.*?>', '', html)
158 # Replace html entities
159 html = unescapeHTML(html)
163 def sanitize_open(filename, open_mode):
164 """Try to open the given filename, and slightly tweak it if this fails.
166 Attempts to open the given filename. If this fails, it tries to change
167 the filename slightly, step by step, until it's either able to open it
168 or it fails and raises a final exception, like the standard open()
171 It returns the tuple (stream, definitive_file_name).
175 if sys.platform == 'win32':
177 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
178 return (sys.stdout, filename)
179 stream = open(encodeFilename(filename), open_mode)
180 return (stream, filename)
181 except (IOError, OSError), err:
182 # In case of error, try to remove win32 forbidden chars
183 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
185 # An exception here should be caught in the caller
186 stream = open(encodeFilename(filename), open_mode)
187 return (stream, filename)
190 def timeconvert(timestr):
191 """Convert RFC 2822 defined time string into system timestamp"""
193 timetuple = email.utils.parsedate_tz(timestr)
194 if timetuple is not None:
195 timestamp = email.utils.mktime_tz(timetuple)
198 def sanitize_filename(s, restricted=False):
199 """Sanitizes a string so it could be used as part of a filename.
200 If restricted is set, use a stricter subset of allowed characters.
202 def replace_insane(char):
203 if char == '?' or ord(char) < 32 or ord(char) == 127:
206 return '' if restricted else '\''
208 return '_-' if restricted else ' -'
209 elif char in '\\/|*<>':
211 if restricted and (char in '!&\'' or char.isspace()):
213 if restricted and ord(char) > 127:
217 result = u''.join(map(replace_insane, s))
218 while '__' in result:
219 result = result.replace('__', '_')
220 result = result.strip('_')
221 # Common case of "Foreign band name - English song title"
222 if restricted and result.startswith('-_'):
228 def orderedSet(iterable):
229 """ Remove all duplicates from the input iterable """
238 @param s a string (of type unicode)
240 assert type(s) == type(u'')
242 result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s)
245 def encodeFilename(s):
247 @param s The name of the file (of type unicode)
250 assert type(s) == type(u'')
252 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
253 # Pass u'' directly to use Unicode APIs on Windows 2000 and up
254 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
255 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
258 return s.encode(sys.getfilesystemencoding(), 'ignore')
260 class DownloadError(Exception):
261 """Download Error exception.
263 This exception may be thrown by FileDownloader objects if they are not
264 configured to continue on errors. They will contain the appropriate
270 class SameFileError(Exception):
271 """Same File exception.
273 This exception will be thrown by FileDownloader objects if they detect
274 multiple files would have to be downloaded to the same file on disk.
279 class PostProcessingError(Exception):
280 """Post Processing exception.
282 This exception may be raised by PostProcessor's .run() method to
283 indicate an error in the postprocessing task.
287 class MaxDownloadsReached(Exception):
288 """ --max-downloads limit has been reached. """
292 class UnavailableVideoError(Exception):
293 """Unavailable Format exception.
295 This exception will be thrown when a video is requested
296 in a format that is not available for that video.
301 class ContentTooShortError(Exception):
302 """Content Too Short exception.
304 This exception may be raised by FileDownloader objects when a file they
305 download is too small for what the server announced first, indicating
306 the connection was probably interrupted.
312 def __init__(self, downloaded, expected):
313 self.downloaded = downloaded
314 self.expected = expected
317 class Trouble(Exception):
318 """Trouble helper exception
320 This is an exception to be handled with
321 FileDownloader.trouble
324 class YoutubeDLHandler(urllib2.HTTPHandler):
325 """Handler for HTTP requests and responses.
327 This class, when installed with an OpenerDirector, automatically adds
328 the standard headers to every HTTP request and handles gzipped and
329 deflated responses from web servers. If compression is to be avoided in
330 a particular request, the original request in the program code only has
331 to include the HTTP header "Youtubedl-No-Compression", which will be
332 removed before making the real request.
334 Part of this code was copied from:
336 http://techknack.net/python-urllib2-handlers/
338 Andrew Rowls, the author of that code, agreed to release it to the
345 return zlib.decompress(data, -zlib.MAX_WBITS)
347 return zlib.decompress(data)
350 def addinfourl_wrapper(stream, headers, url, code):
351 if hasattr(urllib2.addinfourl, 'getcode'):
352 return urllib2.addinfourl(stream, headers, url, code)
353 ret = urllib2.addinfourl(stream, headers, url)
357 def http_request(self, req):
358 for h in std_headers:
361 req.add_header(h, std_headers[h])
362 if 'Youtubedl-no-compression' in req.headers:
363 if 'Accept-encoding' in req.headers:
364 del req.headers['Accept-encoding']
365 del req.headers['Youtubedl-no-compression']
368 def http_response(self, req, resp):
371 if resp.headers.get('Content-encoding', '') == 'gzip':
372 gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
373 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
374 resp.msg = old_resp.msg
376 if resp.headers.get('Content-encoding', '') == 'deflate':
377 gz = StringIO.StringIO(self.deflate(resp.read()))
378 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
379 resp.msg = old_resp.msg