2 # -*- coding: utf-8 -*-
17 import cStringIO as StringIO
22 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
23 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
24 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
25 'Accept-Encoding': 'gzip, deflate',
26 'Accept-Language': 'en-us,en;q=0.5',
30 compat_str = unicode # Python 2
34 def preferredencoding():
35 """Get preferred encoding.
37 Returns the best encoding scheme for the system, based on
38 locale.getpreferredencoding() and some further tweaks.
41 pref = locale.getpreferredencoding()
49 def htmlentity_transform(matchobj):
50 """Transforms an HTML entity to a Unicode character.
52 This function receives a match object and is intended to be used with
53 the re.sub() function.
55 entity = matchobj.group(1)
57 # Known non-numeric HTML entity
58 if entity in htmlentitydefs.name2codepoint:
59 return unichr(htmlentitydefs.name2codepoint[entity])
62 mobj = re.match(ur'(?u)#(x?\d+)', entity)
64 numstr = mobj.group(1)
65 if numstr.startswith(u'x'):
67 numstr = u'0%s' % numstr
70 return unichr(long(numstr, base))
72 # Unknown entity in name, return its literal representation
73 return (u'&%s;' % entity)
75 HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
76 class IDParser(HTMLParser.HTMLParser):
77 """Modified HTMLParser that isolates a tag with the specified id"""
78 def __init__(self, id):
84 self.watch_startpos = False
86 HTMLParser.HTMLParser.__init__(self)
88 def error(self, message):
89 if self.error_count > 10 or self.started:
90 raise HTMLParser.HTMLParseError(message, self.getpos())
91 self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
95 def loads(self, html):
100 def handle_starttag(self, tag, attrs):
103 self.find_startpos(None)
104 if 'id' in attrs and attrs['id'] == self.id:
107 self.watch_startpos = True
109 if not tag in self.depth: self.depth[tag] = 0
112 def handle_endtag(self, tag):
114 if tag in self.depth: self.depth[tag] -= 1
115 if self.depth[self.result[0]] == 0:
117 self.result.append(self.getpos())
119 def find_startpos(self, x):
120 """Needed to put the start position of the result (self.result[1])
121 after the opening tag with the requested id"""
122 if self.watch_startpos:
123 self.watch_startpos = False
124 self.result.append(self.getpos())
125 handle_entityref = handle_charref = handle_data = handle_comment = \
126 handle_decl = handle_pi = unknown_decl = find_startpos
128 def get_result(self):
129 if self.result == None: return None
130 if len(self.result) != 3: return None
131 lines = self.html.split('\n')
132 lines = lines[self.result[1][0]-1:self.result[2][0]]
133 lines[0] = lines[0][self.result[1][1]:]
135 lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
136 lines[-1] = lines[-1][:self.result[2][1]]
137 return '\n'.join(lines).strip()
139 def get_element_by_id(id, html):
140 """Return the content of the tag with the specified id in the passed HTML document"""
141 parser = IDParser(id)
144 except HTMLParser.HTMLParseError:
146 return parser.get_result()
149 def clean_html(html):
150 """Clean an HTML snippet into a readable string"""
152 html = html.replace('\n', ' ')
153 html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
155 html = re.sub('<.*?>', '', html)
156 # Replace html entities
157 html = unescapeHTML(html)
161 def sanitize_open(filename, open_mode):
162 """Try to open the given filename, and slightly tweak it if this fails.
164 Attempts to open the given filename. If this fails, it tries to change
165 the filename slightly, step by step, until it's either able to open it
166 or it fails and raises a final exception, like the standard open()
169 It returns the tuple (stream, definitive_file_name).
173 if sys.platform == 'win32':
175 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
176 return (sys.stdout, filename)
177 stream = open(encodeFilename(filename), open_mode)
178 return (stream, filename)
179 except (IOError, OSError), err:
180 # In case of error, try to remove win32 forbidden chars
181 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
183 # An exception here should be caught in the caller
184 stream = open(encodeFilename(filename), open_mode)
185 return (stream, filename)
188 def timeconvert(timestr):
189 """Convert RFC 2822 defined time string into system timestamp"""
191 timetuple = email.utils.parsedate_tz(timestr)
192 if timetuple is not None:
193 timestamp = email.utils.mktime_tz(timetuple)
196 def sanitize_filename(s, restricted=False):
197 """Sanitizes a string so it could be used as part of a filename.
198 If restricted is set, use a stricter subset of allowed characters.
200 def replace_insane(char):
201 if char == '?' or ord(char) < 32 or ord(char) == 127:
204 return '' if restricted else '\''
206 return '_-' if restricted else ' -'
207 elif char in '\\/|*<>':
209 if restricted and (char in '!&\'' or char.isspace()):
211 if restricted and ord(char) > 127:
215 result = u''.join(map(replace_insane, s))
216 while '__' in result:
217 result = result.replace('__', '_')
218 result = result.strip('_')
219 # Common case of "Foreign band name - English song title"
220 if restricted and result.startswith('-_'):
226 def orderedSet(iterable):
227 """ Remove all duplicates from the input iterable """
236 @param s a string (of type unicode)
238 assert type(s) == type(u'')
240 result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s)
243 def encodeFilename(s):
245 @param s The name of the file (of type unicode)
248 assert type(s) == type(u'')
250 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
251 # Pass u'' directly to use Unicode APIs on Windows 2000 and up
252 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
253 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
256 return s.encode(sys.getfilesystemencoding(), 'ignore')
258 class DownloadError(Exception):
259 """Download Error exception.
261 This exception may be thrown by FileDownloader objects if they are not
262 configured to continue on errors. They will contain the appropriate
268 class SameFileError(Exception):
269 """Same File exception.
271 This exception will be thrown by FileDownloader objects if they detect
272 multiple files would have to be downloaded to the same file on disk.
277 class PostProcessingError(Exception):
278 """Post Processing exception.
280 This exception may be raised by PostProcessor's .run() method to
281 indicate an error in the postprocessing task.
285 class MaxDownloadsReached(Exception):
286 """ --max-downloads limit has been reached. """
290 class UnavailableVideoError(Exception):
291 """Unavailable Format exception.
293 This exception will be thrown when a video is requested
294 in a format that is not available for that video.
299 class ContentTooShortError(Exception):
300 """Content Too Short exception.
302 This exception may be raised by FileDownloader objects when a file they
303 download is too small for what the server announced first, indicating
304 the connection was probably interrupted.
310 def __init__(self, downloaded, expected):
311 self.downloaded = downloaded
312 self.expected = expected
315 class Trouble(Exception):
316 """Trouble helper exception
318 This is an exception to be handled with
319 FileDownloader.trouble
322 class YoutubeDLHandler(urllib2.HTTPHandler):
323 """Handler for HTTP requests and responses.
325 This class, when installed with an OpenerDirector, automatically adds
326 the standard headers to every HTTP request and handles gzipped and
327 deflated responses from web servers. If compression is to be avoided in
328 a particular request, the original request in the program code only has
329 to include the HTTP header "Youtubedl-No-Compression", which will be
330 removed before making the real request.
332 Part of this code was copied from:
334 http://techknack.net/python-urllib2-handlers/
336 Andrew Rowls, the author of that code, agreed to release it to the
343 return zlib.decompress(data, -zlib.MAX_WBITS)
345 return zlib.decompress(data)
348 def addinfourl_wrapper(stream, headers, url, code):
349 if hasattr(urllib2.addinfourl, 'getcode'):
350 return urllib2.addinfourl(stream, headers, url, code)
351 ret = urllib2.addinfourl(stream, headers, url)
355 def http_request(self, req):
356 for h in std_headers:
359 req.add_header(h, std_headers[h])
360 if 'Youtubedl-no-compression' in req.headers:
361 if 'Accept-encoding' in req.headers:
362 del req.headers['Accept-encoding']
363 del req.headers['Youtubedl-no-compression']
366 def http_response(self, req, resp):
369 if resp.headers.get('Content-encoding', '') == 'gzip':
370 gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
371 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
372 resp.msg = old_resp.msg
374 if resp.headers.get('Content-encoding', '') == 'deflate':
375 gz = StringIO.StringIO(self.deflate(resp.read()))
376 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
377 resp.msg = old_resp.msg