import zlib
import urllib2
import email.utils
+import json
try:
import cStringIO as StringIO
except ImportError:
import StringIO
-
-try:
- import json
-except ImportError: # Python <2.6, use trivialjson (https://github.com/phihag/trivialjson):
- import trivialjson as json
std_headers = {
- 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
+ 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
# Unknown entity in name, return its literal representation
return (u'&%s;' % entity)
-
+HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
class IDParser(HTMLParser.HTMLParser):
"""Modified HTMLParser that isolates a tag with the specified id"""
def __init__(self, id):
self.depth = {}
self.html = None
self.watch_startpos = False
+ self.error_count = 0
HTMLParser.HTMLParser.__init__(self)
+ def error(self, message):
+ print >> sys.stderr, self.getpos()
+ if self.error_count > 10 or self.started:
+ raise HTMLParser.HTMLParseError(message, self.getpos())
+ self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
+ self.error_count += 1
+ self.goahead(1)
+
def loads(self, html):
self.html = html
self.feed(html)
return html
-def sanitize_title(utitle):
- """Sanitizes a video title so it could be used as part of a filename."""
- utitle = unescapeHTML(utitle)
- return utitle.replace(unicode(os.sep), u'%')
-
-
def sanitize_open(filename, open_mode):
"""Try to open the given filename, and slightly tweak it if this fails.
if timetuple is not None:
timestamp = email.utils.mktime_tz(timetuple)
return timestamp
-
-def simplify_title(title):
- expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE)
- return expr.sub(u'_', title).strip(u'_')
+
+def sanitize_filename(s):
+ """Sanitizes a string so it could be used as part of a filename."""
+ def replace_insane(char):
+ if char in u' .\\/|?*<>:"' or ord(char) < 32:
+ return '_'
+ return char
+ return u''.join(map(replace_insane, s)).strip('_')
def orderedSet(iterable):
""" Remove all duplicates from the input iterable """
assert type(s) == type(u'')
- if sys.platform == 'win32' and sys.getwindowsversion().major >= 5:
+ if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
# Pass u'' directly to use Unicode APIs on Windows 2000 and up
# (Detecting Windows NT 4 is tricky because 'major >= 4' would
# match Windows 9x series as well. Besides, NT 4 is obsolete.)
self.expected = expected
+class Trouble(Exception):
+ """Trouble helper exception
+
+ This is an exception to be handled with
+ FileDownloader.trouble
+ """
+
class YoutubeDLHandler(urllib2.HTTPHandler):
"""Handler for HTTP requests and responses.