import StringIO
std_headers = {
- 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
+ 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
HTMLParser.HTMLParser.__init__(self)
def error(self, message):
- print self.getpos()
+ print >> sys.stderr, self.getpos()
if self.error_count > 10 or self.started:
raise HTMLParser.HTMLParseError(message, self.getpos())
self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
def sanitize_filename(s):
"""Sanitizes a string so it could be used as part of a filename."""
def replace_insane(char):
- if char in u' .\\/|?*<>:"' or ord(char) < 32:
- return '_'
+ if char == '?' or ord(char) < 32 or ord(char) == 127:
+ return ''
+ elif char == '"':
+ return '\''
+ elif char == ':':
+ return ' -'
+ elif char in '\\/|*<>':
+ return '-'
return char
- return u''.join(map(replace_insane, s)).strip('_')
+
+ result = u''.join(map(replace_insane, s))
+ while '--' in result:
+ result = result.replace('--', '-')
+ return result.strip('-')
def orderedSet(iterable):
""" Remove all duplicates from the input iterable """
assert type(s) == type(u'')
- if sys.platform == 'win32' and sys.getwindowsversion().major >= 5:
+ if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
# Pass u'' directly to use Unicode APIs on Windows 2000 and up
# (Detecting Windows NT 4 is tricky because 'major >= 4' would
# match Windows 9x series as well. Besides, NT 4 is obsolete.)