X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Futils.py;h=56d0461456fe7e96a259100d7e60bdf2f1dc87a2;hb=297d7fd9c0ac076daf509d14034c74f401bb12c5;hp=737cca8e13ad03fffa6848178c65415ec7d7ec7d;hpb=d11d05d07acdd11a93b02d750852dea4ae32be3b;p=youtube-dl

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 737cca8e1..56d046145 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -11,19 +11,15 @@ import sys
 import zlib
 import urllib2
 import email.utils
+import json
 
 try:
 	import cStringIO as StringIO
 except ImportError:
 	import StringIO
-		
-try:
-	import json
-except ImportError: # Python <2.6, use trivialjson (https://github.com/phihag/trivialjson):
-	import trivialjson as json
 
 std_headers = {
-	'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
+	'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
 	'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
 	'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
 	'Accept-Encoding': 'gzip, deflate',
@@ -73,11 +69,91 @@ def htmlentity_transform(matchobj):
 	# Unknown entity in name, return its literal representation
 	return (u'&%s;' % entity)
 
+HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
+class IDParser(HTMLParser.HTMLParser):
+	"""Modified HTMLParser that isolates a tag with the specified id"""
+	def __init__(self, id):
+		self.id = id
+		self.result = None
+		self.started = False
+		self.depth = {}
+		self.html = None
+		self.watch_startpos = False
+		self.error_count = 0
+		HTMLParser.HTMLParser.__init__(self)
+
+	def error(self, message):
+		#print >> sys.stderr, self.getpos()
+		if self.error_count > 10 or self.started:
+			raise HTMLParser.HTMLParseError(message, self.getpos())
+		self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
+		self.error_count += 1
+		self.goahead(1)
+
+	def loads(self, html):
+		self.html = html
+		self.feed(html)
+		self.close()
+
+	def handle_starttag(self, tag, attrs):
+		attrs = dict(attrs)
+		if self.started:
+			self.find_startpos(None)
+		if 'id' in attrs and attrs['id'] == self.id:
+			self.result = [tag]
+			self.started = True
+			self.watch_startpos = True
+		if self.started:
+			if not tag in self.depth: self.depth[tag] = 0
+			self.depth[tag] += 1
+
+	def handle_endtag(self, tag):
+		if self.started:
+			if tag in self.depth: self.depth[tag] -= 1
+			if self.depth[self.result[0]] == 0:
+				self.started = False
+				self.result.append(self.getpos())
+
+	def find_startpos(self, x):
+		"""Needed to put the start position of the result (self.result[1])
+		after the opening tag with the requested id"""
+		if self.watch_startpos:
+			self.watch_startpos = False
+			self.result.append(self.getpos())
+	handle_entityref = handle_charref = handle_data = handle_comment = \
+	handle_decl = handle_pi = unknown_decl = find_startpos
+
+	def get_result(self):
+		if self.result == None: return None
+		if len(self.result) != 3: return None
+		lines = self.html.split('\n')
+		lines = lines[self.result[1][0]-1:self.result[2][0]]
+		lines[0] = lines[0][self.result[1][1]:]
+		if len(lines) == 1:
+			lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
+		lines[-1] = lines[-1][:self.result[2][1]]
+		return '\n'.join(lines).strip()
+
+def get_element_by_id(id, html):
+	"""Return the content of the tag with the specified id in the passed HTML document"""
+	parser = IDParser(id)
+	try:
+		parser.loads(html)
+	except HTMLParser.HTMLParseError:
+		pass
+	return parser.get_result()
+
 
-def sanitize_title(utitle):
-	"""Sanitizes a video title so it could be used as part of a filename."""
-	utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
-	return utitle.replace(unicode(os.sep), u'%')
+def clean_html(html):
+	"""Clean an HTML snippet into a readable string"""
+	# Newline vs <br />
+	html = html.replace('\n', ' ')
+	html = re.sub('\s*<\s*br\s*/?\s*>\s*', '\n', html)
+	# Strip html tags
+	html = re.sub('<.*?>', '', html)
+	# Replace html entities
+	html = unescapeHTML(html)
+	return html
 
 
 def sanitize_open(filename, open_mode):
@@ -114,10 +190,24 @@ def timeconvert(timestr):
 	if timetuple is not None:
 		timestamp = email.utils.mktime_tz(timetuple)
 	return timestamp
-
-def simplify_title(title):
-	expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE)
-	return expr.sub(u'_', title).strip(u'_')
+	
+def sanitize_filename(s):
+	"""Sanitizes a string so it could be used as part of a filename."""
+	def replace_insane(char):
+		if char == '?' or ord(char) < 32 or ord(char) == 127:
+			return ''
+		elif char == '"':
+			return '\''
+		elif char == ':':
+			return ' -'
+		elif char in '\\/|*<>':
+			return '-'
+		return char
+
+	result = u''.join(map(replace_insane, s))
+	while '--' in result:
+		result = result.replace('--', '-')
+	return result.strip('-')
 
 def orderedSet(iterable):
 	""" Remove all duplicates from the input iterable """
@@ -133,8 +223,8 @@ def unescapeHTML(s):
 	"""
 	assert type(s) == type(u'')
 
-	htmlParser = HTMLParser.HTMLParser()
-	return htmlParser.unescape(s)
+	result = re.sub(ur'(?u)&(.+?);', htmlentity_transform, s)
+	return result
 
 def encodeFilename(s):
 	"""
@@ -143,7 +233,7 @@ def encodeFilename(s):
 
 	assert type(s) == type(u'')
 
-	if sys.platform == 'win32' and sys.getwindowsversion().major >= 5:
+	if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
 		# Pass u'' directly to use Unicode APIs on Windows 2000 and up
 		# (Detecting Windows NT 4 is tricky because 'major >= 4' would
 		# match Windows 9x series as well. Besides, NT 4 is obsolete.)
@@ -208,6 +298,13 @@ class ContentTooShortError(Exception):
 		self.expected = expected
 
 
+class Trouble(Exception):
+	"""Trouble helper exception
+	
+	This is an exception to be handled with
+	FileDownloader.trouble
+	"""
+
 class YoutubeDLHandler(urllib2.HTTPHandler):
 	"""Handler for HTTP requests and responses.