X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2FInfoExtractors.py;h=cea30dad81fa4224a848732159aa19684c7d5dbc;hb=33d94a6c999ae784be7529aaaea42adadeab0c27;hp=4314f14022b650cb965707ee5c90e07fbecd67c4;hpb=d11d05d07acdd11a93b02d750852dea4ae32be3b;p=youtube-dl

diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py
index 4314f1402..cea30dad8 100644
--- a/youtube_dl/InfoExtractors.py
+++ b/youtube_dl/InfoExtractors.py
@@ -12,28 +12,16 @@ import time
 import urllib
 import urllib2
 import email.utils
+import xml.etree.ElementTree
+import random
+import math
+from urlparse import parse_qs
 
 try:
 	import cStringIO as StringIO
 except ImportError:
 	import StringIO
 
-# parse_qs was moved from the cgi module to the urlparse module recently.
-try:
-	from urlparse import parse_qs
-except ImportError:
-	from cgi import parse_qs
-
-try:
-	import lxml.etree
-except ImportError:
-	pass # Handled below
-
-try:
-	import xml.etree.ElementTree
-except ImportError: # Python<2.5: Not officially supported, but let it slip
-	warnings.warn('xml.etree.ElementTree support is missing. Consider upgrading to Python >= 2.5 if you get related errors.')
-
 from utils import *
 
 
@@ -53,7 +41,6 @@ class InfoExtractor(object):
 	url:		Final video URL.
 	uploader:	Nickname of the video uploader.
 	title:		Literal title.
-	stitle:		Simplified title.
 	ext:		Video filename extension.
 	format:		Video format.
 	player_url:	SWF Player URL (may be None).
@@ -110,15 +97,34 @@ class InfoExtractor(object):
 class YoutubeIE(InfoExtractor):
 	"""Information extractor for youtube.com."""
 
-	_VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?!view_play_list|my_playlists|artist|playlist)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$'
+	_VALID_URL = r"""^
+	                 (
+	                     (?:https?://)?                                       # http(s):// (optional)
+	                     (?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/|
+	                     	tube\.majestyc\.net/)                             # the various hostnames, with wildcard subdomains
+	                     (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
+	                     (?!view_play_list|my_playlists|artist|playlist)      # ignore playlist URLs
+	                     (?:                                                  # the various things that can precede the ID:
+	                         (?:(?:v|embed|e)/)                               # v/ or embed/ or e/
+	                         |(?:                                             # or the v= param in all its forms
+	                             (?:watch(?:_popup)?(?:\.php)?)?              # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
+	                             (?:\?|\#!?)                                  # the params delimiter ? or # or #!
+	                             (?:.+&)?                                     # any other preceding param (like /?s=tuff&v=xxxx)
+	                             v=
+	                         )
+	                     )?                                                   # optional -> youtube.com/xxxx is OK
+	                 )?                                                       # all until now is optional -> you can pass the naked ID
+	                 ([0-9A-Za-z_-]+)                                         # here is it! the YouTube video ID
+	                 (?(1).+)?                                                # if we found the ID, everything can follow
+	                 $"""
 	_LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 	_LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
 	_AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 	_NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
 	_NETRC_MACHINE = 'youtube'
 	# Listed in order of quality
-	_available_formats = ['38', '37', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
-	_available_formats_prefer_free = ['38', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
+	_available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
+	_available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
 	_video_extensions = {
 		'13': '3gp',
 		'17': 'mp4',
@@ -129,6 +135,7 @@ class YoutubeIE(InfoExtractor):
 		'43': 'webm',
 		'44': 'webm',
 		'45': 'webm',
+		'46': 'webm',
 	}
 	_video_dimensions = {
 		'5': '240x400',
@@ -144,9 +151,14 @@ class YoutubeIE(InfoExtractor):
 		'43': '360x640',
 		'44': '480x854',
 		'45': '720x1280',
+		'46': '1080x1920',
 	}	
 	IE_NAME = u'youtube'
 
+	def suitable(self, url):
+		"""Receives a URL and returns True if suitable for this IE."""
+		return re.match(self._VALID_URL, url, re.VERBOSE) is not None
+
 	def report_lang(self):
 		"""Report attempt to set language."""
 		self._downloader.to_screen(u'[youtube] Setting language')
@@ -193,17 +205,17 @@ class YoutubeIE(InfoExtractor):
 			end = start + float(dur)
 			start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000)
 			end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000)
-			caption = re.sub(ur'(?u)&(.+?);', htmlentity_transform, caption)
-			caption = re.sub(ur'(?u)&(.+?);', htmlentity_transform, caption) # double cycle, inentional
-			srt += str(n) + '\n'
+			caption = unescapeHTML(caption)
+			caption = unescapeHTML(caption) # double cycle, intentional
+			srt += str(n+1) + '\n'
 			srt += start + ' --> ' + end + '\n'
 			srt += caption + '\n\n'
 		return srt
 
 	def _print_formats(self, formats):
-		print 'Available formats:'
+		print('Available formats:')
 		for x in formats:
-			print '%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???'))
+			print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???')))
 
 	def _real_initialize(self):
 		if self._downloader is None:
@@ -226,7 +238,7 @@ class YoutubeIE(InfoExtractor):
 				else:
 					raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 			except (IOError, netrc.NetrcParseError), err:
-				self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
+				self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
 				return
 
 		# Set language
@@ -235,7 +247,7 @@ class YoutubeIE(InfoExtractor):
 			self.report_lang()
 			urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
+			self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err))
 			return
 
 		# No authentication to be performed
@@ -258,7 +270,7 @@ class YoutubeIE(InfoExtractor):
 				self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 				return
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
+			self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
 			return
 
 		# Confirm age
@@ -271,7 +283,7 @@ class YoutubeIE(InfoExtractor):
 			self.report_age_confirmation()
 			age_results = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
 			return
 
 	def _real_extract(self, url):
@@ -281,7 +293,7 @@ class YoutubeIE(InfoExtractor):
 			url = 'http://www.youtube.com/' + urllib.unquote(mobj.group(1)).lstrip('/')
 
 		# Extract video id from URL
-		mobj = re.match(self._VALID_URL, url)
+		mobj = re.match(self._VALID_URL, url, re.VERBOSE)
 		if mobj is None:
 			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 			return
@@ -293,7 +305,7 @@ class YoutubeIE(InfoExtractor):
 		try:
 			video_webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
 			return
 
 		# Attempt to extract SWF player URL
@@ -315,7 +327,7 @@ class YoutubeIE(InfoExtractor):
 				if 'token' in video_info:
 					break
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
+				self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
 				return
 		if 'token' not in video_info:
 			if 'reason' in video_info:
@@ -324,6 +336,11 @@ class YoutubeIE(InfoExtractor):
 				self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
 			return
 
+		# Check for "rental" videos
+		if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
+			self._downloader.trouble(u'ERROR: "rental" videos not supported')
+			return
+
 		# Start extracting information
 		self.report_information_extraction(video_id)
 
@@ -339,10 +356,6 @@ class YoutubeIE(InfoExtractor):
 			return
 		video_title = urllib.unquote_plus(video_info['title'][0])
 		video_title = video_title.decode('utf-8')
-		video_title = sanitize_title(video_title)
-
-		# simplified title
-		simple_title = simplify_title(video_title)
 
 		# thumbnail image
 		if 'thumbnail_url' not in video_info:
@@ -364,49 +377,48 @@ class YoutubeIE(InfoExtractor):
 					pass
 
 		# description
-		try:
-			lxml.etree
-		except NameError:
-			video_description = u'No description available.'
-			mobj = re.search(r'<meta name="description" content="(.*?)">', video_webpage)
-			if mobj is not None:
-				video_description = mobj.group(1).decode('utf-8')
-		else:
-			html_parser = lxml.etree.HTMLParser(encoding='utf-8')
-			vwebpage_doc = lxml.etree.parse(StringIO.StringIO(video_webpage), html_parser)
-			video_description = u''.join(vwebpage_doc.xpath('id("eow-description")//text()'))
-			# TODO use another parser
+		video_description = get_element_by_id("eow-description", video_webpage.decode('utf8'))
+		if video_description: video_description = clean_html(video_description)
+		else: video_description = ''
 			
 		# closed captions
 		video_subtitles = None
 		if self._downloader.params.get('writesubtitles', False):
-			self.report_video_subtitles_download(video_id)
-			request = urllib2.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
 			try:
-				srt_list = urllib2.urlopen(request).read()
-			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
-			else:
-				srt_lang_list = re.findall(r'lang_code="([\w\-]+)"', srt_list)
-				if srt_lang_list:
-					if self._downloader.params.get('subtitleslang', False):
-						srt_lang = self._downloader.params.get('subtitleslang')
-					elif 'en' in srt_lang_list:
-						srt_lang = 'en'
-					else:
-						srt_lang = srt_lang_list[0]
-					if not srt_lang in srt_lang_list:
-						self._downloader.trouble(u'WARNING: no closed captions found in the specified language')
-					else:
-						request = urllib2.Request('http://video.google.com/timedtext?hl=en&lang=%s&v=%s' % (srt_lang, video_id))
-						try:
-							srt_xml = urllib2.urlopen(request).read()
-						except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-							self._downloader.trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
-						else:
-							video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
+				self.report_video_subtitles_download(video_id)
+				request = urllib2.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
+				try:
+					srt_list = urllib2.urlopen(request).read()
+				except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+					raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
+				srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list)
+				srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list)
+				if not srt_lang_list:
+					raise Trouble(u'WARNING: video has no closed captions')
+				if self._downloader.params.get('subtitleslang', False):
+					srt_lang = self._downloader.params.get('subtitleslang')
+				elif 'en' in srt_lang_list:
+					srt_lang = 'en'
 				else:
-					self._downloader.trouble(u'WARNING: video has no closed captions')
+					srt_lang = srt_lang_list.keys()[0]
+				if not srt_lang in srt_lang_list:
+					raise Trouble(u'WARNING: no closed captions found in the specified language')
+				request = urllib2.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id))
+				try:
+					srt_xml = urllib2.urlopen(request).read()
+				except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+					raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
+				if not srt_xml:
+					raise Trouble(u'WARNING: unable to download video subtitles')
+				video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
+			except Trouble as trouble:
+				self._downloader.trouble(trouble[0])
+
+		if 'length_seconds' not in video_info:
+			self._downloader.trouble(u'WARNING: unable to extract video duration')
+			video_duration = ''
+		else:
+			video_duration = urllib.unquote_plus(video_info['length_seconds'][0])
 
 		# token
 		video_token = urllib.unquote_plus(video_info['token'][0])
@@ -421,7 +433,7 @@ class YoutubeIE(InfoExtractor):
 			url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
 			url_data = [parse_qs(uds) for uds in url_data_strs]
 			url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data)
-			url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data)
+			url_map = dict((ud['itag'][0], ud['url'][0] + '&signature=' + ud['sig'][0]) for ud in url_data)
 
 			format_limit = self._downloader.params.get('format_limit', None)
 			available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
@@ -469,13 +481,13 @@ class YoutubeIE(InfoExtractor):
 				'uploader':	video_uploader.decode('utf-8'),
 				'upload_date':	upload_date,
 				'title':	video_title,
-				'stitle':	simple_title,
 				'ext':		video_extension.decode('utf-8'),
 				'format':	(format_param is None and u'NA' or format_param.decode('utf-8')),
 				'thumbnail':	video_thumbnail.decode('utf-8'),
 				'description':	video_description,
 				'player_url':	player_url,
-				'subtitles':	video_subtitles
+				'subtitles':	video_subtitles,
+				'duration':		video_duration
 			})
 		return results
 
@@ -514,7 +526,7 @@ class MetacafeIE(InfoExtractor):
 			self.report_disclaimer()
 			disclaimer = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err))
 			return
 
 		# Confirm age
@@ -527,7 +539,7 @@ class MetacafeIE(InfoExtractor):
 			self.report_age_confirmation()
 			disclaimer = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
 			return
 
 	def _real_extract(self, url):
@@ -545,15 +557,13 @@ class MetacafeIE(InfoExtractor):
 			self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
 			return
 
-		simple_title = mobj.group(2).decode('utf-8')
-
 		# Retrieve video webpage to extract further information
 		request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
 		try:
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
 			return
 
 		# Extract URL, uploader and title from webpage
@@ -592,9 +602,8 @@ class MetacafeIE(InfoExtractor):
 			self._downloader.trouble(u'ERROR: unable to extract title')
 			return
 		video_title = mobj.group(1).decode('utf-8')
-		video_title = sanitize_title(video_title)
 
-		mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
+		mobj = re.search(r'submitter=(.*?);', webpage)
 		if mobj is None:
 			self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 			return
@@ -606,7 +615,6 @@ class MetacafeIE(InfoExtractor):
 			'uploader':	video_uploader.decode('utf-8'),
 			'upload_date':	u'NA',
 			'title':	video_title,
-			'stitle':	simple_title,
 			'ext':		video_extension.decode('utf-8'),
 			'format':	u'NA',
 			'player_url':	None,
@@ -616,7 +624,7 @@ class MetacafeIE(InfoExtractor):
 class DailymotionIE(InfoExtractor):
 	"""Information Extractor for Dailymotion"""
 
-	_VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
+	_VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)'
 	IE_NAME = u'dailymotion'
 
 	def __init__(self, downloader=None):
@@ -637,9 +645,9 @@ class DailymotionIE(InfoExtractor):
 			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 			return
 
-		video_id = mobj.group(1)
+		video_id = mobj.group(1).split('_')[0].split('?')[0]
 
-		video_extension = 'flv'
+		video_extension = 'mp4'
 
 		# Retrieve video webpage to extract further information
 		request = urllib2.Request(url)
@@ -648,47 +656,64 @@ class DailymotionIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
 			return
 
 		# Extract URL, uploader and title from webpage
 		self.report_extraction(video_id)
-		mobj = re.search(r'(?i)addVariable\(\"sequence\"\s*,\s*\"([^\"]+?)\"\)', webpage)
+		mobj = re.search(r'\s*var flashvars = (.*)', webpage)
 		if mobj is None:
 			self._downloader.trouble(u'ERROR: unable to extract media URL')
 			return
-		sequence = urllib.unquote(mobj.group(1))
-		mobj = re.search(r',\"sdURL\"\:\"([^\"]+?)\",', sequence)
+		flashvars = urllib.unquote(mobj.group(1))
+
+		for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']:
+			if key in flashvars:
+				max_quality = key
+				self._downloader.to_screen(u'[dailymotion] Using %s' % key)
+				break
+		else:
+			self._downloader.trouble(u'ERROR: unable to extract video URL')
+			return
+
+		mobj = re.search(r'"' + max_quality + r'":"(.+?)"', flashvars)
 		if mobj is None:
-			self._downloader.trouble(u'ERROR: unable to extract media URL')
+			self._downloader.trouble(u'ERROR: unable to extract video URL')
 			return
-		mediaURL = urllib.unquote(mobj.group(1)).replace('\\', '')
 
-		# if needed add http://www.dailymotion.com/ if relative URL
+		video_url = urllib.unquote(mobj.group(1)).replace('\\/', '/')
 
-		video_url = mediaURL
+		# TODO: support choosing qualities
 
 		mobj = re.search(r'<meta property="og:title" content="(?P<title>[^"]*)" />', webpage)
 		if mobj is None:
 			self._downloader.trouble(u'ERROR: unable to extract title')
 			return
 		video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
-		video_title = sanitize_title(video_title)
-		simple_title = simplify_title(video_title)
 
-		mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
+		video_uploader = u'NA'
+		mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>', webpage)
 		if mobj is None:
-			self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
-			return
-		video_uploader = mobj.group(1)
+			# lookin for official user
+			mobj_official = re.search(r'<span rel="author"[^>]+?>([^<]+?)</span>', webpage)
+			if mobj_official is None:
+				self._downloader.trouble(u'WARNING: unable to extract uploader nickname')
+			else:
+				video_uploader = mobj_official.group(1)
+		else:
+			video_uploader = mobj.group(1)
+
+		video_upload_date = u'NA'
+		mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
+		if mobj is not None:
+			video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
 
 		return [{
 			'id':		video_id.decode('utf-8'),
 			'url':		video_url.decode('utf-8'),
 			'uploader':	video_uploader.decode('utf-8'),
-			'upload_date':	u'NA',
+			'upload_date':	video_upload_date,
 			'title':	video_title,
-			'stitle':	simple_title,
 			'ext':		video_extension.decode('utf-8'),
 			'format':	u'NA',
 			'player_url':	None,
@@ -729,7 +754,7 @@ class GoogleIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 			return
 
 		# Extract URL, uploader, and title from webpage
@@ -752,8 +777,6 @@ class GoogleIE(InfoExtractor):
 			self._downloader.trouble(u'ERROR: unable to extract title')
 			return
 		video_title = mobj.group(1).decode('utf-8')
-		video_title = sanitize_title(video_title)
-		simple_title = simplify_title(video_title)
 
 		# Extract video description
 		mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
@@ -770,7 +793,7 @@ class GoogleIE(InfoExtractor):
 			try:
 				webpage = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+				self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 				return
 			mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
 			if mobj is None:
@@ -786,7 +809,6 @@ class GoogleIE(InfoExtractor):
 			'uploader':	u'NA',
 			'upload_date':	u'NA',
 			'title':	video_title,
-			'stitle':	simple_title,
 			'ext':		video_extension.decode('utf-8'),
 			'format':	u'NA',
 			'player_url':	None,
@@ -827,7 +849,7 @@ class PhotobucketIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 			return
 
 		# Extract URL, uploader, and title from webpage
@@ -845,8 +867,6 @@ class PhotobucketIE(InfoExtractor):
 			self._downloader.trouble(u'ERROR: unable to extract title')
 			return
 		video_title = mobj.group(1).decode('utf-8')
-		video_title = sanitize_title(video_title)
-		simple_title = simplify_title(video_title)
 
 		video_uploader = mobj.group(2).decode('utf-8')
 
@@ -856,7 +876,6 @@ class PhotobucketIE(InfoExtractor):
 			'uploader':	video_uploader,
 			'upload_date':	u'NA',
 			'title':	video_title,
-			'stitle':	simple_title,
 			'ext':		video_extension.decode('utf-8'),
 			'format':	u'NA',
 			'player_url':	None,
@@ -900,7 +919,7 @@ class YahooIE(InfoExtractor):
 			try:
 				webpage = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+				self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 				return
 
 			mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
@@ -924,7 +943,7 @@ class YahooIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 			return
 
 		# Extract uploader and title from webpage
@@ -934,7 +953,6 @@ class YahooIE(InfoExtractor):
 			self._downloader.trouble(u'ERROR: unable to extract video title')
 			return
 		video_title = mobj.group(1).decode('utf-8')
-		simple_title = simplify_title(video_title)
 
 		mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
 		if mobj is None:
@@ -983,7 +1001,7 @@ class YahooIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 			return
 
 		# Extract media URL from playlist XML
@@ -992,7 +1010,7 @@ class YahooIE(InfoExtractor):
 			self._downloader.trouble(u'ERROR: Unable to extract media URL')
 			return
 		video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
-		video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
+		video_url = unescapeHTML(video_url)
 
 		return [{
 			'id':		video_id.decode('utf-8'),
@@ -1000,7 +1018,6 @@ class YahooIE(InfoExtractor):
 			'uploader':	video_uploader,
 			'upload_date':	u'NA',
 			'title':	video_title,
-			'stitle':	simple_title,
 			'ext':		video_extension.decode('utf-8'),
 			'thumbnail':	video_thumbnail.decode('utf-8'),
 			'description':	video_description,
@@ -1013,7 +1030,7 @@ class VimeoIE(InfoExtractor):
 	"""Information extractor for vimeo.com."""
 
 	# _VALID_URL matches Vimeo URLs
-	_VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:groups/[^/]+/)?(?:videos?/)?([0-9]+)'
+	_VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:(?:groups|album)/[^/]+/)?(?:videos?/)?([0-9]+)'
 	IE_NAME = u'vimeo'
 
 	def __init__(self, downloader=None):
@@ -1042,7 +1059,7 @@ class VimeoIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 			return
 
 		# Now we begin extracting as much information as we can from what we
@@ -1060,7 +1077,6 @@ class VimeoIE(InfoExtractor):
 		
 		# Extract title
 		video_title = config["video"]["title"]
-		simple_title = simplify_title(video_title)
 
 		# Extract uploader
 		video_uploader = config["video"]["owner"]["name"]
@@ -1069,18 +1085,9 @@ class VimeoIE(InfoExtractor):
 		video_thumbnail = config["video"]["thumbnail"]
 
 		# Extract video description
-		try:
-			lxml.etree
-		except NameError:
-			video_description = u'No description available.'
-			mobj = re.search(r'<meta name="description" content="(.*?)" />', webpage, re.MULTILINE)
-			if mobj is not None:
-				video_description = mobj.group(1)
-		else:
-			html_parser = lxml.etree.HTMLParser()
-			vwebpage_doc = lxml.etree.parse(StringIO.StringIO(webpage), html_parser)
-			video_description = u''.join(vwebpage_doc.xpath('id("description")//text()')).strip()
-			# TODO use another parser
+		video_description = get_element_by_id("description", webpage.decode('utf8'))
+		if video_description: video_description = clean_html(video_description)
+		else: video_description = ''
 
 		# Extract upload date
 		video_upload_date = u'NA'
@@ -1093,21 +1100,32 @@ class VimeoIE(InfoExtractor):
 		timestamp = config['request']['timestamp']
 
 		# Vimeo specific: extract video codec and quality information
+		# First consider quality, then codecs, then take everything
 		# TODO bind to format param
 		codecs = [('h264', 'mp4'), ('vp8', 'flv'), ('vp6', 'flv')]
-		for codec in codecs:
-			if codec[0] in config["video"]["files"]:
-				video_codec = codec[0]
-				video_extension = codec[1]
-				if 'hd' in config["video"]["files"][codec[0]]: quality = 'hd'
-				else: quality = 'sd'
+		files = { 'hd': [], 'sd': [], 'other': []}
+		for codec_name, codec_extension in codecs:
+			if codec_name in config["video"]["files"]:
+				if 'hd' in config["video"]["files"][codec_name]:
+					files['hd'].append((codec_name, codec_extension, 'hd'))
+				elif 'sd' in config["video"]["files"][codec_name]:
+					files['sd'].append((codec_name, codec_extension, 'sd'))
+				else:
+					files['other'].append((codec_name, codec_extension, config["video"]["files"][codec_name][0]))
+
+		for quality in ('hd', 'sd', 'other'):
+			if len(files[quality]) > 0:
+				video_quality = files[quality][0][2]
+				video_codec = files[quality][0][0]
+				video_extension = files[quality][0][1]
+				self._downloader.to_screen(u'[vimeo] %s: Downloading %s file at %s quality' % (video_id, video_codec.upper(), video_quality))
 				break
 		else:
 			self._downloader.trouble(u'ERROR: no known codec found')
 			return
 
 		video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
-					%(video_id, sig, timestamp, quality, video_codec.upper())
+					%(video_id, sig, timestamp, video_quality, video_codec.upper())
 
 		return [{
 			'id':		video_id,
@@ -1115,7 +1133,6 @@ class VimeoIE(InfoExtractor):
 			'uploader':	video_uploader,
 			'upload_date':	video_upload_date,
 			'title':	video_title,
-			'stitle':	simple_title,
 			'ext':		video_extension,
 			'thumbnail':	video_thumbnail,
 			'description':	video_description,
@@ -1123,6 +1140,143 @@ class VimeoIE(InfoExtractor):
 		}]
 
 
+class ArteTvIE(InfoExtractor):
+	"""arte.tv information extractor."""
+
+	_VALID_URL = r'(?:http://)?videos\.arte\.tv/(?:fr|de)/videos/.*'
+	_LIVE_URL = r'index-[0-9]+\.html$'
+
+	IE_NAME = u'arte.tv'
+
+	def __init__(self, downloader=None):
+		InfoExtractor.__init__(self, downloader)
+
+	def report_download_webpage(self, video_id):
+		"""Report webpage download."""
+		self._downloader.to_screen(u'[arte.tv] %s: Downloading webpage' % video_id)
+
+	def report_extraction(self, video_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[arte.tv] %s: Extracting information' % video_id)
+
+	def fetch_webpage(self, url):
+		self._downloader.increment_downloads()
+		request = urllib2.Request(url)
+		try:
+			self.report_download_webpage(url)
+			webpage = urllib2.urlopen(request).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+			return
+		except ValueError, err:
+			self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
+			return
+		return webpage
+
+	def grep_webpage(self, url, regex, regexFlags, matchTuples):
+		page = self.fetch_webpage(url)
+		mobj = re.search(regex, page, regexFlags)
+		info = {}
+
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
+			return
+
+		for (i, key, err) in matchTuples:
+			if mobj.group(i) is None:
+				self._downloader.trouble(err)
+				return
+			else:
+				info[key] = mobj.group(i)
+
+		return info
+
+	def extractLiveStream(self, url):
+		video_lang = url.split('/')[-4]
+		info = self.grep_webpage(
+			url,
+			r'src="(.*?/videothek_js.*?\.js)',
+			0,
+			[
+				(1, 'url', u'ERROR: Invalid URL: %s' % url)
+			]
+		)
+		http_host = url.split('/')[2]
+		next_url = 'http://%s%s' % (http_host, urllib.unquote(info.get('url')))
+		info = self.grep_webpage(
+			next_url,
+			r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' +
+				'(http://.*?\.swf).*?' +
+				'(rtmp://.*?)\'',
+			re.DOTALL,
+			[
+				(1, 'path',   u'ERROR: could not extract video path: %s' % url),
+				(2, 'player', u'ERROR: could not extract video player: %s' % url),
+				(3, 'url',    u'ERROR: could not extract video url: %s' % url)
+			]
+		)
+		video_url = u'%s/%s' % (info.get('url'), info.get('path'))
+
+	def extractPlus7Stream(self, url):
+		video_lang = url.split('/')[-3]
+		info = self.grep_webpage(
+			url,
+			r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)',
+			0,
+			[
+				(1, 'url', u'ERROR: Invalid URL: %s' % url)
+			]
+		)
+		next_url = urllib.unquote(info.get('url'))
+		info = self.grep_webpage(
+			next_url,
+			r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang,
+			0,
+			[
+				(1, 'url', u'ERROR: Could not find <video> tag: %s' % url)
+			]
+		)
+		next_url = urllib.unquote(info.get('url'))
+
+		info = self.grep_webpage(
+			next_url,
+			r'<video id="(.*?)".*?>.*?' +
+				'<name>(.*?)</name>.*?' +
+				'<dateVideo>(.*?)</dateVideo>.*?' +
+				'<url quality="hd">(.*?)</url>',
+			re.DOTALL,
+			[
+				(1, 'id',    u'ERROR: could not extract video id: %s' % url),
+				(2, 'title', u'ERROR: could not extract video title: %s' % url),
+				(3, 'date',  u'ERROR: could not extract video date: %s' % url),
+				(4, 'url',   u'ERROR: could not extract video url: %s' % url)
+			]
+		)
+
+		return {
+			'id':           info.get('id'),
+			'url':          urllib.unquote(info.get('url')),
+			'uploader':     u'arte.tv',
+			'upload_date':  info.get('date'),
+			'title':        info.get('title'),
+			'ext':          u'mp4',
+			'format':       u'NA',
+			'player_url':   None,
+		}
+
+	def _real_extract(self, url):
+		video_id = url.split('/')[-1]
+		self.report_extraction(video_id)
+
+		if re.search(self._LIVE_URL, video_id) is not None:
+			self.extractLiveStream(url)
+			return
+		else:
+			info = self.extractPlus7Stream(url)
+
+		return [info]
+
+
 class GenericIE(InfoExtractor):
 	"""Generic last-resort information extractor."""
 
@@ -1208,7 +1362,7 @@ class GenericIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 			return
 		except ValueError, err:
 			# since this is the last-resort InfoExtractor, if
@@ -1250,8 +1404,6 @@ class GenericIE(InfoExtractor):
 			self._downloader.trouble(u'ERROR: unable to extract title')
 			return
 		video_title = mobj.group(1).decode('utf-8')
-		video_title = sanitize_title(video_title)
-		simple_title = simplify_title(video_title)
 
 		# video uploader is domain name
 		mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
@@ -1266,7 +1418,6 @@ class GenericIE(InfoExtractor):
 			'uploader':	video_uploader,
 			'upload_date':	u'NA',
 			'title':	video_title,
-			'stitle':	simple_title,
 			'ext':		video_extension.decode('utf-8'),
 			'format':	u'NA',
 			'player_url':	None,
@@ -1284,7 +1435,7 @@ class YoutubeSearchIE(InfoExtractor):
 		InfoExtractor.__init__(self, downloader)
 
 	def report_download_page(self, query, pagenum):
-		"""Report attempt to download playlist page with given number."""
+		"""Report attempt to download search page with given number."""
 		query = query.decode(preferredencoding())
 		self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
 
@@ -1332,7 +1483,7 @@ class YoutubeSearchIE(InfoExtractor):
 			try:
 				data = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download API page: %s' % str(err))
+				self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err))
 				return
 			api_response = json.loads(data)['data']
 
@@ -1409,7 +1560,7 @@ class GoogleSearchIE(InfoExtractor):
 			try:
 				page = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
+				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
 				return
 
 			# Extract video identifiers
@@ -1492,7 +1643,7 @@ class YahooSearchIE(InfoExtractor):
 			try:
 				page = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
+				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
 				return
 
 			# Extract video identifiers
@@ -1518,10 +1669,10 @@ class YahooSearchIE(InfoExtractor):
 class YoutubePlaylistIE(InfoExtractor):
 	"""Information Extractor for YouTube playlists."""
 
-	_VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
+	_VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
 	_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
-	_VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&amp;list=PL%s&'
-	_MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
+	_VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&amp;([^&"]+&amp;)*list=.*?%s'
+	_MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
 	IE_NAME = u'youtube:playlist'
 
 	def __init__(self, downloader=None):
@@ -1562,7 +1713,7 @@ class YoutubePlaylistIE(InfoExtractor):
 			try:
 				page = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
+				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
 				return
 
 			# Extract video identifiers
@@ -1588,6 +1739,56 @@ class YoutubePlaylistIE(InfoExtractor):
 		return
 
 
+class YoutubeChannelIE(InfoExtractor):
+	"""Information Extractor for YouTube channels."""
+
+	_VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$"
+	_TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
+	_MORE_PAGES_INDICATOR = r'yt-uix-button-content">Next' # TODO
+	IE_NAME = u'youtube:channel'
+
+	def report_download_page(self, channel_id, pagenum):
+		"""Report attempt to download channel page with given number."""
+		self._downloader.to_screen(u'[youtube] Channel %s: Downloading page #%s' % (channel_id, pagenum))
+
+	def _real_extract(self, url):
+		# Extract channel id
+		mobj = re.match(self._VALID_URL, url)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: invalid url: %s' % url)
+			return
+
+		# Download channel pages
+		channel_id = mobj.group(1)
+		video_ids = []
+		pagenum = 1
+
+		while True:
+			self.report_download_page(channel_id, pagenum)
+			url = self._TEMPLATE_URL % (channel_id, pagenum)
+			request = urllib2.Request(url)
+			try:
+				page = urllib2.urlopen(request).read()
+			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+				return
+
+			# Extract video identifiers
+			ids_in_page = []
+			for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&', page):
+				if mobj.group(1) not in ids_in_page:
+					ids_in_page.append(mobj.group(1))
+			video_ids.extend(ids_in_page)
+
+			if re.search(self._MORE_PAGES_INDICATOR, page) is None:
+				break
+			pagenum = pagenum + 1
+
+		for id in video_ids:
+			self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
+		return
+
+
 class YoutubeUserIE(InfoExtractor):
 	"""Information Extractor for YouTube users."""
 
@@ -1632,7 +1833,7 @@ class YoutubeUserIE(InfoExtractor):
 			try:
 				page = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
+				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
 				return
 
 			# Extract video identifiers
@@ -1671,6 +1872,98 @@ class YoutubeUserIE(InfoExtractor):
 			self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id])
 
 
+class BlipTVUserIE(InfoExtractor):
+	"""Information Extractor for blip.tv users."""
+
+	_VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$'
+	_PAGE_SIZE = 12
+	IE_NAME = u'blip.tv:user'
+
+	def __init__(self, downloader=None):
+		InfoExtractor.__init__(self, downloader)
+
+	def report_download_page(self, username, pagenum):
+		"""Report attempt to download user page."""
+		self._downloader.to_screen(u'[%s] user %s: Downloading video ids from page %d' %
+				(self.IE_NAME, username, pagenum))
+
+	def _real_extract(self, url):
+		# Extract username
+		mobj = re.match(self._VALID_URL, url)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: invalid url: %s' % url)
+			return
+
+		username = mobj.group(1)
+
+		page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
+
+		request = urllib2.Request(url)
+
+		try:
+			page = urllib2.urlopen(request).read().decode('utf-8')
+			mobj = re.search(r'data-users-id="([^"]+)"', page)
+			page_base = page_base % mobj.group(1)
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+			return
+
+
+		# Download video ids using BlipTV Ajax calls. Result size per
+		# query is limited (currently to 12 videos) so we need to query
+		# page by page until there are no video ids - it means we got
+		# all of them.
+
+		video_ids = []
+		pagenum = 1
+
+		while True:
+			self.report_download_page(username, pagenum)
+
+			request = urllib2.Request( page_base + "&page=" + str(pagenum) )
+
+			try:
+				page = urllib2.urlopen(request).read().decode('utf-8')
+			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
+				return
+
+			# Extract video identifiers
+			ids_in_page = []
+
+			for mobj in re.finditer(r'href="/([^"]+)"', page):
+				if mobj.group(1) not in ids_in_page:
+					ids_in_page.append(unescapeHTML(mobj.group(1)))
+
+			video_ids.extend(ids_in_page)
+
+			# A little optimization - if current page is not
+			# "full", ie. does not contain PAGE_SIZE video ids then
+			# we can assume that this page is the last one - there
+			# are no more ids on further pages - no need to query
+			# again.
+
+			if len(ids_in_page) < self._PAGE_SIZE:
+				break
+
+			pagenum += 1
+
+		all_ids_count = len(video_ids)
+		playliststart = self._downloader.params.get('playliststart', 1) - 1
+		playlistend = self._downloader.params.get('playlistend', -1)
+
+		if playlistend == -1:
+			video_ids = video_ids[playliststart:]
+		else:
+			video_ids = video_ids[playliststart:playlistend]
+
+		self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" %
+				(self.IE_NAME, username, all_ids_count, len(video_ids)))
+
+		for video_id in video_ids:
+			self._downloader.download([u'http://blip.tv/'+video_id])
+
+
 class DepositFilesIE(InfoExtractor):
 	"""Information extractor for depositfiles.com"""
 
@@ -1700,7 +1993,7 @@ class DepositFilesIE(InfoExtractor):
 			self.report_download_webpage(file_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err))
 			return
 
 		# Search for the real file URL
@@ -1731,7 +2024,6 @@ class DepositFilesIE(InfoExtractor):
 			'uploader':	u'NA',
 			'upload_date':	u'NA',
 			'title':	file_title,
-			'stitle':	file_title,
 			'ext':		file_extension.decode('utf-8'),
 			'format':	u'NA',
 			'player_url':	None,
@@ -1818,7 +2110,7 @@ class FacebookIE(InfoExtractor):
 				else:
 					raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 			except (IOError, netrc.NetrcParseError), err:
-				self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
+				self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
 				return
 
 		if useremail is None:
@@ -1838,7 +2130,7 @@ class FacebookIE(InfoExtractor):
 				self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
 				return
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
+			self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
 			return
 
 	def _real_extract(self, url):
@@ -1855,7 +2147,7 @@ class FacebookIE(InfoExtractor):
 			page = urllib2.urlopen(request)
 			video_webpage = page.read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
 			return
 
 		# Start extracting information
@@ -1876,9 +2168,6 @@ class FacebookIE(InfoExtractor):
 			return
 		video_title = video_info['title']
 		video_title = video_title.decode('utf-8')
-		video_title = sanitize_title(video_title)
-
-		simple_title = simplify_title(video_title)
 
 		# thumbnail image
 		if 'thumbnail' not in video_info:
@@ -1939,7 +2228,6 @@ class FacebookIE(InfoExtractor):
 				'uploader':	video_uploader.decode('utf-8'),
 				'upload_date':	upload_date,
 				'title':	video_title,
-				'stitle':	simple_title,
 				'ext':		video_extension.decode('utf-8'),
 				'format':	(format_param is None and u'NA' or format_param.decode('utf-8')),
 				'thumbnail':	video_thumbnail.decode('utf-8'),
@@ -1974,7 +2262,7 @@ class BlipTVIE(InfoExtractor):
 		else:
 			cchar = '?'
 		json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
-		request = urllib2.Request(json_url)
+		request = urllib2.Request(json_url.encode('utf-8'))
 		self.report_extraction(mobj.group(1))
 		info = None
 		try:
@@ -1989,18 +2277,17 @@ class BlipTVIE(InfoExtractor):
 					'id': title,
 					'url': url,
 					'title': title,
-					'stitle': simplify_title(title),
 					'ext': ext,
 					'urlhandle': urlh
 				}
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
 			return
 		if info is None: # Regular URL
 			try:
 				json_code = urlh.read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % str(err))
+				self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err))
 				return
 
 			try:
@@ -2009,21 +2296,20 @@ class BlipTVIE(InfoExtractor):
 					data = json_data['Post']
 				else:
 					data = json_data
-	
+
 				upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
 				video_url = data['media']['url']
 				umobj = re.match(self._URL_EXT, video_url)
 				if umobj is None:
 					raise ValueError('Can not determine filename extension')
 				ext = umobj.group(1)
-	
+
 				info = {
 					'id': data['item_id'],
 					'url': video_url,
 					'uploader': data['display_name'],
 					'upload_date': upload_date,
 					'title': data['title'],
-					'stitle': simplify_title(data['title']),
 					'ext': ext,
 					'format': data['media']['mimeType'],
 					'thumbnail': data['thumbnailUrl'],
@@ -2034,6 +2320,7 @@ class BlipTVIE(InfoExtractor):
 				self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
 				return
 
+		std_headers['User-Agent'] = 'iTunes/10.6.1'
 		return [info]
 
 
@@ -2068,7 +2355,7 @@ class MyVideoIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
 			return
 
 		self.report_extraction(video_id)
@@ -2085,9 +2372,6 @@ class MyVideoIE(InfoExtractor):
 			return
 
 		video_title = mobj.group(1)
-		video_title = sanitize_title(video_title)
-
-		simple_title = simplify_title(video_title)
 
 		return [{
 			'id':		video_id,
@@ -2095,7 +2379,6 @@ class MyVideoIE(InfoExtractor):
 			'uploader':	u'NA',
 			'upload_date':  u'NA',
 			'title':	video_title,
-			'stitle':	simple_title,
 			'ext':		u'flv',
 			'format':	u'NA',
 			'player_url':	None,
@@ -2107,9 +2390,28 @@ class ComedyCentralIE(InfoExtractor):
 	_VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
 	IE_NAME = u'comedycentral'
 
+	_available_formats = ['3500', '2200', '1700', '1200', '750', '400']
+
+	_video_extensions = {
+		'3500': 'mp4',
+		'2200': 'mp4',
+		'1700': 'mp4',
+		'1200': 'mp4',
+		'750': 'mp4',
+		'400': 'mp4',
+	}
+	_video_dimensions = {
+		'3500': '1280x720',
+		'2200': '960x540',
+		'1700': '768x432',
+		'1200': '640x360',
+		'750': '512x288',
+		'400': '384x216',
+	}
+
 	def report_extraction(self, episode_id):
 		self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
-	
+
 	def report_config_download(self, episode_id):
 		self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
 
@@ -2119,6 +2421,13 @@ class ComedyCentralIE(InfoExtractor):
 	def report_player_url(self, episode_id):
 		self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
 
+
+	def _print_formats(self, formats):
+		print('Available formats:')
+		for x in formats:
+			print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
+
+
 	def _real_extract(self, url):
 		mobj = re.match(self._VALID_URL, url)
 		if mobj is None:
@@ -2159,10 +2468,19 @@ class ComedyCentralIE(InfoExtractor):
 			epTitle = mobj.group('episode')
 
 		mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"', html)
+
 		if len(mMovieParams) == 0:
-			self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
-			return
+			# The Colbert Report embeds the information in a without
+			# a URL prefix; so extract the alternate reference
+			# and then add the URL prefix manually.
 
+			altMovieParams = re.findall('data-mgid="([^"]*episode.*?:.*?)"', html)
+			if len(altMovieParams) == 0:
+				self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
+				return
+			else:
+				mMovieParams = [("http://media.mtvnservices.com/" + altMovieParams[0], altMovieParams[0])]
+		
 		playerUrl_raw = mMovieParams[0][0]
 		self.report_player_url(epTitle)
 		try:
@@ -2211,10 +2529,31 @@ class ComedyCentralIE(InfoExtractor):
 			if len(turls) == 0:
 				self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
 				continue
+			
+			if self._downloader.params.get('listformats', None):
+				self._print_formats([i[0] for i in turls])
+				return
 
 			# For now, just pick the highest bitrate
 			format,video_url = turls[-1]
 
+			# Get the format arg from the arg stream
+			req_format = self._downloader.params.get('format', None)
+
+			# Select format if we can find one
+			for f,v in turls:
+				if f == req_format:
+					format, video_url = f, v
+					break
+
+			# Patch to download from alternative CDN, which does not
+			# break on current RTMPDump builds
+			broken_cdn = "rtmpe://viacomccstrmfs.fplive.net/viacomccstrm/gsp.comedystor/"
+			better_cdn = "rtmpe://cp10740.edgefcs.net/ondemand/mtvnorigin/gsp.comedystor/"
+
+			if video_url.startswith(broken_cdn):
+				video_url = video_url.replace(broken_cdn, better_cdn)
+
 			effTitle = showId + u'-' + epTitle
 			info = {
 				'id': shortMediaId,
@@ -2222,12 +2561,11 @@ class ComedyCentralIE(InfoExtractor):
 				'uploader': showId,
 				'upload_date': officialDate,
 				'title': effTitle,
-				'stitle': simplify_title(effTitle),
 				'ext': 'mp4',
 				'format': format,
 				'thumbnail': None,
 				'description': officialTitle,
-				'player_url': playerUrl
+				'player_url': None #playerUrl
 			}
 
 			results.append(info)
@@ -2248,8 +2586,6 @@ class EscapistIE(InfoExtractor):
 		self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
 
 	def _real_extract(self, url):
-		htmlParser = HTMLParser.HTMLParser()
-
 		mobj = re.match(self._VALID_URL, url)
 		if mobj is None:
 			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
@@ -2259,17 +2595,20 @@ class EscapistIE(InfoExtractor):
 
 		self.report_extraction(showName)
 		try:
-			webPage = urllib2.urlopen(url).read()
+			webPage = urllib2.urlopen(url)
+			webPageBytes = webPage.read()
+			m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
+			webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 			self._downloader.trouble(u'ERROR: unable to download webpage: ' + unicode(err))
 			return
 
 		descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
-		description = htmlParser.unescape(descMatch.group(1))
+		description = unescapeHTML(descMatch.group(1))
 		imgMatch = re.search('<meta property="og:image" content="([^"]*)"', webPage)
-		imgUrl = htmlParser.unescape(imgMatch.group(1))
+		imgUrl = unescapeHTML(imgMatch.group(1))
 		playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage)
-		playerUrl = htmlParser.unescape(playerUrlMatch.group(1))
+		playerUrl = unescapeHTML(playerUrlMatch.group(1))
 		configUrlMatch = re.search('config=(.*)$', playerUrl)
 		configUrl = urllib2.unquote(configUrlMatch.group(1))
 
@@ -2298,7 +2637,6 @@ class EscapistIE(InfoExtractor):
 			'uploader': showName,
 			'upload_date': None,
 			'title': showName,
-			'stitle': simplify_title(showName),
 			'ext': 'flv',
 			'format': 'flv',
 			'thumbnail': imgUrl,
@@ -2324,8 +2662,6 @@ class CollegeHumorIE(InfoExtractor):
 		self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
 
 	def _real_extract(self, url):
-		htmlParser = HTMLParser.HTMLParser()
-
 		mobj = re.match(self._VALID_URL, url)
 		if mobj is None:
 			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
@@ -2337,7 +2673,7 @@ class CollegeHumorIE(InfoExtractor):
 		try:
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
 			return
 
 		m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
@@ -2356,7 +2692,7 @@ class CollegeHumorIE(InfoExtractor):
 		try:
 			metaXml = urllib2.urlopen(xmlUrl).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
 			return
 
 		mdoc = xml.etree.ElementTree.fromstring(metaXml)
@@ -2364,7 +2700,6 @@ class CollegeHumorIE(InfoExtractor):
 			videoNode = mdoc.findall('./video')[0]
 			info['description'] = videoNode.findall('./description')[0].text
 			info['title'] = videoNode.findall('./caption')[0].text
-			info['stitle'] = simplify_title(info['title'])
 			info['url'] = videoNode.findall('./file')[0].text
 			info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
 			info['ext'] = info['url'].rpartition('.')[2]
@@ -2391,8 +2726,6 @@ class XVideosIE(InfoExtractor):
 		self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
 
 	def _real_extract(self, url):
-		htmlParser = HTMLParser.HTMLParser()
-
 		mobj = re.match(self._VALID_URL, url)
 		if mobj is None:
 			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
@@ -2405,7 +2738,7 @@ class XVideosIE(InfoExtractor):
 		try:
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
 			return
 
 		self.report_extraction(video_id)
@@ -2428,11 +2761,11 @@ class XVideosIE(InfoExtractor):
 
 
 		# Extract video thumbnail
-		mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]/[a-fA-F0-9]/[a-fA-F0-9]/([a-fA-F0-9.]+jpg)', webpage)
+		mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/([a-fA-F0-9.]+jpg)', webpage)
 		if mobj is None:
 			self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
 			return
-		video_thumbnail = mobj.group(1).decode('utf-8')
+		video_thumbnail = mobj.group(0).decode('utf-8')
 
 		info = {
 			'id': video_id,
@@ -2440,7 +2773,6 @@ class XVideosIE(InfoExtractor):
 			'uploader': None,
 			'upload_date': None,
 			'title': video_title,
-			'stitle': simplify_title(video_title),
 			'ext': 'flv',
 			'format': 'flv',
 			'thumbnail': video_thumbnail,
@@ -2475,8 +2807,6 @@ class SoundcloudIE(InfoExtractor):
 		self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
 
 	def _real_extract(self, url):
-		htmlParser = HTMLParser.HTMLParser()
-
 		mobj = re.match(self._VALID_URL, url)
 		if mobj is None:
 			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
@@ -2486,7 +2816,7 @@ class SoundcloudIE(InfoExtractor):
 		uploader = mobj.group(1).decode('utf-8')
 		# extract simple title (uploader + slug of song title)
 		slug_title =  mobj.group(2).decode('utf-8')
-		simple_title = uploader + '-' + slug_title
+		simple_title = uploader + u'-' + slug_title
 
 		self.report_webpage('%s/%s' % (uploader, slug_title))
 
@@ -2494,7 +2824,7 @@ class SoundcloudIE(InfoExtractor):
 		try:
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
 			return
 
 		self.report_extraction('%s/%s' % (uploader, slug_title))
@@ -2508,7 +2838,9 @@ class SoundcloudIE(InfoExtractor):
 		# extract unsimplified title
 		mobj = re.search('"title":"(.*?)",', webpage)
 		if mobj:
-			title = mobj.group(1)
+			title = mobj.group(1).decode('utf-8')
+		else:
+			title = simple_title
 
 		# construct media url (with uid/token)
 		mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
@@ -2519,7 +2851,7 @@ class SoundcloudIE(InfoExtractor):
 		mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
 		if mobj:
 			description = mobj.group(1)
-		
+
 		# upload date
 		upload_date = None
 		mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
@@ -2527,7 +2859,7 @@ class SoundcloudIE(InfoExtractor):
 			try:
 				upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
 			except Exception, e:
-				print str(e)
+				self._downloader.to_stderr(compat_str(e))
 
 		# for soundcloud, a request to a cross domain is required for cookies
 		request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
@@ -2537,8 +2869,7 @@ class SoundcloudIE(InfoExtractor):
 			'url':		mediaURL,
 			'uploader':	uploader.decode('utf-8'),
 			'upload_date':  upload_date,
-			'title':	simple_title.decode('utf-8'),
-			'stitle':	simple_title.decode('utf-8'),
+			'title':	title,
 			'ext':		u'mp3',
 			'format':	u'NA',
 			'player_url':	None,
@@ -2561,8 +2892,6 @@ class InfoQIE(InfoExtractor):
 		self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
 
 	def _real_extract(self, url):
-		htmlParser = HTMLParser.HTMLParser()
-
 		mobj = re.match(self._VALID_URL, url)
 		if mobj is None:
 			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
@@ -2574,7 +2903,7 @@ class InfoQIE(InfoExtractor):
 		try:
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
 			return
 
 		self.report_extraction(url)
@@ -2610,7 +2939,6 @@ class InfoQIE(InfoExtractor):
 			'uploader': None,
 			'upload_date': None,
 			'title': video_title,
-			'stitle': simplify_title(video_title),
 			'ext': extension,
 			'format': extension, # Extension is always(?) mp4, but seems to be flv
 			'thumbnail': None,
@@ -2647,7 +2975,6 @@ class MixcloudIE(InfoExtractor):
 			url_list = jsonData[fmt][bitrate]
 		except TypeError: # we have no bitrate info.
 			url_list = jsonData[fmt]
-				
 		return url_list
 
 	def check_urls(self, url_list):
@@ -2662,15 +2989,15 @@ class MixcloudIE(InfoExtractor):
 		return None
 
 	def _print_formats(self, formats):
-		print 'Available formats:'
+		print('Available formats:')
 		for fmt in formats.keys():
 			for b in formats[fmt]:
 				try:
 					ext = formats[fmt][b][0]
-					print '%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1])
+					print('%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1]))
 				except TypeError: # we have no bitrate info
 					ext = formats[fmt][0]
-					print '%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1])
+					print('%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1]))
 					break
 
 	def _real_extract(self, url):
@@ -2690,7 +3017,7 @@ class MixcloudIE(InfoExtractor):
 			self.report_download_json(file_url)
 			jsonData = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err))
 			return
 
 		# parse JSON
@@ -2727,7 +3054,6 @@ class MixcloudIE(InfoExtractor):
 			'uploader':	uploader.decode('utf-8'),
 			'upload_date': u'NA',
 			'title': json_data['name'],
-			'stitle': simplify_title(json_data['name']),
 			'ext': file_url.split('.')[-1].decode('utf-8'),
 			'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
 			'thumbnail': json_data['thumbnail_url'],
@@ -2759,9 +3085,9 @@ class StanfordOpenClassroomIE(InfoExtractor):
 			course = mobj.group('course')
 			video = mobj.group('video')
 			info = {
-				'id': simplify_title(course + '_' + video),
+				'id': course + '_' + video,
 			}
-	
+
 			self.report_extraction(info['id'])
 			baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
 			xmlUrl = baseUrl + video + '.xml'
@@ -2777,16 +3103,13 @@ class StanfordOpenClassroomIE(InfoExtractor):
 			except IndexError:
 				self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
 				return
-			info['stitle'] = simplify_title(info['title'])
 			info['ext'] = info['url'].rpartition('.')[2]
 			info['format'] = info['ext']
 			return [info]
 		elif mobj.group('course'): # A course page
-			unescapeHTML = HTMLParser.HTMLParser().unescape
-
 			course = mobj.group('course')
 			info = {
-				'id': simplify_title(course),
+				'id': course,
 				'type': 'playlist',
 			}
 
@@ -2802,7 +3125,6 @@ class StanfordOpenClassroomIE(InfoExtractor):
 				info['title'] = unescapeHTML(m.group(1))
 			else:
 				info['title'] = info['id']
-			info['stitle'] = simplify_title(info['title'])
 
 			m = re.search('<description>([^<]+)</description>', coursepage)
 			if m:
@@ -2822,8 +3144,6 @@ class StanfordOpenClassroomIE(InfoExtractor):
 			return results
 			
 		else: # Root page
-			unescapeHTML = HTMLParser.HTMLParser().unescape
-
 			info = {
 				'id': 'Stanford OpenClassroom',
 				'type': 'playlist',
@@ -2838,7 +3158,6 @@ class StanfordOpenClassroomIE(InfoExtractor):
 				return
 
 			info['title'] = info['id']
-			info['stitle'] = simplify_title(info['title'])
 
 			links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
 			info['list'] = [
@@ -2882,7 +3201,7 @@ class MTVIE(InfoExtractor):
 		try:
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
 			return
 
 		mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
@@ -2915,7 +3234,7 @@ class MTVIE(InfoExtractor):
 		try:
 			metadataXml = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % str(err))
+			self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err))
 			return
 
 		mdoc = xml.etree.ElementTree.fromstring(metadataXml)
@@ -2937,9 +3256,319 @@ class MTVIE(InfoExtractor):
 			'url': video_url,
 			'uploader': performer,
 			'title': video_title,
-			'stitle': simplify_title(video_title),
 			'ext': ext,
 			'format': format,
 		}
 
 		return [info]
+
+
+class YoukuIE(InfoExtractor):
+
+	_VALID_URL =  r'(?:http://)?v\.youku\.com/v_show/id_(?P<ID>[A-Za-z0-9]+)\.html'
+	IE_NAME = u'Youku'
+
+	def __init__(self, downloader=None):
+		InfoExtractor.__init__(self, downloader)
+
+	def report_download_webpage(self, file_id):
+		"""Report webpage download."""
+		self._downloader.to_screen(u'[Youku] %s: Downloading webpage' % file_id)
+
+	def report_extraction(self, file_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[Youku] %s: Extracting information' % file_id)
+
+	def _gen_sid(self):
+		nowTime = int(time.time() * 1000)
+		random1 = random.randint(1000,1998)
+		random2 = random.randint(1000,9999)
+
+		return "%d%d%d" %(nowTime,random1,random2)
+
+	def _get_file_ID_mix_string(self, seed):
+		mixed = []
+		source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
+		seed = float(seed)
+		for i in range(len(source)):
+			seed  =  (seed * 211 + 30031 ) % 65536
+			index  =  math.floor(seed / 65536 * len(source) )
+			mixed.append(source[int(index)])
+			source.remove(source[int(index)])
+		#return ''.join(mixed)
+		return mixed
+
+	def _get_file_id(self, fileId, seed):
+		mixed = self._get_file_ID_mix_string(seed)
+		ids = fileId.split('*')
+		realId = []
+		for ch in ids:
+			if ch:
+				realId.append(mixed[int(ch)])
+		return ''.join(realId)
+
+	def _real_extract(self, url):
+		mobj = re.match(self._VALID_URL, url)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+			return
+		video_id = mobj.group('ID')
+
+		info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
+
+		request = urllib2.Request(info_url, None, std_headers)
+		try:
+			self.report_download_webpage(video_id)
+			jsondata = urllib2.urlopen(request).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+			return
+
+		self.report_extraction(video_id)
+		try:
+			config = json.loads(jsondata)
+
+			video_title =  config['data'][0]['title']
+			seed = config['data'][0]['seed']
+
+			format = self._downloader.params.get('format', None)
+			supported_format = config['data'][0]['streamfileids'].keys()
+
+			if format is None or format == 'best':
+				if 'hd2' in supported_format:
+					format = 'hd2'
+				else:
+					format = 'flv'
+				ext = u'flv'
+			elif format == 'worst':
+				format = 'mp4'
+				ext = u'mp4'
+			else:
+				format = 'flv'
+				ext = u'flv'
+
+
+			fileid = config['data'][0]['streamfileids'][format]
+			seg_number = len(config['data'][0]['segs'][format])
+
+			keys=[]
+			for i in xrange(seg_number):
+				keys.append(config['data'][0]['segs'][format][i]['k'])
+
+			#TODO check error
+			#youku only could be viewed from mainland china
+		except:
+			self._downloader.trouble(u'ERROR: unable to extract info section')
+			return
+
+		files_info=[]
+		sid = self._gen_sid()
+		fileid = self._get_file_id(fileid, seed)
+
+		#column 8,9 of fileid represent the segment number
+		#fileid[7:9] should be changed
+		for index, key in enumerate(keys):
+
+			temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
+			download_url = 'http://f.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
+
+			info = {
+				'id': '%s_part%02d' % (video_id, index),
+				'url': download_url,
+				'uploader': None,
+				'title': video_title,
+				'ext': ext,
+				'format': u'NA'
+			}
+			files_info.append(info)
+
+		return files_info
+
+
+class XNXXIE(InfoExtractor):
+	"""Information extractor for xnxx.com"""
+
+	_VALID_URL = r'^http://video\.xnxx\.com/video([0-9]+)/(.*)'
+	IE_NAME = u'xnxx'
+	VIDEO_URL_RE = r'flv_url=(.*?)&amp;'
+	VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
+	VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&amp;'
+
+	def report_webpage(self, video_id):
+		"""Report information extraction"""
+		self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+	def report_extraction(self, video_id):
+		"""Report information extraction"""
+		self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+	def _real_extract(self, url):
+		mobj = re.match(self._VALID_URL, url)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+			return
+		video_id = mobj.group(1).decode('utf-8')
+
+		self.report_webpage(video_id)
+
+		# Get webpage content
+		try:
+			webpage = urllib2.urlopen(url).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
+			return
+
+		result = re.search(self.VIDEO_URL_RE, webpage)
+		if result is None:
+			self._downloader.trouble(u'ERROR: unable to extract video url')
+			return
+		video_url = urllib.unquote(result.group(1).decode('utf-8'))
+
+		result = re.search(self.VIDEO_TITLE_RE, webpage)
+		if result is None:
+			self._downloader.trouble(u'ERROR: unable to extract video title')
+			return
+		video_title = result.group(1).decode('utf-8')
+
+		result = re.search(self.VIDEO_THUMB_RE, webpage)
+		if result is None:
+			self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
+			return
+		video_thumbnail = result.group(1).decode('utf-8')
+
+		info = {'id': video_id,
+				'url': video_url,
+				'uploader': None,
+				'upload_date': None,
+				'title': video_title,
+				'ext': 'flv',
+				'format': 'flv',
+				'thumbnail': video_thumbnail,
+				'description': None,
+				'player_url': None}
+
+		return [info]
+
+
+class GooglePlusIE(InfoExtractor):
+	"""Information extractor for plus.google.com."""
+
+	_VALID_URL = r'(?:https://)?plus\.google\.com/(?:\w+/)*?(\d+)/posts/(\w+)'
+	IE_NAME = u'plus.google'
+
+	def __init__(self, downloader=None):
+		InfoExtractor.__init__(self, downloader)
+
+	def report_extract_entry(self, url):
+		"""Report downloading extry"""
+		self._downloader.to_screen(u'[plus.google] Downloading entry: %s' % url.decode('utf-8'))
+
+	def report_date(self, upload_date):
+		"""Report downloading extry"""
+		self._downloader.to_screen(u'[plus.google] Entry date: %s' % upload_date)
+
+	def report_uploader(self, uploader):
+		"""Report downloading extry"""
+		self._downloader.to_screen(u'[plus.google] Uploader: %s' % uploader.decode('utf-8'))
+
+	def report_title(self, video_title):
+		"""Report downloading extry"""
+		self._downloader.to_screen(u'[plus.google] Title: %s' % video_title.decode('utf-8'))
+
+	def report_extract_vid_page(self, video_page):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[plus.google] Extracting video page: %s' % video_page.decode('utf-8'))
+
+	def _real_extract(self, url):
+		# Extract id from URL
+		mobj = re.match(self._VALID_URL, url)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
+			return
+
+		post_url = mobj.group(0)
+		video_id = mobj.group(2)
+
+		video_extension = 'flv'
+
+		# Step 1, Retrieve post webpage to extract further information
+		self.report_extract_entry(post_url)
+		request = urllib2.Request(post_url)
+		try:
+			webpage = urllib2.urlopen(request).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % compat_str(err))
+			return
+
+		# Extract update date
+		upload_date = u'NA'
+		pattern = 'title="Timestamp">(.*?)</a>'
+		mobj = re.search(pattern, webpage)
+		if mobj:
+			upload_date = mobj.group(1)
+			# Convert timestring to a format suitable for filename
+			upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d")
+			upload_date = upload_date.strftime('%Y%m%d')
+		self.report_date(upload_date)
+
+		# Extract uploader
+		uploader = u'NA'
+		pattern = r'rel\="author".*?>(.*?)</a>'
+		mobj = re.search(pattern, webpage)
+		if mobj:
+			uploader = mobj.group(1)
+		self.report_uploader(uploader)
+
+		# Extract title
+		# Get the first line for title
+		video_title = u'NA'
+		pattern = r'<meta name\=\"Description\" content\=\"(.*?)[\n<"]'
+		mobj = re.search(pattern, webpage)
+		if mobj:
+			video_title = mobj.group(1)
+		self.report_title(video_title)
+
+		# Step 2, Stimulate clicking the image box to launch video
+		pattern = '"(https\://plus\.google\.com/photos/.*?)",,"image/jpeg","video"\]'
+		mobj = re.search(pattern, webpage)
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: unable to extract video page URL')
+
+		video_page = mobj.group(1)
+		request = urllib2.Request(video_page)
+		try:
+			webpage = urllib2.urlopen(request).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+			return
+		self.report_extract_vid_page(video_page)
+
+
+		# Extract video links on video page
+		"""Extract video links of all sizes"""
+		pattern = '\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"'
+		mobj = re.findall(pattern, webpage)
+		if len(mobj) == 0:
+			self._downloader.trouble(u'ERROR: unable to extract video links')
+
+		# Sort in resolution
+		links = sorted(mobj)
+
+		# Choose the lowest of the sort, i.e. highest resolution
+		video_url = links[-1]
+		# Only get the url. The resolution part in the tuple has no use anymore
+		video_url = video_url[-1]
+		# Treat escaped \u0026 style hex
+		video_url = unicode(video_url, "unicode_escape")
+
+
+		return [{
+			'id':		video_id.decode('utf-8'),
+			'url':		video_url,
+			'uploader':	uploader.decode('utf-8'),
+			'upload_date':	upload_date.decode('utf-8'),
+			'title':	video_title.decode('utf-8'),
+			'ext':		video_extension.decode('utf-8'),
+			'format':	u'NA',
+			'player_url':	None,
+		}]