X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;ds=sidebyside;f=youtube_dl%2FInfoExtractors.py;h=6442888d88eb3d097d92ee792b1fcbb7b343a230;hb=ce3674430b0647e9918ce5d53f447e0ca35752ab;hp=4314f14022b650cb965707ee5c90e07fbecd67c4;hpb=d11d05d07acdd11a93b02d750852dea4ae32be3b;p=youtube-dl
diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py
index 4314f1402..6442888d8 100644
--- a/youtube_dl/InfoExtractors.py
+++ b/youtube_dl/InfoExtractors.py
@@ -12,28 +12,16 @@ import time
import urllib
import urllib2
import email.utils
+import xml.etree.ElementTree
+import random
+import math
+from urlparse import parse_qs
try:
import cStringIO as StringIO
except ImportError:
import StringIO
-# parse_qs was moved from the cgi module to the urlparse module recently.
-try:
- from urlparse import parse_qs
-except ImportError:
- from cgi import parse_qs
-
-try:
- import lxml.etree
-except ImportError:
- pass # Handled below
-
-try:
- import xml.etree.ElementTree
-except ImportError: # Python<2.5: Not officially supported, but let it slip
- warnings.warn('xml.etree.ElementTree support is missing. Consider upgrading to Python >= 2.5 if you get related errors.')
-
from utils import *
@@ -53,7 +41,6 @@ class InfoExtractor(object):
url: Final video URL.
uploader: Nickname of the video uploader.
title: Literal title.
- stitle: Simplified title.
ext: Video filename extension.
format: Video format.
player_url: SWF Player URL (may be None).
@@ -110,15 +97,15 @@ class InfoExtractor(object):
class YoutubeIE(InfoExtractor):
"""Information extractor for youtube.com."""
- _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?!view_play_list|my_playlists|artist|playlist)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$'
+ _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/|tube\.majestyc\.net/)(?!view_play_list|my_playlists|artist|playlist)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$'
_LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
_LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
_AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
_NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
_NETRC_MACHINE = 'youtube'
# Listed in order of quality
- _available_formats = ['38', '37', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
- _available_formats_prefer_free = ['38', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
+ _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
+ _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
_video_extensions = {
'13': '3gp',
'17': 'mp4',
@@ -129,6 +116,7 @@ class YoutubeIE(InfoExtractor):
'43': 'webm',
'44': 'webm',
'45': 'webm',
+ '46': 'webm',
}
_video_dimensions = {
'5': '240x400',
@@ -144,6 +132,7 @@ class YoutubeIE(InfoExtractor):
'43': '360x640',
'44': '480x854',
'45': '720x1280',
+ '46': '1080x1920',
}
IE_NAME = u'youtube'
@@ -193,9 +182,9 @@ class YoutubeIE(InfoExtractor):
end = start + float(dur)
start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000)
end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000)
- caption = re.sub(ur'(?u)&(.+?);', htmlentity_transform, caption)
- caption = re.sub(ur'(?u)&(.+?);', htmlentity_transform, caption) # double cycle, inentional
- srt += str(n) + '\n'
+ caption = unescapeHTML(caption)
+ caption = unescapeHTML(caption) # double cycle, intentional
+ srt += str(n+1) + '\n'
srt += start + ' --> ' + end + '\n'
srt += caption + '\n\n'
return srt
@@ -324,6 +313,11 @@ class YoutubeIE(InfoExtractor):
self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
return
+ # Check for "rental" videos
+ if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
+ self._downloader.trouble(u'ERROR: "rental" videos not supported')
+ return
+
# Start extracting information
self.report_information_extraction(video_id)
@@ -339,10 +333,6 @@ class YoutubeIE(InfoExtractor):
return
video_title = urllib.unquote_plus(video_info['title'][0])
video_title = video_title.decode('utf-8')
- video_title = sanitize_title(video_title)
-
- # simplified title
- simple_title = simplify_title(video_title)
# thumbnail image
if 'thumbnail_url' not in video_info:
@@ -364,49 +354,42 @@ class YoutubeIE(InfoExtractor):
pass
# description
- try:
- lxml.etree
- except NameError:
- video_description = u'No description available.'
- mobj = re.search(r'', video_webpage)
- if mobj is not None:
- video_description = mobj.group(1).decode('utf-8')
- else:
- html_parser = lxml.etree.HTMLParser(encoding='utf-8')
- vwebpage_doc = lxml.etree.parse(StringIO.StringIO(video_webpage), html_parser)
- video_description = u''.join(vwebpage_doc.xpath('id("eow-description")//text()'))
- # TODO use another parser
+ video_description = get_element_by_id("eow-description", video_webpage.decode('utf8'))
+ if video_description: video_description = clean_html(video_description)
+ else: video_description = ''
# closed captions
video_subtitles = None
if self._downloader.params.get('writesubtitles', False):
- self.report_video_subtitles_download(video_id)
- request = urllib2.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
try:
- srt_list = urllib2.urlopen(request).read()
- except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
- else:
- srt_lang_list = re.findall(r'lang_code="([\w\-]+)"', srt_list)
- if srt_lang_list:
- if self._downloader.params.get('subtitleslang', False):
- srt_lang = self._downloader.params.get('subtitleslang')
- elif 'en' in srt_lang_list:
- srt_lang = 'en'
- else:
- srt_lang = srt_lang_list[0]
- if not srt_lang in srt_lang_list:
- self._downloader.trouble(u'WARNING: no closed captions found in the specified language')
- else:
- request = urllib2.Request('http://video.google.com/timedtext?hl=en&lang=%s&v=%s' % (srt_lang, video_id))
- try:
- srt_xml = urllib2.urlopen(request).read()
- except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
- else:
- video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
+ self.report_video_subtitles_download(video_id)
+ request = urllib2.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
+ try:
+ srt_list = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ raise Trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
+ srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list)
+ srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list)
+ if not srt_lang_list:
+ raise Trouble(u'WARNING: video has no closed captions')
+ if self._downloader.params.get('subtitleslang', False):
+ srt_lang = self._downloader.params.get('subtitleslang')
+ elif 'en' in srt_lang_list:
+ srt_lang = 'en'
else:
- self._downloader.trouble(u'WARNING: video has no closed captions')
+ srt_lang = srt_lang_list.keys()[0]
+ if not srt_lang in srt_lang_list:
+ raise Trouble(u'WARNING: no closed captions found in the specified language')
+ request = urllib2.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id))
+ try:
+ srt_xml = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ raise Trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
+ if not srt_xml:
+ raise Trouble(u'WARNING: unable to download video subtitles')
+ video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
+ except Trouble as trouble:
+ self._downloader.trouble(trouble[0])
# token
video_token = urllib.unquote_plus(video_info['token'][0])
@@ -421,7 +404,7 @@ class YoutubeIE(InfoExtractor):
url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
url_data = [parse_qs(uds) for uds in url_data_strs]
url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data)
- url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data)
+ url_map = dict((ud['itag'][0], ud['url'][0] + '&signature=' + ud['sig'][0]) for ud in url_data)
format_limit = self._downloader.params.get('format_limit', None)
available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
@@ -469,7 +452,6 @@ class YoutubeIE(InfoExtractor):
'uploader': video_uploader.decode('utf-8'),
'upload_date': upload_date,
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': video_thumbnail.decode('utf-8'),
@@ -545,8 +527,6 @@ class MetacafeIE(InfoExtractor):
self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
return
- simple_title = mobj.group(2).decode('utf-8')
-
# Retrieve video webpage to extract further information
request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
try:
@@ -592,7 +572,6 @@ class MetacafeIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
- video_title = sanitize_title(video_title)
mobj = re.search(r'(?ms)By:\s*(.+?)<', webpage)
if mobj is None:
@@ -606,7 +585,6 @@ class MetacafeIE(InfoExtractor):
'uploader': video_uploader.decode('utf-8'),
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
@@ -673,8 +651,6 @@ class DailymotionIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
- video_title = sanitize_title(video_title)
- simple_title = simplify_title(video_title)
mobj = re.search(r'(?im)[^<]+?]+?>([^<]+?)', webpage)
if mobj is None:
@@ -688,7 +664,6 @@ class DailymotionIE(InfoExtractor):
'uploader': video_uploader.decode('utf-8'),
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
@@ -752,8 +727,6 @@ class GoogleIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
- video_title = sanitize_title(video_title)
- simple_title = simplify_title(video_title)
# Extract video description
mobj = re.search(r'([^<]*)', webpage)
@@ -786,7 +759,6 @@ class GoogleIE(InfoExtractor):
'uploader': u'NA',
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
@@ -845,8 +817,6 @@ class PhotobucketIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
- video_title = sanitize_title(video_title)
- simple_title = simplify_title(video_title)
video_uploader = mobj.group(2).decode('utf-8')
@@ -856,7 +826,6 @@ class PhotobucketIE(InfoExtractor):
'uploader': video_uploader,
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
@@ -934,7 +903,6 @@ class YahooIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract video title')
return
video_title = mobj.group(1).decode('utf-8')
- simple_title = simplify_title(video_title)
mobj = re.search(r'', webpage)
if mobj is None:
@@ -992,7 +960,7 @@ class YahooIE(InfoExtractor):
self._downloader.trouble(u'ERROR: Unable to extract media URL')
return
video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
- video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
+ video_url = unescapeHTML(video_url)
return [{
'id': video_id.decode('utf-8'),
@@ -1000,7 +968,6 @@ class YahooIE(InfoExtractor):
'uploader': video_uploader,
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'thumbnail': video_thumbnail.decode('utf-8'),
'description': video_description,
@@ -1060,7 +1027,6 @@ class VimeoIE(InfoExtractor):
# Extract title
video_title = config["video"]["title"]
- simple_title = simplify_title(video_title)
# Extract uploader
video_uploader = config["video"]["owner"]["name"]
@@ -1069,18 +1035,9 @@ class VimeoIE(InfoExtractor):
video_thumbnail = config["video"]["thumbnail"]
# Extract video description
- try:
- lxml.etree
- except NameError:
- video_description = u'No description available.'
- mobj = re.search(r'', webpage, re.MULTILINE)
- if mobj is not None:
- video_description = mobj.group(1)
- else:
- html_parser = lxml.etree.HTMLParser()
- vwebpage_doc = lxml.etree.parse(StringIO.StringIO(webpage), html_parser)
- video_description = u''.join(vwebpage_doc.xpath('id("description")//text()')).strip()
- # TODO use another parser
+ video_description = get_element_by_id("description", webpage.decode('utf8'))
+ if video_description: video_description = clean_html(video_description)
+ else: video_description = ''
# Extract upload date
video_upload_date = u'NA'
@@ -1115,7 +1072,6 @@ class VimeoIE(InfoExtractor):
'uploader': video_uploader,
'upload_date': video_upload_date,
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension,
'thumbnail': video_thumbnail,
'description': video_description,
@@ -1250,8 +1206,6 @@ class GenericIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
- video_title = sanitize_title(video_title)
- simple_title = simplify_title(video_title)
# video uploader is domain name
mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
@@ -1266,7 +1220,6 @@ class GenericIE(InfoExtractor):
'uploader': video_uploader,
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
@@ -1284,7 +1237,7 @@ class YoutubeSearchIE(InfoExtractor):
InfoExtractor.__init__(self, downloader)
def report_download_page(self, query, pagenum):
- """Report attempt to download playlist page with given number."""
+ """Report attempt to download search page with given number."""
query = query.decode(preferredencoding())
self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
@@ -1520,8 +1473,8 @@ class YoutubePlaylistIE(InfoExtractor):
_VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
- _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&list=PL%s&'
- _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*'
+ _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&list=.*?%s'
+ _MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
IE_NAME = u'youtube:playlist'
def __init__(self, downloader=None):
@@ -1671,6 +1624,98 @@ class YoutubeUserIE(InfoExtractor):
self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id])
+class BlipTVUserIE(InfoExtractor):
+ """Information Extractor for blip.tv users."""
+
+ _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$'
+ _PAGE_SIZE = 12
+ IE_NAME = u'blip.tv:user'
+
+ def __init__(self, downloader=None):
+ InfoExtractor.__init__(self, downloader)
+
+ def report_download_page(self, username, pagenum):
+ """Report attempt to download user page."""
+ self._downloader.to_screen(u'[%s] user %s: Downloading video ids from page %d' %
+ (self.IE_NAME, username, pagenum))
+
+ def _real_extract(self, url):
+ # Extract username
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid url: %s' % url)
+ return
+
+ username = mobj.group(1)
+
+ page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
+
+ request = urllib2.Request(url)
+
+ try:
+ page = urllib2.urlopen(request).read().decode('utf-8')
+ mobj = re.search(r'data-users-id="([^"]+)"', page)
+ page_base = page_base % mobj.group(1)
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
+ return
+
+
+ # Download video ids using BlipTV Ajax calls. Result size per
+ # query is limited (currently to 12 videos) so we need to query
+ # page by page until there are no video ids - it means we got
+ # all of them.
+
+ video_ids = []
+ pagenum = 1
+
+ while True:
+ self.report_download_page(username, pagenum)
+
+ request = urllib2.Request( page_base + "&page=" + str(pagenum) )
+
+ try:
+ page = urllib2.urlopen(request).read().decode('utf-8')
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
+ return
+
+ # Extract video identifiers
+ ids_in_page = []
+
+ for mobj in re.finditer(r'href="/([^"]+)"', page):
+ if mobj.group(1) not in ids_in_page:
+ ids_in_page.append(unescapeHTML(mobj.group(1)))
+
+ video_ids.extend(ids_in_page)
+
+ # A little optimization - if current page is not
+ # "full", ie. does not contain PAGE_SIZE video ids then
+ # we can assume that this page is the last one - there
+ # are no more ids on further pages - no need to query
+ # again.
+
+ if len(ids_in_page) < self._PAGE_SIZE:
+ break
+
+ pagenum += 1
+
+ all_ids_count = len(video_ids)
+ playliststart = self._downloader.params.get('playliststart', 1) - 1
+ playlistend = self._downloader.params.get('playlistend', -1)
+
+ if playlistend == -1:
+ video_ids = video_ids[playliststart:]
+ else:
+ video_ids = video_ids[playliststart:playlistend]
+
+ self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" %
+ (self.IE_NAME, username, all_ids_count, len(video_ids)))
+
+ for video_id in video_ids:
+ self._downloader.download([u'http://blip.tv/'+video_id])
+
+
class DepositFilesIE(InfoExtractor):
"""Information extractor for depositfiles.com"""
@@ -1731,7 +1776,6 @@ class DepositFilesIE(InfoExtractor):
'uploader': u'NA',
'upload_date': u'NA',
'title': file_title,
- 'stitle': file_title,
'ext': file_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
@@ -1876,9 +1920,6 @@ class FacebookIE(InfoExtractor):
return
video_title = video_info['title']
video_title = video_title.decode('utf-8')
- video_title = sanitize_title(video_title)
-
- simple_title = simplify_title(video_title)
# thumbnail image
if 'thumbnail' not in video_info:
@@ -1939,7 +1980,6 @@ class FacebookIE(InfoExtractor):
'uploader': video_uploader.decode('utf-8'),
'upload_date': upload_date,
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': video_thumbnail.decode('utf-8'),
@@ -1974,7 +2014,7 @@ class BlipTVIE(InfoExtractor):
else:
cchar = '?'
json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
- request = urllib2.Request(json_url)
+ request = urllib2.Request(json_url.encode('utf-8'))
self.report_extraction(mobj.group(1))
info = None
try:
@@ -1989,7 +2029,6 @@ class BlipTVIE(InfoExtractor):
'id': title,
'url': url,
'title': title,
- 'stitle': simplify_title(title),
'ext': ext,
'urlhandle': urlh
}
@@ -2009,21 +2048,20 @@ class BlipTVIE(InfoExtractor):
data = json_data['Post']
else:
data = json_data
-
+
upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
video_url = data['media']['url']
umobj = re.match(self._URL_EXT, video_url)
if umobj is None:
raise ValueError('Can not determine filename extension')
ext = umobj.group(1)
-
+
info = {
'id': data['item_id'],
'url': video_url,
'uploader': data['display_name'],
'upload_date': upload_date,
'title': data['title'],
- 'stitle': simplify_title(data['title']),
'ext': ext,
'format': data['media']['mimeType'],
'thumbnail': data['thumbnailUrl'],
@@ -2034,6 +2072,7 @@ class BlipTVIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
return
+ std_headers['User-Agent'] = 'iTunes/10.6.1'
return [info]
@@ -2085,9 +2124,6 @@ class MyVideoIE(InfoExtractor):
return
video_title = mobj.group(1)
- video_title = sanitize_title(video_title)
-
- simple_title = simplify_title(video_title)
return [{
'id': video_id,
@@ -2095,7 +2131,6 @@ class MyVideoIE(InfoExtractor):
'uploader': u'NA',
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': u'flv',
'format': u'NA',
'player_url': None,
@@ -2109,7 +2144,7 @@ class ComedyCentralIE(InfoExtractor):
def report_extraction(self, episode_id):
self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
-
+
def report_config_download(self, episode_id):
self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
@@ -2222,7 +2257,6 @@ class ComedyCentralIE(InfoExtractor):
'uploader': showId,
'upload_date': officialDate,
'title': effTitle,
- 'stitle': simplify_title(effTitle),
'ext': 'mp4',
'format': format,
'thumbnail': None,
@@ -2248,8 +2282,6 @@ class EscapistIE(InfoExtractor):
self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
def _real_extract(self, url):
- htmlParser = HTMLParser.HTMLParser()
-
mobj = re.match(self._VALID_URL, url)
if mobj is None:
self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
@@ -2259,17 +2291,20 @@ class EscapistIE(InfoExtractor):
self.report_extraction(showName)
try:
- webPage = urllib2.urlopen(url).read()
+ webPage = urllib2.urlopen(url)
+ webPageBytes = webPage.read()
+ m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
+ webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self._downloader.trouble(u'ERROR: unable to download webpage: ' + unicode(err))
return
descMatch = re.search('([^<]+)', coursepage)
if m:
@@ -2822,8 +2840,6 @@ class StanfordOpenClassroomIE(InfoExtractor):
return results
else: # Root page
- unescapeHTML = HTMLParser.HTMLParser().unescape
-
info = {
'id': 'Stanford OpenClassroom',
'type': 'playlist',
@@ -2838,7 +2854,6 @@ class StanfordOpenClassroomIE(InfoExtractor):
return
info['title'] = info['id']
- info['stitle'] = simplify_title(info['title'])
links = orderedSet(re.findall('', rootpage))
info['list'] = [
@@ -2937,9 +2952,195 @@ class MTVIE(InfoExtractor):
'url': video_url,
'uploader': performer,
'title': video_title,
- 'stitle': simplify_title(video_title),
'ext': ext,
'format': format,
}
return [info]
+
+
+class YoukuIE(InfoExtractor):
+
+ _VALID_URL = r'(?:http://)?v\.youku\.com/v_show/id_(?P[A-Za-z0-9]+)\.html'
+ IE_NAME = u'Youku'
+
+ def __init__(self, downloader=None):
+ InfoExtractor.__init__(self, downloader)
+
+ def report_download_webpage(self, file_id):
+ """Report webpage download."""
+ self._downloader.to_screen(u'[Youku] %s: Downloading webpage' % file_id)
+
+ def report_extraction(self, file_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[Youku] %s: Extracting information' % file_id)
+
+ def _gen_sid(self):
+ nowTime = int(time.time() * 1000)
+ random1 = random.randint(1000,1998)
+ random2 = random.randint(1000,9999)
+
+ return "%d%d%d" %(nowTime,random1,random2)
+
+ def _get_file_ID_mix_string(self, seed):
+ mixed = []
+ source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
+ seed = float(seed)
+ for i in range(len(source)):
+ seed = (seed * 211 + 30031 ) % 65536
+ index = math.floor(seed / 65536 * len(source) )
+ mixed.append(source[int(index)])
+ source.remove(source[int(index)])
+ #return ''.join(mixed)
+ return mixed
+
+ def _get_file_id(self, fileId, seed):
+ mixed = self._get_file_ID_mix_string(seed)
+ ids = fileId.split('*')
+ realId = []
+ for ch in ids:
+ if ch:
+ realId.append(mixed[int(ch)])
+ return ''.join(realId)
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+ video_id = mobj.group('ID')
+
+ info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
+
+ request = urllib2.Request(info_url, None, std_headers)
+ try:
+ self.report_download_webpage(video_id)
+ jsondata = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
+ self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
+ return
+
+ self.report_extraction(video_id)
+ try:
+ config = json.loads(jsondata)
+
+ video_title = config['data'][0]['title']
+ seed = config['data'][0]['seed']
+
+ format = self._downloader.params.get('format', None)
+ supported_format = config['data'][0]['streamfileids'].keys()
+
+ if format is None or format == 'best':
+ if 'hd2' in supported_format:
+ format = 'hd2'
+ else:
+ format = 'flv'
+ ext = u'flv'
+ elif format == 'worst':
+ format = 'mp4'
+ ext = u'mp4'
+ else:
+ format = 'flv'
+ ext = u'flv'
+
+
+ fileid = config['data'][0]['streamfileids'][format]
+ seg_number = len(config['data'][0]['segs'][format])
+
+ keys=[]
+ for i in xrange(seg_number):
+ keys.append(config['data'][0]['segs'][format][i]['k'])
+
+ #TODO check error
+ #youku only could be viewed from mainland china
+ except:
+ self._downloader.trouble(u'ERROR: unable to extract info section')
+ return
+
+ files_info=[]
+ sid = self._gen_sid()
+ fileid = self._get_file_id(fileid, seed)
+
+ #column 8,9 of fileid represent the segment number
+ #fileid[7:9] should be changed
+ for index, key in enumerate(keys):
+
+ temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
+ download_url = 'http://f.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
+
+ info = {
+ 'id': '%s_part%02d' % (video_id, index),
+ 'url': download_url,
+ 'uploader': None,
+ 'title': video_title,
+ 'ext': ext,
+ 'format': u'NA'
+ }
+ files_info.append(info)
+
+ return files_info
+
+
+class XNXXIE(InfoExtractor):
+ """Information extractor for xnxx.com"""
+
+ _VALID_URL = r'^http://video\.xnxx\.com/video([0-9]+)/(.*)'
+ IE_NAME = u'xnxx'
+ VIDEO_URL_RE = r'flv_url=(.*?)&'
+ VIDEO_TITLE_RE = r'(.*?)\s+-\s+XNXX.COM'
+ VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&'
+
+ def report_webpage(self, video_id):
+ """Report information extraction"""
+ self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+ def report_extraction(self, video_id):
+ """Report information extraction"""
+ self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+ video_id = mobj.group(1).decode('utf-8')
+
+ self.report_webpage(video_id)
+
+ # Get webpage content
+ try:
+ webpage = urllib2.urlopen(url).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
+ return
+
+ result = re.search(self.VIDEO_URL_RE, webpage)
+ if result is None:
+ self._downloader.trouble(u'ERROR: unable to extract video url')
+ return
+ video_url = urllib.unquote(result.group(1).decode('utf-8'))
+
+ result = re.search(self.VIDEO_TITLE_RE, webpage)
+ if result is None:
+ self._downloader.trouble(u'ERROR: unable to extract video title')
+ return
+ video_title = result.group(1).decode('utf-8')
+
+ result = re.search(self.VIDEO_THUMB_RE, webpage)
+ if result is None:
+ self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
+ return
+ video_thumbnail = result.group(1).decode('utf-8')
+
+ info = {'id': video_id,
+ 'url': video_url,
+ 'uploader': None,
+ 'upload_date': None,
+ 'title': video_title,
+ 'ext': 'flv',
+ 'format': 'flv',
+ 'thumbnail': video_thumbnail,
+ 'description': None,
+ 'player_url': None}
+
+ return [info]