X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2FInfoExtractors.py;h=5d54e93e7808c630fe1519d7c9a06a32da88cbfd;hb=8409501206e37d57f01e5fe72bfc54a5562e4e0a;hp=938d2d805d2e3ad8ca881677a8a2c18709ce3201;hpb=d4f76f1674c5fd6d5714a7500bc119b4b230f2b2;p=youtube-dl
diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py
index 938d2d805..5d54e93e7 100755
--- a/youtube_dl/InfoExtractors.py
+++ b/youtube_dl/InfoExtractors.py
@@ -16,6 +16,9 @@ import xml.etree.ElementTree
import random
import math
import operator
+import hashlib
+import binascii
+import urllib
from .utils import *
@@ -188,6 +191,38 @@ class InfoExtractor(object):
video_info['title'] = playlist_title
return video_info
+ def _search_regex(self, pattern, string, name, default=None, fatal=True, flags=0):
+ """
+ Perform a regex search on the given string, using a single or a list of
+ patterns returning the first matching group.
+ In case of failure return a default value or raise a WARNING or a
+ ExtractorError, depending on fatal, specifying the field name.
+ """
+ if isinstance(pattern, (str, compat_str, compiled_regex_type)):
+ mobj = re.search(pattern, string, flags)
+ else:
+ for p in pattern:
+ mobj = re.search(p, string, flags)
+ if mobj: break
+
+ if sys.stderr.isatty() and os.name != 'nt':
+ _name = u'\033[0;34m%s\033[0m' % name
+ else:
+ _name = name
+
+ if mobj:
+ # return the first matching group
+ return next(g for g in mobj.groups() if g is not None)
+ elif default is not None:
+ return default
+ elif fatal:
+ raise ExtractorError(u'Unable to extract %s; '
+ u'please report this issue on GitHub.' % _name)
+ else:
+ self._downloader.report_warning(u'unable to extract %s; '
+ u'please report this issue on GitHub.' % _name)
+ return None
+
class SearchInfoExtractor(InfoExtractor):
"""
Base class for paged search queries extractors.
@@ -373,6 +408,34 @@ class YoutubeIE(InfoExtractor):
return (u'Did not fetch video subtitles', None, None)
return (None, sub_lang, sub)
+ def _request_automatic_caption(self, video_id, webpage):
+ """We need the webpage for getting the captions url, pass it as an
+ argument to speed up the process."""
+ sub_lang = self._downloader.params.get('subtitleslang')
+ sub_format = self._downloader.params.get('subtitlesformat')
+ self.to_screen(u'%s: Looking for automatic captions' % video_id)
+ mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
+ err_msg = u'Couldn\'t find automatic captions for "%s"' % sub_lang
+ if mobj is None:
+ return [(err_msg, None, None)]
+ player_config = json.loads(mobj.group(1))
+ try:
+ args = player_config[u'args']
+ caption_url = args[u'ttsurl']
+ timestamp = args[u'timestamp']
+ params = compat_urllib_parse.urlencode({
+ 'lang': 'en',
+ 'tlang': sub_lang,
+ 'fmt': sub_format,
+ 'ts': timestamp,
+ 'kind': 'asr',
+ })
+ subtitles_url = caption_url + '&' + params
+ sub = self._download_webpage(subtitles_url, video_id, u'Downloading automatic captions')
+ return [(None, sub_lang, sub)]
+ except KeyError:
+ return [(err_msg, None, None)]
+
def _extract_subtitle(self, video_id):
"""
Return a list with a tuple:
@@ -620,7 +683,14 @@ class YoutubeIE(InfoExtractor):
if video_subtitles:
(sub_error, sub_lang, sub) = video_subtitles[0]
if sub_error:
- self._downloader.report_error(sub_error)
+ # We try with the automatic captions
+ video_subtitles = self._request_automatic_caption(video_id, video_webpage)
+ (sub_error_auto, sub_lang, sub) = video_subtitles[0]
+ if sub is not None:
+ pass
+ else:
+ # We report the original error
+ self._downloader.report_error(sub_error)
if self._downloader.params.get('allsubtitles', False):
video_subtitles = self._extract_all_subtitles(video_id)
@@ -926,18 +996,13 @@ class PhotobucketIE(InfoExtractor):
}]
# We try looking in other parts of the webpage
- mobj = re.search(r'', webpage)
- if mobj is None:
- raise ExtractorError(u'Unable to extract media URL')
- mediaURL = compat_urllib_parse.unquote(mobj.group(1))
-
- video_url = mediaURL
+ video_url = self._search_regex(r'',
+ webpage, u'video URL')
mobj = re.search(r'
(.*) video by (.*) - Photobucket', webpage)
if mobj is None:
raise ExtractorError(u'Unable to extract title')
video_title = mobj.group(1).decode('utf-8')
-
video_uploader = mobj.group(2).decode('utf-8')
return [{
@@ -1022,7 +1087,7 @@ class VimeoIE(InfoExtractor):
"""Information extractor for vimeo.com."""
# _VALID_URL matches Vimeo URLs
- _VALID_URL = r'(?Phttps?://)?(?:(?:www|player)\.)?vimeo\.com/(?:(?:groups|album)/[^/]+/)?(?Pplay_redirect_hls\?clip_id=)?(?:videos?/)?(?P[0-9]+)'
+ _VALID_URL = r'(?Phttps?://)?(?:(?:www|player)\.)?vimeo(?Ppro)?\.com/(?:(?:(?:groups|album)/[^/]+)|(?:.*?)/)?(?Pplay_redirect_hls\?clip_id=)?(?:videos?/)?(?P[0-9]+)'
IE_NAME = u'vimeo'
def _real_extract(self, url, new_video=True):
@@ -1034,7 +1099,7 @@ class VimeoIE(InfoExtractor):
video_id = mobj.group('id')
if not mobj.group('proto'):
url = 'https://' + url
- if mobj.group('direct_link'):
+ if mobj.group('direct_link') or mobj.group('pro'):
url = 'https://vimeo.com/' + video_id
# Retrieve video webpage to extract further information
@@ -1061,7 +1126,7 @@ class VimeoIE(InfoExtractor):
# Extract uploader and uploader_id
video_uploader = config["video"]["owner"]["name"]
- video_uploader_id = config["video"]["owner"]["url"].split('/')[-1]
+ video_uploader_id = config["video"]["owner"]["url"].split('/')[-1] if config["video"]["owner"]["url"] else None
# Extract video thumbnail
video_thumbnail = config["video"]["thumbnail"]
@@ -1765,10 +1830,7 @@ class DepositFilesIE(InfoExtractor):
file_extension = os.path.splitext(file_url)[1][1:]
# Search for file title
- mobj = re.search(r'', webpage)
- if mobj is None:
- raise ExtractorError(u'Unable to extract title')
- file_title = mobj.group(1).decode('utf-8')
+ file_title = self._search_regex(r'', webpage, u'title')
return [{
'id': file_id.decode('utf-8'),
@@ -1862,10 +1924,9 @@ class FacebookIE(InfoExtractor):
video_duration = int(video_data['video_duration'])
thumbnail = video_data['thumbnail_src']
- m = re.search('
([^<]+)
', webpage)
- if not m:
- raise ExtractorError(u'Cannot find title in webpage')
- video_title = unescapeHTML(m.group(1))
+ video_title = self._search_regex('
([^<]+)
',
+ webpage, u'title')
+ video_title = unescapeHTML(video_title)
info = {
'id': video_id,
@@ -1881,7 +1942,7 @@ class FacebookIE(InfoExtractor):
class BlipTVIE(InfoExtractor):
"""Information extractor for blip.tv"""
- _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
+ _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv/((.+/)|(play/)|(api\.swf#))(.+)$'
_URL_EXT = r'^.*\.([a-z0-9]+)$'
IE_NAME = u'blip.tv'
@@ -1894,6 +1955,10 @@ class BlipTVIE(InfoExtractor):
if mobj is None:
raise ExtractorError(u'Invalid URL: %s' % url)
+ # See https://github.com/rg3/youtube-dl/issues/857
+ api_mobj = re.match(r'http://a\.blip\.tv/api\.swf#(?P[\d\w]+)', url)
+ if api_mobj is not None:
+ url = 'http://blip.tv/play/g_%s' % api_mobj.group('video_id')
urlp = compat_urllib_parse_urlparse(url)
if urlp.path.startswith('/play/'):
request = compat_urllib_request.Request(url)
@@ -1979,39 +2044,150 @@ class MyVideoIE(InfoExtractor):
_VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
IE_NAME = u'myvideo'
+ # Original Code from: https://github.com/dersphere/plugin.video.myvideo_de.git
+ # Released into the Public Domain by Tristan Fischer on 2013-05-19
+ # https://github.com/rg3/youtube-dl/pull/842
+ def __rc4crypt(self,data, key):
+ x = 0
+ box = list(range(256))
+ for i in list(range(256)):
+ x = (x + box[i] + compat_ord(key[i % len(key)])) % 256
+ box[i], box[x] = box[x], box[i]
+ x = 0
+ y = 0
+ out = ''
+ for char in data:
+ x = (x + 1) % 256
+ y = (y + box[x]) % 256
+ box[x], box[y] = box[y], box[x]
+ out += chr(compat_ord(char) ^ box[(box[x] + box[y]) % 256])
+ return out
+
+ def __md5(self,s):
+ return hashlib.md5(s).hexdigest().encode()
+
def _real_extract(self,url):
mobj = re.match(self._VALID_URL, url)
if mobj is None:
- raise ExtractorError(u'Invalid URL: %s' % url)
+ raise ExtractorError(u'invalid URL: %s' % url)
video_id = mobj.group(1)
+ GK = (
+ b'WXpnME1EZGhNRGhpTTJNM01XVmhOREU0WldNNVpHTTJOakpt'
+ b'TW1FMU5tVTBNR05pWkRaa05XRXhNVFJoWVRVd1ptSXhaVEV3'
+ b'TnpsbA0KTVRkbU1tSTRNdz09'
+ )
+
# Get video webpage
webpage_url = 'http://www.myvideo.de/watch/%s' % video_id
webpage = self._download_webpage(webpage_url, video_id)
- self.report_extraction(video_id)
- mobj = re.search(r'([^<]+)',
+ webpage, u'title')
+
+ video_ext = self._search_regex('[.](.+?)$', video_url, u'extension')
- mobj = re.search('([^<]+)', webpage)
+ return [{
+ 'id': video_id,
+ 'url': video_url,
+ 'uploader': None,
+ 'upload_date': None,
+ 'title': video_title,
+ 'ext': u'flv',
+ }]
+
+ # try encxml
+ mobj = re.search('var flashvars={(.+?)}', webpage)
if mobj is None:
- raise ExtractorError(u'Unable to extract title')
+ raise ExtractorError(u'Unable to extract video')
+
+ params = {}
+ encxml = ''
+ sec = mobj.group(1)
+ for (a, b) in re.findall('(.+?):\'(.+?)\',?', sec):
+ if not a == '_encxml':
+ params[a] = b
+ else:
+ encxml = compat_urllib_parse.unquote(b)
+ if not params.get('domain'):
+ params['domain'] = 'www.myvideo.de'
+ xmldata_url = '%s?%s' % (encxml, compat_urllib_parse.urlencode(params))
+ if 'flash_playertype=MTV' in xmldata_url:
+ self._downloader.report_warning(u'avoiding MTV player')
+ xmldata_url = (
+ 'http://www.myvideo.de/dynamic/get_player_video_xml.php'
+ '?flash_playertype=D&ID=%s&_countlimit=4&autorun=yes'
+ ) % video_id
+
+ # get enc data
+ enc_data = self._download_webpage(xmldata_url, video_id).split('=')[1]
+ enc_data_b = binascii.unhexlify(enc_data)
+ sk = self.__md5(
+ base64.b64decode(base64.b64decode(GK)) +
+ self.__md5(
+ str(video_id).encode('utf-8')
+ )
+ )
+ dec_data = self.__rc4crypt(enc_data_b, sk)
- video_title = mobj.group(1)
+ # extracting infos
+ self.report_extraction(video_id)
+
+ video_url = None
+ mobj = re.search('connectionurl=\'(.*?)\'', dec_data)
+ if mobj:
+ video_url = compat_urllib_parse.unquote(mobj.group(1))
+ if 'myvideo2flash' in video_url:
+ self._downloader.report_warning(u'forcing RTMPT ...')
+ video_url = video_url.replace('rtmpe://', 'rtmpt://')
+
+ if not video_url:
+ # extract non rtmp videos
+ mobj = re.search('path=\'(http.*?)\' source=\'(.*?)\'', dec_data)
+ if mobj is None:
+ raise ExtractorError(u'unable to extract url')
+ video_url = compat_urllib_parse.unquote(mobj.group(1)) + compat_urllib_parse.unquote(mobj.group(2))
+
+ video_file = self._search_regex('source=\'(.*?)\'', dec_data, u'video file')
+ video_file = compat_urllib_parse.unquote(video_file)
+
+ if not video_file.endswith('f4m'):
+ ppath, prefix = video_file.split('.')
+ video_playpath = '%s:%s' % (prefix, ppath)
+ video_hls_playlist = ''
+ else:
+ video_playpath = ''
+ video_hls_playlist = (
+ video_filepath + video_file
+ ).replace('.f4m', '.m3u8')
+
+ video_swfobj = self._search_regex('swfobject.embedSWF\(\'(.+?)\'', webpage, u'swfobj')
+ video_swfobj = compat_urllib_parse.unquote(video_swfobj)
+
+ video_title = self._search_regex("
(.*?)
",
+ webpage, u'title')
return [{
- 'id': video_id,
- 'url': video_url,
- 'uploader': None,
- 'upload_date': None,
- 'title': video_title,
- 'ext': u'flv',
+ 'id': video_id,
+ 'url': video_url,
+ 'tc_url': video_url,
+ 'uploader': None,
+ 'upload_date': None,
+ 'title': video_title,
+ 'ext': u'flv',
+ 'play_path': video_playpath,
+ 'video_file': video_file,
+ 'video_hls_playlist': video_hls_playlist,
+ 'player_url': video_swfobj,
}]
+
class ComedyCentralIE(InfoExtractor):
"""Information extractor for The Daily Show and Colbert Report """
@@ -2194,16 +2370,22 @@ class EscapistIE(InfoExtractor):
videoId = mobj.group('episode')
self.report_extraction(showName)
- webPage = self._download_webpage(url, showName)
+ webpage = self._download_webpage(url, showName)
+
+ videoDesc = self._search_regex('(.*?)\s+-\s+XVID', webpage)
- if mobj is None:
- raise ExtractorError(u'Unable to extract video title')
- video_title = mobj.group(1)
-
+ video_title = self._search_regex(r'(.*?)\s+-\s+XVID',
+ webpage, u'title')
# Extract video thumbnail
- mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/([a-fA-F0-9.]+jpg)', webpage)
- if mobj is None:
- raise ExtractorError(u'Unable to extract video thumbnail')
- video_thumbnail = mobj.group(0)
+ video_thumbnail = self._search_regex(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/([a-fA-F0-9.]+jpg)',
+ webpage, u'thumbnail', fatal=False)
info = {
'id': video_id,
@@ -2489,16 +2662,12 @@ class InfoQIE(InfoExtractor):
video_url = 'rtmpe://video.infoq.com/cfx/st/' + real_id
# Extract title
- mobj = re.search(r'contentTitle = "(.*?)";', webpage)
- if mobj is None:
- raise ExtractorError(u'Unable to extract video title')
- video_title = mobj.group(1)
+ video_title = self._search_regex(r'contentTitle = "(.*?)";',
+ webpage, u'title')
# Extract description
- video_description = u'No description available.'
- mobj = re.search(r'', webpage)
- if mobj is not None:
- video_description = mobj.group(1)
+ video_description = self._search_regex(r'',
+ webpage, u'description', fatal=False)
video_filename = video_url.split('/')[-1]
video_id, extension = video_filename.split('.')
@@ -2669,15 +2838,12 @@ class StanfordOpenClassroomIE(InfoExtractor):
note='Downloading course info page',
errnote='Unable to download course info page')
- m = re.search('