class SearchInfoExtractor(InfoExtractor):
"""
- Base class for search queries extractors
+ Base class for paged search queries extractors.
They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
+ Instances should define _SEARCH_KEY and _MAX_RESULTS.
"""
- _max_results = 0 # The maximum number of results the extractor can get
@classmethod
- def _VALID_URL(cls):
- return r'%s(?P<prefix>|\d+|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
+ def _make_valid_url(cls):
+ return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
@classmethod
def suitable(cls, url):
- return re.match(cls._VALID_URL() , url) is not None
+ return re.match(cls._make_valid_url(), url) is not None
def _real_extract(self, query):
- mobj = re.match(self._VALID_URL(), query)
+ mobj = re.match(self._make_valid_url(), query)
if mobj is None:
raise ExtractorError(u'Invalid search query "%s"' % query)
if prefix == '':
return self._get_n_results(query, 1)
elif prefix == 'all':
- return self._get_n_results(query, self._max_results)
+ return self._get_n_results(query, self._MAX_RESULTS)
else:
n = int(prefix)
if n <= 0:
raise ExtractorError(u'invalid download number %s for query "%s"' % (n, query))
- elif n > self._max_results:
- self._downloader.report_warning(u'%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._max_results, n))
- n = self._max_results
+ elif n > self._MAX_RESULTS:
+ self._downloader.report_warning(u'%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
+ n = self._MAX_RESULTS
return self._get_n_results(query, n)
def _get_n_results(self, query, n):
opener.add_handler(handler())
response = opener.open(HeadRequest(url))
+ if response is None:
+ raise ExtractorError(u'Invalid URL protocol')
new_url = response.geturl()
if url == new_url:
class YoutubeSearchIE(SearchInfoExtractor):
"""Information Extractor for YouTube search queries."""
_API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
- _max_results = 1000
+ _MAX_RESULTS = 1000
IE_NAME = u'youtube:search'
_SEARCH_KEY = 'ytsearch'
class GoogleSearchIE(SearchInfoExtractor):
"""Information Extractor for Google Video search queries."""
_MORE_PAGES_INDICATOR = r'id="pnnext" class="pn"'
- _max_results = 1000
+ _MAX_RESULTS = 1000
IE_NAME = u'video.google:search'
_SEARCH_KEY = 'gvsearch'
class YahooSearchIE(SearchInfoExtractor):
"""Information Extractor for Yahoo! Video search queries."""
- _max_results = 1000
+ _MAX_RESULTS = 1000
IE_NAME = u'screen.yahoo:search'
_SEARCH_KEY = 'yvsearch'
video_id = m.group('videoID')
video_url = u'http://tcdn.ustream.tv/video/%s' % video_id
webpage = self._download_webpage(url, video_id)
- m = re.search(r'data-title="(?P<title>.+)"',webpage)
- title = m.group('title')
- m = re.search(r'<a class="state" data-content-type="channel" data-content-id="(?P<uploader>\d+)"',webpage)
- uploader = m.group('uploader')
+ self.report_extraction(video_id)
+ try:
+ m = re.search(r'data-title="(?P<title>.+)"',webpage)
+ title = m.group('title')
+ m = re.search(r'data-content-type="channel".*?>(?P<uploader>.*?)</a>',
+ webpage, re.DOTALL)
+ uploader = unescapeHTML(m.group('uploader').strip())
+ m = re.search(r'<link rel="image_src" href="(?P<thumb>.*?)"', webpage)
+ thumb = m.group('thumb')
+ except AttributeError:
+ raise ExtractorError(u'Unable to extract info')
info = {
'id':video_id,
'url':video_url,
'ext': 'flv',
'title': title,
- 'uploader': uploader
+ 'uploader': uploader,
+ 'thumbnail': thumb,
}
- return [info]
+ return info
class WorldStarHipHopIE(InfoExtractor):
_VALID_URL = r'https?://(?:www|m)\.worldstar(?:candy|hiphop)\.com/videos/video\.php\?v=(?P<id>.*)'
class InaIE(InfoExtractor):
"""Information Extractor for Ina.fr"""
- _VALID_URL = r'(?:http://)?(?:www.)?ina\.fr/video/(?P<id>I[0-9]+)/.*'
+ _VALID_URL = r'(?:http://)?(?:www\.)?ina\.fr/video/(?P<id>I[0-9]+)/.*'
def _real_extract(self,url):
mobj = re.match(self._VALID_URL, url)
'title': video_title,
}]
+class HowcastIE(InfoExtractor):
+ """Information Extractor for Ina.fr"""
+ _VALID_URL = r'(?:https?://)?(?:www\.)?howcast\.com/videos/(?P<id>[\d]+)'
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+
+ video_id = mobj.group('id')
+ webpage_url = 'http://www.howcast.com/videos/' + video_id
+ webpage = self._download_webpage(webpage_url, video_id)
+
+ mobj = re.search(r'\'file\': "(http://mobile-media\.howcast\.com/\d+\.mp4)"', webpage)
+ if mobj is None:
+ raise ExtractorError(u'Unable to extract video URL')
+ video_url = mobj.group(1)
+
+ mobj = re.search(r'<meta content=(?:"([^"]+)"|\'([^\']+)\') property=\'og:title\'', webpage)
+ if mobj is None:
+ raise ExtractorError(u'Unable to extract title')
+ video_title = mobj.group(1) or mobj.group(2)
+
+ mobj = re.search(r'<meta content=(?:"([^"]+)"|\'([^\']+)\') name=\'description\'', webpage)
+ if mobj is None:
+ self._downloader.report_warning(u'unable to extract description')
+ video_description = None
+ else:
+ video_description = mobj.group(1) or mobj.group(2)
+
+ return [{
+ 'id': video_id,
+ 'url': video_url,
+ 'ext': 'mp4',
+ 'title': video_title,
+ 'description': video_description,
+ }]
+
def gen_extractors():
""" Return a list of an instance of every supported extractor.
The order does matter; the first extractor matched is the one handling the URL.
BandcampIE(),
RedTubeIE(),
InaIE(),
+ HowcastIE(),
GenericIE()
]
projects / youtube-dl / blobdiff