X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Fgeneric.py;h=5a493314699b131febf06e4af25695b3f862cded;hb=ecbe1ad2073ffa4b084a69854e000f6e03cba09c;hp=082da9c77b138f35651e2de8a2d035f8fee1f517;hpb=90dc5e8693992f2e94dcf1d96ce069681778921c;p=youtube-dl diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 082da9c77..5a4933146 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -12,9 +12,11 @@ from ..utils import ( compat_urllib_parse, compat_urllib_request, compat_urlparse, + compat_xml_parse_error, ExtractorError, HEADRequest, + parse_xml, smuggle_url, unescapeHTML, unified_strdate, @@ -22,6 +24,7 @@ from ..utils import ( ) from .brightcove import BrightcoveIE from .ooyala import OoyalaIE +from .rutv import RUTVIE class GenericIE(InfoExtractor): @@ -81,10 +84,10 @@ class GenericIE(InfoExtractor): # Direct link to a video { 'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4', - 'file': 'trailer.mp4', 'md5': '67d406c2bcb6af27fa886f31aa934bbe', 'info_dict': { 'id': 'trailer', + 'ext': 'mp4', 'title': 'trailer', 'upload_date': '20100513', } @@ -92,7 +95,6 @@ class GenericIE(InfoExtractor): # ooyala video { 'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219', - 'file': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ.mp4', 'md5': '5644c6ca5d5782c1d0d350dad9bd840c', 'info_dict': { 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ', @@ -100,6 +102,64 @@ class GenericIE(InfoExtractor): 'title': '2cc213299525360.mov', # that's what we get }, }, + # google redirect + { + 'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE', + 'info_dict': { + 'id': 'cmQHVoWB5FY', + 'ext': 'mp4', + 'upload_date': '20130224', + 'uploader_id': 'TheVerge', + 'description': 'Chris Ziegler takes a look at the Alcatel OneTouch Fire and the ZTE Open; two of the first Firefox OS handsets to be officially announced.', + 'uploader': 'The Verge', + 'title': 'First Firefox OS phones side-by-side', + }, + 'params': { + 'skip_download': False, + } + }, + # embed.ly video + { + 'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/', + 'info_dict': { + 'id': '9ODmcdjQcHQ', + 'ext': 'mp4', + 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second', + 'upload_date': '20140225', + 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff', + 'uploader': 'Tested', + 'uploader_id': 'testedcom', + }, + # No need to test YoutubeIE here + 'params': { + 'skip_download': True, + }, + }, + # funnyordie embed + { + 'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns', + 'md5': '7cf780be104d40fea7bae52eed4a470e', + 'info_dict': { + 'id': '18e820ec3f', + 'ext': 'mp4', + 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama', + 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.', + }, + }, + # RUTV embed + { + 'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html', + 'info_dict': { + 'id': '776940', + 'ext': 'mp4', + 'title': 'Охотское море стало целиком российским', + 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43', + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + } ] def report_download_webpage(self, video_id): @@ -125,9 +185,14 @@ class GenericIE(InfoExtractor): newurl = newurl.replace(' ', '%20') newheaders = dict((k,v) for k,v in req.headers.items() if k.lower() not in ("content-length", "content-type")) + try: + # This function was deprecated in python 3.3 and removed in 3.4 + origin_req_host = req.get_origin_req_host() + except AttributeError: + origin_req_host = req.origin_req_host return HEADRequest(newurl, headers=newheaders, - origin_req_host=req.get_origin_req_host(), + origin_req_host=origin_req_host, unverifiable=True) else: raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp) @@ -159,6 +224,25 @@ class GenericIE(InfoExtractor): raise ExtractorError('Invalid URL protocol') return response + def _extract_rss(self, url, video_id, doc): + playlist_title = doc.find('./channel/title').text + playlist_desc_el = doc.find('./channel/description') + playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text + + entries = [{ + '_type': 'url', + 'url': e.find('link').text, + 'title': e.find('title').text, + } for e in doc.findall('./channel/item')] + + return { + '_type': 'playlist', + 'id': url, + 'title': playlist_title, + 'description': playlist_desc, + 'entries': entries, + } + def _real_extract(self, url): parsed_url = compat_urlparse.urlparse(url) if not parsed_url.scheme: @@ -175,7 +259,7 @@ class GenericIE(InfoExtractor): else: assert ':' in default_search return self.url_result(default_search + url) - video_id = os.path.splitext(url.split('/')[-1])[0] + video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0] self.to_screen('%s: Requesting header' % video_id) @@ -219,6 +303,14 @@ class GenericIE(InfoExtractor): self.report_extraction(video_id) + # Is it an RSS feed? + try: + doc = parse_xml(webpage) + if doc.tag == 'rss': + return self._extract_rss(url, video_id, doc) + except compat_xml_parse_error: + pass + # it's tempting to parse this further, but you would # have to take into account all the variations like # Video Title - Site Name @@ -234,17 +326,27 @@ class GenericIE(InfoExtractor): r'^(?:https?://)?([^/]*)/.*', url, 'video uploader') # Look for BrightCove: - bc_url = BrightcoveIE._extract_brightcove_url(webpage) - if bc_url is not None: + bc_urls = BrightcoveIE._extract_brightcove_urls(webpage) + if bc_urls: self.to_screen('Brightcove video detected.') - surl = smuggle_url(bc_url, {'Referer': url}) - return self.url_result(surl, 'Brightcove') + entries = [{ + '_type': 'url', + 'url': smuggle_url(bc_url, {'Referer': url}), + 'ie_key': 'Brightcove' + } for bc_url in bc_urls] + + return { + '_type': 'playlist', + 'title': video_title, + 'id': video_id, + 'entries': entries, + } # Look for embedded (iframe) Vimeo player mobj = re.search( - r']+?src="((?:https?:)?//player\.vimeo\.com/video/.+?)"', webpage) + r']+?src=(["\'])(?P(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage) if mobj: - player_url = unescapeHTML(mobj.group(1)) + player_url = unescapeHTML(mobj.group('url')) surl = smuggle_url(player_url, {'Referer': url}) return self.url_result(surl, 'Vimeo') @@ -324,11 +426,17 @@ class GenericIE(InfoExtractor): if mobj is not None: return self.url_result(mobj.group(1), 'Mpora') - # Look for embedded Novamov player + # Look for embedded NovaMov player mobj = re.search( r']+?src=(["\'])(?Phttp://(?:(?:embed|www)\.)?novamov\.com/embed\.php.+?)\1', webpage) if mobj is not None: - return self.url_result(mobj.group('url'), 'Novamov') + return self.url_result(mobj.group('url'), 'NovaMov') + + # Look for embedded NowVideo player + mobj = re.search( + r']+?src=(["\'])(?Phttp://(?:(?:embed|www)\.)?nowvideo\.(?:ch|sx|eu)/embed\.php.+?)\1', webpage) + if mobj is not None: + return self.url_result(mobj.group('url'), 'NowVideo') # Look for embedded Facebook player mobj = re.search( @@ -336,12 +444,38 @@ class GenericIE(InfoExtractor): if mobj is not None: return self.url_result(mobj.group('url'), 'Facebook') + # Look for embedded VK player + mobj = re.search(r']+?src=(["\'])(?Phttps?://vk\.com/video_ext\.php.+?)\1', webpage) + if mobj is not None: + return self.url_result(mobj.group('url'), 'VK') + # Look for embedded Huffington Post player mobj = re.search( r']+?src=(["\'])(?Phttps?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage) if mobj is not None: return self.url_result(mobj.group('url'), 'HuffPost') + # Look for embed.ly + mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P[^"\']+)', webpage) + if mobj is not None: + return self.url_result(mobj.group('url')) + mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P[^&]+)', webpage) + if mobj is not None: + return self.url_result(compat_urllib_parse.unquote(mobj.group('url'))) + + # Look for funnyordie embed + matches = re.findall(r']+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage) + if matches: + urlrs = [self.url_result(unescapeHTML(eurl), 'FunnyOrDie') + for eurl in matches] + return self.playlist_result( + urlrs, playlist_id=video_id, playlist_title=video_title) + + # Look for embedded RUTV player + rutv_url = RUTVIE._extract_url(webpage) + if rutv_url: + return self.url_result(rutv_url, 'RUTV') + # Start with something easy: JW Player in SWFObject mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) if mobj is None: @@ -366,6 +500,18 @@ class GenericIE(InfoExtractor): if mobj is None: # HTML5 video mobj = re.search(r'.*?