X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Fgeneric.py;h=595ba268fe5c223108ede65f520c78a77d58008c;hb=91346358b0316d1fae2c030dce950fc1e643d045;hp=e0d51996ded5f1604bebe5d0c03c1dbc815f31ce;hpb=c3f51436bf3b36572b403d3240c947e6961bd25f;p=youtube-dl diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index e0d51996d..595ba268f 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import os import re +import xml.etree.ElementTree from .common import InfoExtractor from .youtube import YoutubeIE @@ -12,6 +13,7 @@ from ..utils import ( compat_urllib_parse, compat_urllib_request, compat_urlparse, + compat_xml_parse_error, ExtractorError, HEADRequest, @@ -38,18 +40,6 @@ class GenericIE(InfoExtractor): 'title': 'R\u00e9gis plante sa Jeep', } }, - # embedded vimeo video - { - 'add_ie': ['Vimeo'], - 'url': 'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references', - 'file': '22444065.mp4', - 'md5': '2903896e23df39722c33f015af0666e2', - 'info_dict': { - 'title': 'ACCU 2011: Move Semantics,Perfect Forwarding, and Rvalue references- Scott Meyers- 13/04/2011', - 'uploader_id': 'skillsmatter', - 'uploader': 'Skills Matter', - } - }, # bandcamp page with custom domain { 'add_ie': ['Bandcamp'], @@ -93,10 +83,10 @@ class GenericIE(InfoExtractor): # Direct link to a video { 'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4', - 'file': 'trailer.mp4', 'md5': '67d406c2bcb6af27fa886f31aa934bbe', 'info_dict': { 'id': 'trailer', + 'ext': 'mp4', 'title': 'trailer', 'upload_date': '20100513', } @@ -104,7 +94,6 @@ class GenericIE(InfoExtractor): # ooyala video { 'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219', - 'file': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ.mp4', 'md5': '5644c6ca5d5782c1d0d350dad9bd840c', 'info_dict': { 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ', @@ -112,6 +101,22 @@ class GenericIE(InfoExtractor): 'title': '2cc213299525360.mov', # that's what we get }, }, + # google redirect + { + 'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE', + 'info_dict': { + 'id': 'cmQHVoWB5FY', + 'ext': 'mp4', + 'upload_date': '20130224', + 'uploader_id': 'TheVerge', + 'description': 'Chris Ziegler takes a look at the Alcatel OneTouch Fire and the ZTE Open; two of the first Firefox OS handsets to be officially announced.', + 'uploader': 'The Verge', + 'title': 'First Firefox OS phones side-by-side', + }, + 'params': { + 'skip_download': False, + } + } ] def report_download_webpage(self, video_id): @@ -171,6 +176,25 @@ class GenericIE(InfoExtractor): raise ExtractorError('Invalid URL protocol') return response + def _extract_rss(self, url, video_id, doc): + playlist_title = doc.find('./channel/title').text + playlist_desc_el = doc.find('./channel/description') + playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text + + entries = [{ + '_type': 'url', + 'url': e.find('link').text, + 'title': e.find('title').text, + } for e in doc.findall('./channel/item')] + + return { + '_type': 'playlist', + 'id': url, + 'title': playlist_title, + 'description': playlist_desc, + 'entries': entries, + } + def _real_extract(self, url): parsed_url = compat_urlparse.urlparse(url) if not parsed_url.scheme: @@ -231,6 +255,14 @@ class GenericIE(InfoExtractor): self.report_extraction(video_id) + # Is it an RSS feed? + try: + doc = xml.etree.ElementTree.fromstring(webpage.encode('utf-8')) + if doc.tag == 'rss': + return self._extract_rss(url, video_id, doc) + except compat_xml_parse_error: + pass + # it's tempting to parse this further, but you would # have to take into account all the variations like # Video Title - Site Name @@ -246,11 +278,21 @@ class GenericIE(InfoExtractor): r'^(?:https?://)?([^/]*)/.*', url, 'video uploader') # Look for BrightCove: - bc_url = BrightcoveIE._extract_brightcove_url(webpage) - if bc_url is not None: + bc_urls = BrightcoveIE._extract_brightcove_urls(webpage) + if bc_urls: self.to_screen('Brightcove video detected.') - surl = smuggle_url(bc_url, {'Referer': url}) - return self.url_result(surl, 'Brightcove') + entries = [{ + '_type': 'url', + 'url': smuggle_url(bc_url, {'Referer': url}), + 'ie_key': 'Brightcove' + } for bc_url in bc_urls] + + return { + '_type': 'playlist', + 'title': video_title, + 'id': video_id, + 'entries': entries, + } # Look for embedded (iframe) Vimeo player mobj = re.search( @@ -336,11 +378,17 @@ class GenericIE(InfoExtractor): if mobj is not None: return self.url_result(mobj.group(1), 'Mpora') - # Look for embedded Novamov player + # Look for embedded NovaMov player mobj = re.search( r']+?src=(["\'])(?Phttp://(?:(?:embed|www)\.)?novamov\.com/embed\.php.+?)\1', webpage) if mobj is not None: - return self.url_result(mobj.group('url'), 'Novamov') + return self.url_result(mobj.group('url'), 'NovaMov') + + # Look for embedded NowVideo player + mobj = re.search( + r']+?src=(["\'])(?Phttp://(?:(?:embed|www)\.)?nowvideo\.(?:ch|sx|eu)/embed\.php.+?)\1', webpage) + if mobj is not None: + return self.url_result(mobj.group('url'), 'NowVideo') # Look for embedded Facebook player mobj = re.search( @@ -358,7 +406,7 @@ class GenericIE(InfoExtractor): mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) if mobj is None: # Look for gorilla-vid style embedding - mobj = re.search(r'(?s)jw_plugins.*?file:\s*["\'](.*?)["\']', webpage) + mobj = re.search(r'(?s)(?:jw_plugins|JWPlayerOptions).*?file\s*:\s*["\'](.*?)["\']', webpage) if mobj is None: # Broaden the search a little bit mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage) @@ -378,6 +426,18 @@ class GenericIE(InfoExtractor): if mobj is None: # HTML5 video mobj = re.search(r'.*?