From: Philipp Hagemeister Date: Mon, 17 Mar 2014 13:31:22 +0000 (+0100) Subject: Merge remote-tracking branch 'origin/wheels' X-Git-Url: http://git.bitcoin.ninja/index.cgi?p=youtube-dl;a=commitdiff_plain;h=6f50f63382d2fbf0fddc59de07b9843b82123990;hp=4056ad8f3634f72661e4e856178b6db97af3aa7f Merge remote-tracking branch 'origin/wheels' --- diff --git a/test/helper.py b/test/helper.py index b1f421ac5..17de951c5 100644 --- a/test/helper.py +++ b/test/helper.py @@ -71,7 +71,7 @@ class FakeYDL(YoutubeDL): old_report_warning(message) self.report_warning = types.MethodType(report_warning, self) -def get_testcases(): +def gettestcases(): for ie in youtube_dl.extractor.gen_extractors(): t = getattr(ie, '_TEST', None) if t: diff --git a/test/test_all_urls.py b/test/test_all_urls.py index 047e84f19..3ce0b910a 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -9,7 +9,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import get_testcases +from test.helper import gettestcases from youtube_dl.extractor import ( FacebookIE, @@ -105,7 +105,7 @@ class TestAllURLsMatching(unittest.TestCase): def test_no_duplicates(self): ies = gen_extractors() - for tc in get_testcases(): + for tc in gettestcases(): url = tc['url'] for ie in ies: if type(ie).__name__ in ('GenericIE', tc['name'] + 'IE'): diff --git a/test/test_download.py b/test/test_download.py index ca8c82f71..8fccdaf9e 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -8,7 +8,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import ( get_params, - get_testcases, + gettestcases, try_rm, md5, report_warning @@ -51,7 +51,7 @@ def _file_md5(fn): with open(fn, 'rb') as f: return hashlib.md5(f.read()).hexdigest() -defs = get_testcases() +defs = gettestcases() class TestDownload(unittest.TestCase): diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 313414e7d..bbdb04069 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -196,6 +196,7 @@ from .rutube import ( RutubeMovieIE, RutubePersonIE, ) +from .rutv import RUTVIE from .savefrom import SaveFromIE from .servingsys import ServingSysIE from .sina import SinaIE @@ -251,8 +252,8 @@ from .ustream import UstreamIE, UstreamChannelIE from .vbox7 import Vbox7IE from .veehd import VeeHDIE from .veoh import VeohIE +from .vesti import VestiIE from .vevo import VevoIE -from .vgtrk import VGTRKIE from .vice import ViceIE from .viddler import ViddlerIE from .videobam import VideoBamIE diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 66e189da1..71b9c541e 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -24,6 +24,7 @@ from ..utils import ( ) from .brightcove import BrightcoveIE from .ooyala import OoyalaIE +from .rutv import RUTVIE class GenericIE(InfoExtractor): @@ -143,8 +144,22 @@ class GenericIE(InfoExtractor): 'ext': 'mp4', 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama', 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.', - } + }, }, + # RUTV embed + { + 'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html', + 'info_dict': { + 'id': '776940', + 'ext': 'mp4', + 'title': 'Охотское море стало целиком российским', + 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43', + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + } ] def report_download_webpage(self, video_id): @@ -451,6 +466,11 @@ class GenericIE(InfoExtractor): return self.playlist_result( urlrs, playlist_id=video_id, playlist_title=video_title) + # Look for embedded RUTV player + rutv_url = RUTVIE._extract_url(webpage) + if rutv_url: + return self.url_result(rutv_url, 'RUTV') + # Start with something easy: JW Player in SWFObject mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) if mobj is None: diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py new file mode 100644 index 000000000..5c38cbc02 --- /dev/null +++ b/youtube_dl/extractor/rutv.py @@ -0,0 +1,183 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + int_or_none +) + + +class RUTVIE(InfoExtractor): + IE_DESC = 'RUTV.RU' + _VALID_URL = r'https?://player\.(?:rutv\.ru|vgtrk\.com)/(?:flash2v/container\.swf\?id=|iframe/(?Pswf|video|live)/id/)(?P\d+)' + + _TESTS = [ + { + 'url': 'http://player.rutv.ru/flash2v/container.swf?id=774471&sid=kultura&fbv=true&isPlay=true&ssl=false&i=560&acc_video_id=episode_id/972347/video_id/978186/brand_id/31724', + 'info_dict': { + 'id': '774471', + 'ext': 'mp4', + 'title': 'Монологи на все времена', + 'description': 'md5:18d8b5e6a41fb1faa53819471852d5d5', + 'duration': 2906, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'https://player.vgtrk.com/flash2v/container.swf?id=774016&sid=russiatv&fbv=true&isPlay=true&ssl=false&i=560&acc_video_id=episode_id/972098/video_id/977760/brand_id/57638', + 'info_dict': { + 'id': '774016', + 'ext': 'mp4', + 'title': 'Чужой в семье Сталина', + 'description': '', + 'duration': 2539, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://player.rutv.ru/iframe/swf/id/766888/sid/hitech/?acc_video_id=4000', + 'info_dict': { + 'id': '766888', + 'ext': 'mp4', + 'title': 'Вести.net: интернет-гиганты начали перетягивание программных "одеял"', + 'description': 'md5:65ddd47f9830c4f42ed6475f8730c995', + 'duration': 279, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://player.rutv.ru/iframe/video/id/771852/start_zoom/true/showZoomBtn/false/sid/russiatv/?acc_video_id=episode_id/970443/video_id/975648/brand_id/5169', + 'info_dict': { + 'id': '771852', + 'ext': 'mp4', + 'title': 'Прямой эфир. Жертвы загадочной болезни: смерть от старости в 17 лет', + 'description': 'md5:b81c8c55247a4bd996b43ce17395b2d8', + 'duration': 3096, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://player.rutv.ru/iframe/live/id/51499/showZoomBtn/false/isPlay/true/sid/sochi2014', + 'info_dict': { + 'id': '51499', + 'ext': 'flv', + 'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ', + 'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c', + }, + 'params': { + # rtmp download + 'skip_download': True, + }, + 'skip': 'Translation has finished', + }, + ] + + @classmethod + def _extract_url(cls, webpage): + mobj = re.search( + r']+?src=(["\'])(?Phttps?://player\.rutv\.ru/iframe/(?:swf|video|live)/id/.+?)\1', webpage) + if mobj: + return mobj.group('url') + + mobj = re.search( + r']+?property=(["\'])og:video\1[^>]+?content=(["\'])(?Phttp://player\.(?:rutv\.ru|vgtrk\.com)/flash2v/container\.swf\?id=.+?\2)', + webpage) + if mobj: + return mobj.group('url') + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + video_type = mobj.group('type') + + if not video_type or video_type == 'swf': + video_type = 'video' + + json_data = self._download_json( + 'http://player.rutv.ru/iframe/%splay/id/%s' % ('live-' if video_type == 'live' else '', video_id), + video_id, 'Downloading JSON') + + if json_data['errors']: + raise ExtractorError('%s said: %s' % (self.IE_NAME, json_data['errors']), expected=True) + + playlist = json_data['data']['playlist'] + medialist = playlist['medialist'] + media = medialist[0] + + if media['errors']: + raise ExtractorError('%s said: %s' % (self.IE_NAME, media['errors']), expected=True) + + view_count = playlist.get('count_views') + priority_transport = playlist['priority_transport'] + + thumbnail = media['picture'] + width = int_or_none(media['width']) + height = int_or_none(media['height']) + description = media['anons'] + title = media['title'] + duration = int_or_none(media.get('duration')) + + formats = [] + + for transport, links in media['sources'].items(): + for quality, url in links.items(): + if transport == 'rtmp': + mobj = re.search(r'^(?Prtmp://[^/]+/(?P.+))/(?P.+)$', url) + if not mobj: + continue + fmt = { + 'url': mobj.group('url'), + 'play_path': mobj.group('playpath'), + 'app': mobj.group('app'), + 'page_url': 'http://player.rutv.ru', + 'player_url': 'http://player.rutv.ru/flash2v/osmf.swf?i=22', + 'rtmp_live': True, + 'ext': 'flv', + 'vbr': int(quality), + } + elif transport == 'm3u8': + fmt = { + 'url': url, + 'ext': 'mp4', + } + else: + fmt = { + 'url': url + } + fmt.update({ + 'width': width, + 'height': height, + 'format_id': '%s-%s' % (transport, quality), + 'preference': -1 if priority_transport == transport else -2, + }) + formats.append(fmt) + + if not formats: + raise ExtractorError('No media links available for %s' % video_id) + + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'view_count': view_count, + 'duration': duration, + 'formats': formats, + } \ No newline at end of file diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py index cf10be2d0..3968b718e 100644 --- a/youtube_dl/extractor/ted.py +++ b/youtube_dl/extractor/ted.py @@ -93,11 +93,14 @@ class TEDIE(SubtitlesInfoExtractor): self._list_available_subtitles(video_id, talk_info) return + thumbnail = talk_info['thumb'] + if not thumbnail.startswith('http'): + thumbnail = 'http://' + thumbnail return { 'id': video_id, 'title': talk_info['title'], 'uploader': talk_info['speaker'], - 'thumbnail': talk_info['thumb'], + 'thumbnail': thumbnail, 'description': self._og_search_description(webpage), 'subtitles': video_subtitles, 'formats': formats, diff --git a/youtube_dl/extractor/vesti.py b/youtube_dl/extractor/vesti.py new file mode 100644 index 000000000..27f9acb67 --- /dev/null +++ b/youtube_dl/extractor/vesti.py @@ -0,0 +1,121 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ExtractorError +from .rutv import RUTVIE + + +class VestiIE(InfoExtractor): + IE_DESC = 'Вести.Ru' + _VALID_URL = r'http://(?:.+?\.)?vesti\.ru/(?P.+)' + + _TESTS = [ + { + 'url': 'http://www.vesti.ru/videos?vid=575582&cid=1', + 'info_dict': { + 'id': '765035', + 'ext': 'mp4', + 'title': 'Вести.net: биткоины в России не являются законными', + 'description': 'md5:d4bb3859dc1177b28a94c5014c35a36b', + 'duration': 302, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://www.vesti.ru/doc.html?id=1349233', + 'info_dict': { + 'id': '773865', + 'ext': 'mp4', + 'title': 'Участники митинга штурмуют Донецкую областную администрацию', + 'description': 'md5:1a160e98b3195379b4c849f2f4958009', + 'duration': 210, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://www.vesti.ru/only_video.html?vid=576180', + 'info_dict': { + 'id': '766048', + 'ext': 'mp4', + 'title': 'США заморозило, Британию затопило', + 'description': 'md5:f0ed0695ec05aed27c56a70a58dc4cc1', + 'duration': 87, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://hitech.vesti.ru/news/view/id/4000', + 'info_dict': { + 'id': '766888', + 'ext': 'mp4', + 'title': 'Вести.net: интернет-гиганты начали перетягивание программных "одеял"', + 'description': 'md5:65ddd47f9830c4f42ed6475f8730c995', + 'duration': 279, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + }, + { + 'url': 'http://sochi2014.vesti.ru/video/index/video_id/766403', + 'info_dict': { + 'id': '766403', + 'ext': 'mp4', + 'title': 'XXII зимние Олимпийские игры. Российские хоккеисты стартовали на Олимпиаде с победы', + 'description': 'md5:55805dfd35763a890ff50fa9e35e31b3', + 'duration': 271, + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + 'skip': 'Blocked outside Russia', + }, + { + 'url': 'http://sochi2014.vesti.ru/live/play/live_id/301', + 'info_dict': { + 'id': '51499', + 'ext': 'flv', + 'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ', + 'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c', + }, + 'params': { + # rtmp download + 'skip_download': True, + }, + 'skip': 'Translation has finished' + }, + ] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + page = self._download_webpage(url, video_id, 'Downloading page') + + mobj = re.search( + r']+?property="og:video"[^>]+?content="http://www\.vesti\.ru/i/flvplayer_videoHost\.swf\?vid=(?P\d+)', + page) + if mobj: + video_id = mobj.group('id') + page = self._download_webpage('http://www.vesti.ru/only_video.html?vid=%s' % video_id, video_id, + 'Downloading video page') + + rutv_url = RUTVIE._extract_url(page) + if rutv_url: + return self.url_result(rutv_url, 'RUTV') + + raise ExtractorError('No video found', expected=True) \ No newline at end of file diff --git a/youtube_dl/extractor/vgtrk.py b/youtube_dl/extractor/vgtrk.py deleted file mode 100644 index 429b8bc72..000000000 --- a/youtube_dl/extractor/vgtrk.py +++ /dev/null @@ -1,278 +0,0 @@ -# encoding: utf-8 -from __future__ import unicode_literals - -import re - -from .common import InfoExtractor -from ..utils import ( - ExtractorError, - int_or_none -) - - -class VGTRKIE(InfoExtractor): - IE_DESC = 'ВГТРК' - _VALID_URL = r'http://(?:.+?\.)?(?:vesti\.ru|russia2?\.tv|tvkultura\.ru|rutv\.ru)/(?P.+)' - - _TESTS = [ - { - 'url': 'http://www.vesti.ru/videos?vid=575582&cid=1', - 'info_dict': { - 'id': '765035', - 'ext': 'mp4', - 'title': 'Вести.net: биткоины в России не являются законными', - 'description': 'md5:d4bb3859dc1177b28a94c5014c35a36b', - 'duration': 302, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://www.vesti.ru/doc.html?id=1349233', - 'info_dict': { - 'id': '773865', - 'ext': 'mp4', - 'title': 'Участники митинга штурмуют Донецкую областную администрацию', - 'description': 'md5:1a160e98b3195379b4c849f2f4958009', - 'duration': 210, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://www.vesti.ru/only_video.html?vid=576180', - 'info_dict': { - 'id': '766048', - 'ext': 'mp4', - 'title': 'США заморозило, Британию затопило', - 'description': 'md5:f0ed0695ec05aed27c56a70a58dc4cc1', - 'duration': 87, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://hitech.vesti.ru/news/view/id/4000', - 'info_dict': { - 'id': '766888', - 'ext': 'mp4', - 'title': 'Вести.net: интернет-гиганты начали перетягивание программных "одеял"', - 'description': 'md5:65ddd47f9830c4f42ed6475f8730c995', - 'duration': 279, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://sochi2014.vesti.ru/video/index/video_id/766403', - 'info_dict': { - 'id': '766403', - 'ext': 'mp4', - 'title': 'XXII зимние Олимпийские игры. Российские хоккеисты стартовали на Олимпиаде с победы', - 'description': 'md5:55805dfd35763a890ff50fa9e35e31b3', - 'duration': 271, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - 'skip': 'Blocked outside Russia', - }, - { - 'url': 'http://sochi2014.vesti.ru/live/play/live_id/301', - 'info_dict': { - 'id': '51499', - 'ext': 'flv', - 'title': 'Сочи-2014. Биатлон. Индивидуальная гонка. Мужчины ', - 'description': 'md5:9e0ed5c9d2fa1efbfdfed90c9a6d179c', - }, - 'params': { - # rtmp download - 'skip_download': True, - }, - 'skip': 'Translation has finished' - }, - { - 'url': 'http://russia.tv/video/show/brand_id/5169/episode_id/970443/video_id/975648', - 'info_dict': { - 'id': '771852', - 'ext': 'mp4', - 'title': 'Прямой эфир. Жертвы загадочной болезни: смерть от старости в 17 лет', - 'description': 'md5:b81c8c55247a4bd996b43ce17395b2d8', - 'duration': 3096, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://russia.tv/brand/show/brand_id/57638', - 'info_dict': { - 'id': '774016', - 'ext': 'mp4', - 'title': 'Чужой в семье Сталина', - 'description': '', - 'duration': 2539, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://2.russia.tv/video/show/brand_id/48863/episode_id/972920/video_id/978667/viewtype/picture', - 'info_dict': { - 'id': '775081', - 'ext': 'mp4', - 'title': 'XXII зимние Олимпийские игры. Россияне заняли весь пьедестал в лыжных гонках', - 'description': 'md5:15d3741dd8d04b203fbc031c6a47fb0f', - 'duration': 101, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - 'skip': 'Blocked outside Russia', - }, - { - 'url': 'http://tvkultura.ru/video/show/brand_id/31724/episode_id/972347/video_id/978186', - 'info_dict': { - 'id': '774471', - 'ext': 'mp4', - 'title': 'Монологи на все времена', - 'description': 'md5:18d8b5e6a41fb1faa53819471852d5d5', - 'duration': 2906, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - { - 'url': 'http://rutv.ru/brand/show/id/6792/channel/75', - 'info_dict': { - 'id': '125521', - 'ext': 'mp4', - 'title': 'Грустная дама червей. Х/ф', - 'description': '', - 'duration': 4882, - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - }, - ] - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') - - page = self._download_webpage(url, video_id, 'Downloading page') - - mobj = re.search( - r'[^/]+)/id/(?P\d+)[^"]*".*?>', - page) - - if not mobj: - raise ExtractorError('No media found', expected=True) - - video_type = mobj.group('type') - video_id = mobj.group('id') - - json_data = self._download_json( - 'http://player.rutv.ru/iframe/%splay/id/%s' % ('live-' if video_type == 'live' else '', video_id), - video_id, 'Downloading JSON') - - if json_data['errors']: - raise ExtractorError('vesti returned error: %s' % json_data['errors'], expected=True) - - playlist = json_data['data']['playlist'] - medialist = playlist['medialist'] - media = medialist[0] - - if media['errors']: - raise ExtractorError('vesti returned error: %s' % media['errors'], expected=True) - - view_count = playlist.get('count_views') - priority_transport = playlist['priority_transport'] - - thumbnail = media['picture'] - width = int_or_none(media['width']) - height = int_or_none(media['height']) - description = media['anons'] - title = media['title'] - duration = int_or_none(media.get('duration')) - - formats = [] - - for transport, links in media['sources'].items(): - for quality, url in links.items(): - if transport == 'rtmp': - mobj = re.search(r'^(?Prtmp://[^/]+/(?P.+))/(?P.+)$', url) - if not mobj: - continue - fmt = { - 'url': mobj.group('url'), - 'play_path': mobj.group('playpath'), - 'app': mobj.group('app'), - 'page_url': 'http://player.rutv.ru', - 'player_url': 'http://player.rutv.ru/flash2v/osmf.swf?i=22', - 'rtmp_live': True, - 'ext': 'flv', - 'vbr': int(quality), - } - elif transport == 'm3u8': - fmt = { - 'url': url, - 'ext': 'mp4', - } - else: - fmt = { - 'url': url - } - fmt.update({ - 'width': width, - 'height': height, - 'format_id': '%s-%s' % (transport, quality), - 'preference': -1 if priority_transport == transport else -2, - }) - formats.append(fmt) - - if not formats: - raise ExtractorError('No media links available for %s' % video_id) - - self._sort_formats(formats) - - return { - 'id': video_id, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - 'view_count': view_count, - 'duration': duration, - 'formats': formats, - } \ No newline at end of file