X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Ftwitter.py;h=5d2b5ec3515277980f2da45a91a0c73fae17923d;hb=2812c24c167dfaeed62737f2b5ba2a5c7c0de97f;hp=2bd5946aca5d424643b4993ead1842544edaf84d;hpb=a2973eb59733c5f86a249c627d654b789020bc7d;p=youtube-dl diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py index 2bd5946ac..5d2b5ec35 100644 --- a/youtube_dl/extractor/twitter.py +++ b/youtube_dl/extractor/twitter.py @@ -4,23 +4,29 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..compat import compat_urllib_request from ..utils import ( float_or_none, xpath_text, remove_end, int_or_none, ExtractorError, + sanitized_Request, ) -class TwitterCardIE(InfoExtractor): +class TwitterBaseIE(InfoExtractor): + def _get_vmap_video_url(self, vmap_url, video_id): + vmap_data = self._download_xml(vmap_url, video_id) + return xpath_text(vmap_data, './/MediaFile').strip() + + +class TwitterCardIE(TwitterBaseIE): IE_NAME = 'twitter:card' _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P\d+)' _TESTS = [ { 'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889', - 'md5': '4fa26a35f9d1bf4b646590ba8e84be19', + # MD5 checksums are different in different places 'info_dict': { 'id': '560070183650213889', 'ext': 'mp4', @@ -42,7 +48,7 @@ class TwitterCardIE(InfoExtractor): }, { 'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977', - 'md5': 'b6f35e8b08a0bec6c8af77a2f4b3a814', + 'md5': 'd4724ffe6d2437886d004fa5de1043b3', 'info_dict': { 'id': 'dq4Oj5quskI', 'ext': 'mp4', @@ -52,6 +58,20 @@ class TwitterCardIE(InfoExtractor): 'uploader': 'OMG! Ubuntu!', 'uploader_id': 'omgubuntu', }, + 'add_ie': ['Youtube'], + }, + { + 'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568', + 'md5': 'ab2745d0b0ce53319a534fccaa986439', + 'info_dict': { + 'id': 'iBb2x00UVlv', + 'ext': 'mp4', + 'upload_date': '20151113', + 'uploader_id': '1189339351084113920', + 'uploader': 'ArsenalTerje', + 'title': 'Vine by ArsenalTerje', + }, + 'add_ie': ['Vine'], } ] @@ -67,25 +87,23 @@ class TwitterCardIE(InfoExtractor): config = None formats = [] for user_agent in USER_AGENTS: - request = compat_urllib_request.Request(url) + request = sanitized_Request(url) request.add_header('User-Agent', user_agent) webpage = self._download_webpage(request, video_id) - youtube_url = self._html_search_regex( - r']+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"', - webpage, 'youtube iframe', default=None) - if youtube_url: - return self.url_result(youtube_url, 'Youtube') + iframe_url = self._html_search_regex( + r']+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"', + webpage, 'video iframe', default=None) + if iframe_url: + return self.url_result(iframe_url) config = self._parse_json(self._html_search_regex( r'data-player-config="([^"]+)"', webpage, 'data player config'), video_id) if 'playlist' not in config: if 'vmapUrl' in config: - vmap_data = self._download_xml(config['vmapUrl'], video_id) - video_url = xpath_text(vmap_data, './/MediaFile').strip() formats.append({ - 'url': video_url, + 'url': self._get_vmap_video_url(config['vmapUrl'], video_id), }) break # same video regardless of UA continue @@ -124,7 +142,7 @@ class TwitterIE(InfoExtractor): _TESTS = [{ 'url': 'https://twitter.com/freethenipple/status/643211948184596480', - 'md5': 'db6612ec5d03355953c3ca9250c97e5e', + # MD5 checksums are different in different places 'info_dict': { 'id': '643211948184596480', 'ext': 'mp4', @@ -147,6 +165,7 @@ class TwitterIE(InfoExtractor): 'uploader': 'Gifs', 'uploader_id': 'giphz', }, + 'expected_warnings': ['height', 'width'], }, { 'url': 'https://twitter.com/starwars/status/665052190608723968', 'md5': '39b7199856dee6cd4432e72c74bc69d4', @@ -194,21 +213,82 @@ class TwitterIE(InfoExtractor): return info mobj = re.search(r'''(?x) - ]+class="animated-gif"[^>]+ - (?:data-height="(?P\d+)")?[^>]+ - (?:data-width="(?P\d+)")?[^>]+ - (?:poster="(?P[^"]+)")?[^>]*>\s* + ]+class="animated-gif"(?P[^>]+)>\s* ]+video-src="(?P[^"]+)" ''', webpage) if mobj: + more_info = mobj.group('more_info') + height = int_or_none(self._search_regex( + r'data-height="(\d+)"', more_info, 'height', fatal=False)) + width = int_or_none(self._search_regex( + r'data-width="(\d+)"', more_info, 'width', fatal=False)) + thumbnail = self._search_regex( + r'poster="([^"]+)"', more_info, 'poster', fatal=False) info.update({ 'id': twid, 'url': mobj.group('url'), - 'height': int_or_none(mobj.group('height')), - 'width': int_or_none(mobj.group('width')), - 'thumbnail': mobj.group('poster'), + 'height': height, + 'width': width, + 'thumbnail': thumbnail, }) return info - raise ExtractorError('There\'s not video in this tweet.') + raise ExtractorError('There\'s no video in this tweet.') + + +class TwitterAmplifyIE(TwitterBaseIE): + IE_NAME = 'twitter:amplify' + _VALID_URL = 'https?://amp\.twimg\.com/v/(?P[0-9a-f\-]{36})' + + _TEST = { + 'url': 'https://amp.twimg.com/v/0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951', + 'md5': '7df102d0b9fd7066b86f3159f8e81bf6', + 'info_dict': { + 'id': '0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951', + 'ext': 'mp4', + 'title': 'Twitter Video', + 'thumbnail': 're:^https?://.*', + }, + } + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + vmap_url = self._html_search_meta( + 'twitter:amplify:vmap', webpage, 'vmap url') + video_url = self._get_vmap_video_url(vmap_url, video_id) + + thumbnails = [] + thumbnail = self._html_search_meta( + 'twitter:image:src', webpage, 'thumbnail', fatal=False) + + def _find_dimension(target): + w = int_or_none(self._html_search_meta( + 'twitter:%s:width' % target, webpage, fatal=False)) + h = int_or_none(self._html_search_meta( + 'twitter:%s:height' % target, webpage, fatal=False)) + return w, h + + if thumbnail: + thumbnail_w, thumbnail_h = _find_dimension('image') + thumbnails.append({ + 'url': thumbnail, + 'width': thumbnail_w, + 'height': thumbnail_h, + }) + + video_w, video_h = _find_dimension('player') + formats = [{ + 'url': video_url, + 'width': video_w, + 'height': video_h, + }] + + return { + 'id': video_id, + 'title': 'Twitter Video', + 'formats': formats, + 'thumbnails': thumbnails, + }