_ Git - youtube-dl/blob - youtube_dl/extractor/twitter.py

   1 from __future__ import unicode_literals
   2
   3 import re
   4
   5 from .common import InfoExtractor
   6 from ..compat import compat_urllib_request
   7 from ..utils import (
   8     float_or_none,
   9     unescapeHTML,
  10 )
  11
  12
  13 class TwitterCardIE(InfoExtractor):
  14     _VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
  15     _TESTS = [
  16         {
  17             'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
  18             'md5': 'a74f50b310c83170319ba16de6955192',
  19             'info_dict': {
  20                 'id': '560070183650213889',
  21                 'ext': 'mp4',
  22                 'title': 'TwitterCard',
  23                 'thumbnail': 're:^https?://.*\.jpg$',
  24                 'duration': 30.033,
  25             }
  26         },
  27         {
  28             'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
  29             'md5': '7ee2a553b63d1bccba97fbed97d9e1c8',
  30             'info_dict': {
  31                 'id': '623160978427936768',
  32                 'ext': 'mp4',
  33                 'title': 'TwitterCard',
  34                 'thumbnail': 're:^https?://.*\.jpg',
  35                 'duration': 80.155,
  36             },
  37         }
  38     ]
  39
  40     def _real_extract(self, url):
  41         video_id = self._match_id(url)
  42
  43         # Different formats served for different User-Agents
  44         USER_AGENTS = [
  45             'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',  # mp4
  46             'Mozilla/5.0 (Windows NT 5.2; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0',  # webm
  47         ]
  48
  49         config = None
  50         formats = []
  51         for user_agent in USER_AGENTS:
  52             request = compat_urllib_request.Request(url)
  53             request.add_header('User-Agent', user_agent)
  54             webpage = self._download_webpage(request, video_id)
  55
  56             config = self._parse_json(
  57                 unescapeHTML(self._search_regex(
  58                     r'data-player-config="([^"]+)"', webpage, 'data player config')),
  59                 video_id)
  60             if 'playlist' not in config:
  61                 if 'vmapUrl' in config:
  62                     webpage = self._download_webpage(config['vmapUrl'], video_id + ' (xml)')
  63                     video_url = self._search_regex(
  64                         r'<MediaFile>\s*<!\[CDATA\[(https?://.+?)\]\]>', webpage, 'data player config (xml)')
  65                     f = {
  66                         'url': video_url,
  67                     }
  68                     ext = re.search(r'\.([a-z0-9]{2,4})(\?.+)?$', video_url)
  69                     if ext:
  70                         f['ext'] = ext.group(1)
  71                     formats.append(f)
  72                     break   # same video regardless of UA
  73                 continue
  74
  75             video_url = config['playlist'][0]['source']
  76
  77             f = {
  78                 'url': video_url,
  79             }
  80
  81             m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
  82             if m:
  83                 f.update({
  84                     'width': int(m.group('width')),
  85                     'height': int(m.group('height')),
  86                 })
  87             formats.append(f)
  88         self._sort_formats(formats)
  89
  90         thumbnail = config.get('posterImageUrl')
  91         duration = float_or_none(config.get('duration'))
  92
  93         return {
  94             'id': video_id,
  95             'title': 'TwitterCard',
  96             'thumbnail': thumbnail,
  97             'duration': duration,
  98             'formats': formats,
  99         }
 100
 101
 102 class TwitterIE(TwitterCardIE):
 103     _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
 104
 105     _TESTS = [{
 106         'url': 'https://m.twitter.com/thereaIbanksy/status/614301758345490432',
 107         'md5': '8bbccb487bd7a31349b775915fcd412f',
 108         'info_dict': {
 109             'id': '614301758345490432',
 110             'ext': 'mp4',
 111             'title': 'thereaIbanksy - This time lapse is so pretty \U0001f60d\U0001f60d',
 112             'thumbnail': 're:^https?://.*\.jpg',
 113             'duration': 29.5,
 114             'description': 'banksy on Twitter: "This time lapse is so pretty \U0001f60d\U0001f60d http://t.co/QB8DDbqiR1"',
 115             'uploader': 'banksy',
 116             'uploader_id': 'thereaIbanksy',
 117         },
 118     }]
 119
 120     def _real_extract(self, url):
 121         id = self._match_id(url)
 122         username, twid = re.match(r'([^/]+)/status/(\d+)', id).groups()
 123         name = username
 124         url = re.sub(r'https?://(m|mobile)\.', 'https://', url)
 125         webpage = self._download_webpage(url, 'tweet: ' + url)
 126         description = unescapeHTML(self._search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title'))
 127         title = description.replace('\n', ' ')
 128         splitdesc = re.match(r'^(.+?)\s*on Twitter:\s* "(.+?)"$', title)
 129         if splitdesc:
 130             name, title = splitdesc.groups()
 131         title = re.sub(r'\s*https?://[^ ]+', '', title)  # strip  'https -_t.co_BJYgOjSeGA' junk from filenames
 132         card_id = self._search_regex(r'["\']/i/cards/tfw/v1/(\d+)', webpage, '/i/card/...')
 133         card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
 134         return {
 135             '_type': 'url_transparent',
 136             'ie_key': 'TwitterCard',
 137             'uploader_id': username,
 138             'uploader': name,
 139             'url': card_url,
 140             'webpage_url': url,
 141             'description': description,
 142             'title': username + ' - ' + title,
 143         }