[twitter] Don't check /cards/ URLs

[youtube-dl] / youtube_dl / extractor / twitter.py

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 97a4a5948aceb84e9378dcad5bdd6e5cc95c3555..ea673054fdc7135a203cca8db00dc128344b0829 100644 (file)
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -102,20 +102,26 @@ class TwitterCardIE(TwitterBaseIE):
             r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
             video_id)
 
-        playlist = config.get('playlist')
-        if playlist:
-            video_url = playlist[0]['source']
-
-            f = {
-                'url': video_url,
-            }
+        if config.get('source_type') == 'vine':
+            return self.url_result(config['player_url'], 'Vine')
 
+        def _search_dimensions_in_video_url(a_format, video_url):
             m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
             if m:
-                f.update({
+                a_format.update({
                     'width': int(m.group('width')),
                     'height': int(m.group('height')),
                 })
+
+        video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')
+
+        if video_url:
+            f = {
+                'url': video_url,
+            }
+
+            _search_dimensions_in_video_url(f, video_url)
+
             formats.append(f)
 
         vmap_url = config.get('vmapUrl') or config.get('vmap_url')
@@ -124,7 +130,12 @@ class TwitterCardIE(TwitterBaseIE):
                 'url': self._get_vmap_video_url(vmap_url, video_id),
             })
 
-        media_info = config.get('status', {}).get('entities', [{}])[0].get('mediaInfo', {})
+        media_info = None
+
+        for entity in config.get('status', {}).get('entities', []):
+            if 'mediaInfo' in entity:
+                media_info = entity['mediaInfo']
+
         if media_info:
             for media_variant in media_info['variants']:
                 media_url = media_variant['url']
@@ -143,6 +154,8 @@ class TwitterCardIE(TwitterBaseIE):
                     if not a_format['vbr']:
                         del a_format['vbr']
 
+                    _search_dimensions_in_video_url(a_format, media_url)
+
                     formats.append(a_format)
 
             duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
@@ -174,7 +187,6 @@ class TwitterIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'FREE THE NIPPLE - FTN supporters on Hollywood Blvd today!',
             'thumbnail': 're:^https?://.*\.jpg',
-            'duration': 12.922,
             'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
             'uploader': 'FREE THE NIPPLE',
             'uploader_id': 'freethenipple',
@@ -221,6 +233,44 @@ class TwitterIE(InfoExtractor):
             # Test case of TwitterCardIE
             'skip_download': True,
         },
+    }, {
+        'url': 'https://twitter.com/jaydingeer/status/700207533655363584',
+        'md5': '',
+        'info_dict': {
+            'id': '700207533655363584',
+            'ext': 'mp4',
+            'title': 'jay - BEAT PROD: @suhmeduh #Damndaniel',
+            'description': 'jay on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'uploader': 'jay',
+            'uploader_id': 'jaydingeer',
+        },
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        },
+    }, {
+        'url': 'https://twitter.com/Filmdrunk/status/713801302971588609',
+        'md5': '89a15ed345d13b86e9a5a5e051fa308a',
+        'info_dict': {
+            'id': 'MIOxnrUteUd',
+            'ext': 'mp4',
+            'title': 'Dr.Pepperの飲み方 #japanese #バカ #ドクペ #電動ガン',
+            'uploader': 'TAKUMA',
+            'uploader_id': '1004126642786242560',
+            'upload_date': '20140615',
+        },
+        'add_ie': ['Vine'],
+    }, {
+        'url': 'https://twitter.com/captainamerica/status/719944021058060289',
+        # md5 constantly changes
+        'info_dict': {
+            'id': '719944021058060289',
+            'ext': 'mp4',
+            'title': 'Captain America - @King0fNerd Are you sure you made the right choice? Find out in theaters.',
+            'description': 'Captain America on Twitter: "@King0fNerd Are you sure you made the right choice? Find out in theaters. https://t.co/GpgYi9xMJI"',
+            'uploader_id': 'captainamerica',
+            'uploader': 'Captain America',
+        },
     }]
 
     def _real_extract(self, url):
@@ -245,17 +295,6 @@ class TwitterIE(InfoExtractor):
             'title': username + ' - ' + title,
         }
 
-        card_id = self._search_regex(
-            r'["\']/i/cards/tfw/v1/(\d+)', webpage, 'twitter card url', default=None)
-        if card_id:
-            card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
-            info.update({
-                '_type': 'url_transparent',
-                'ie_key': 'TwitterCard',
-                'url': card_url,
-            })
-            return info
-
         mobj = re.search(r'''(?x)
             <video[^>]+class="animated-gif"(?P<more_info>[^>]+)>\s*
                 <source[^>]+video-src="(?P<url>[^"]+)"