Merge branch 'polskie-radio-programme' of https://github.com/JakubAdamWieczorek/youtu...
[youtube-dl] / youtube_dl / extractor / twitter.py
index 67762a003075c71c8067bd763772642d77888d93..c5a5843b6107b993b9a36f902e32c2cb97611dd1 100644 (file)
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
     float_or_none,
     xpath_text,
     remove_end,
@@ -52,7 +53,7 @@ class TwitterCardIE(TwitterBaseIE):
                 'id': 'dq4Oj5quskI',
                 'ext': 'mp4',
                 'title': 'Ubuntu 11.10 Overview',
-                'description': 'Take a quick peek at what\'s new and improved in Ubuntu 11.10.\n\nOnce installed take a look at 10 Things to Do After Installing: http://www.omgubuntu.co.uk/2011/10/10-things-to-do-after-installing-ubuntu-11-10/',
+                'description': 'Take a quick peek at what\'s new and improved in Ubuntu 11.10.\n\nOnce installed take a look at 10 Things to Do After Installing: http://www.omgubuntu.co.uk/2011/10/10...',
                 'upload_date': '20111013',
                 'uploader': 'OMG! Ubuntu!',
                 'uploader_id': 'omgubuntu',
@@ -102,21 +103,30 @@ class TwitterCardIE(TwitterBaseIE):
             r'data-(?:player-)?config="([^"]+)"', webpage, 'data player config'),
             video_id)
 
-        playlist = config.get('playlist')
-        if playlist:
-            video_url = playlist[0]['source']
-
-            f = {
-                'url': video_url,
-            }
+        if config.get('source_type') == 'vine':
+            return self.url_result(config['player_url'], 'Vine')
 
+        def _search_dimensions_in_video_url(a_format, video_url):
             m = re.search(r'/(?P<width>\d+)x(?P<height>\d+)/', video_url)
             if m:
-                f.update({
+                a_format.update({
                     'width': int(m.group('width')),
                     'height': int(m.group('height')),
                 })
-            formats.append(f)
+
+        video_url = config.get('video_url') or config.get('playlist', [{}])[0].get('source')
+
+        if video_url:
+            if determine_ext(video_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, ext='mp4', m3u8_id='hls'))
+            else:
+                f = {
+                    'url': video_url,
+                }
+
+                _search_dimensions_in_video_url(f, video_url)
+
+                formats.append(f)
 
         vmap_url = config.get('vmapUrl') or config.get('vmap_url')
         if vmap_url:
@@ -148,6 +158,8 @@ class TwitterCardIE(TwitterBaseIE):
                     if not a_format['vbr']:
                         del a_format['vbr']
 
+                    _search_dimensions_in_video_url(a_format, media_url)
+
                     formats.append(a_format)
 
             duration = float_or_none(media_info.get('duration', {}).get('nanos'), scale=1e9)
@@ -179,7 +191,6 @@ class TwitterIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'FREE THE NIPPLE - FTN supporters on Hollywood Blvd today!',
             'thumbnail': 're:^https?://.*\.jpg',
-            'duration': 12.922,
             'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
             'uploader': 'FREE THE NIPPLE',
             'uploader_id': 'freethenipple',
@@ -200,6 +211,7 @@ class TwitterIE(InfoExtractor):
             'uploader_id': 'giphz',
         },
         'expected_warnings': ['height', 'width'],
+        'skip': 'Account suspended',
     }, {
         'url': 'https://twitter.com/starwars/status/665052190608723968',
         'md5': '39b7199856dee6cd4432e72c74bc69d4',
@@ -232,15 +244,40 @@ class TwitterIE(InfoExtractor):
         'info_dict': {
             'id': '700207533655363584',
             'ext': 'mp4',
-            'title': 'jay - BEAT PROD: @suhmeduh #Damndaniel',
-            'description': 'jay on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
+            'title': 'Donte The Dumbass - BEAT PROD: @suhmeduh #Damndaniel',
+            'description': 'Donte The Dumbass on Twitter: "BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ"',
             'thumbnail': 're:^https?://.*\.jpg',
-            'uploader': 'jay',
+            'uploader': 'Donte The Dumbass',
             'uploader_id': 'jaydingeer',
         },
         'params': {
             'skip_download': True,  # requires ffmpeg
         },
+    }, {
+        'url': 'https://twitter.com/Filmdrunk/status/713801302971588609',
+        'md5': '89a15ed345d13b86e9a5a5e051fa308a',
+        'info_dict': {
+            'id': 'MIOxnrUteUd',
+            'ext': 'mp4',
+            'title': 'Dr.Pepperの飲み方 #japanese #バカ #ドクペ #電動ガン',
+            'uploader': 'TAKUMA',
+            'uploader_id': '1004126642786242560',
+            'upload_date': '20140615',
+        },
+        'add_ie': ['Vine'],
+    }, {
+        'url': 'https://twitter.com/captainamerica/status/719944021058060289',
+        'info_dict': {
+            'id': '719944021058060289',
+            'ext': 'mp4',
+            'title': 'Captain America - @King0fNerd Are you sure you made the right choice? Find out in theaters.',
+            'description': 'Captain America on Twitter: "@King0fNerd Are you sure you made the right choice? Find out in theaters. https://t.co/GpgYi9xMJI"',
+            'uploader_id': 'captainamerica',
+            'uploader': 'Captain America',
+        },
+        'params': {
+            'skip_download': True,  # requires ffmpeg
+        },
     }]
 
     def _real_extract(self, url):
@@ -248,7 +285,11 @@ class TwitterIE(InfoExtractor):
         user_id = mobj.group('user_id')
         twid = mobj.group('id')
 
-        webpage = self._download_webpage(self._TEMPLATE_URL % (user_id, twid), twid)
+        webpage, urlh = self._download_webpage_handle(
+            self._TEMPLATE_URL % (user_id, twid), twid)
+
+        if 'twitter.com/account/suspended' in urlh.geturl():
+            raise ExtractorError('Account suspended by Twitter.', expected=True)
 
         username = remove_end(self._og_search_title(webpage), ' on Twitter')
 
@@ -265,17 +306,6 @@ class TwitterIE(InfoExtractor):
             'title': username + ' - ' + title,
         }
 
-        card_id = self._search_regex(
-            r'["\']/i/cards/tfw/v1/(\d+)', webpage, 'twitter card url', default=None)
-        if card_id:
-            card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
-            info.update({
-                '_type': 'url_transparent',
-                'ie_key': 'TwitterCard',
-                'url': card_url,
-            })
-            return info
-
         mobj = re.search(r'''(?x)
             <video[^>]+class="animated-gif"(?P<more_info>[^>]+)>\s*
                 <source[^>]+video-src="(?P<url>[^"]+)"
@@ -312,7 +342,7 @@ class TwitterIE(InfoExtractor):
 
 class TwitterAmplifyIE(TwitterBaseIE):
     IE_NAME = 'twitter:amplify'
-    _VALID_URL = 'https?://amp\.twimg\.com/v/(?P<id>[0-9a-f\-]{36})'
+    _VALID_URL = r'https?://amp\.twimg\.com/v/(?P<id>[0-9a-f\-]{36})'
 
     _TEST = {
         'url': 'https://amp.twimg.com/v/0ba0c3c7-0af3-4c0a-bed5-7efd1ffa2951',