[youtube] Add support for invidious.nixnet.xyz and yt.elukerio.org (#22223)

[youtube-dl] / youtube_dl / extractor / twitter.py

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index f3fccbf1d8b21343e75cfcc2a87930d31bae4158..cebb6238c561cbfdf77c5ada514d55c216c19a34 100644 (file)
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -108,6 +108,8 @@ class TwitterCardIE(TwitterBaseIE):
         },
     ]
 
+    _API_BASE = 'https://api.twitter.com/1.1'
+
     def _parse_media_info(self, media_info, video_id):
         formats = []
         for media_variant in media_info.get('variants', []):
@@ -149,7 +151,7 @@ class TwitterCardIE(TwitterBaseIE):
             main_script, 'bearer token')
         # https://developer.twitter.com/en/docs/tweets/post-and-engage/api-reference/get-statuses-show-id
         api_data = self._download_json(
-            'https://api.twitter.com/1.1/statuses/show/%s.json' % video_id,
+            '%s/statuses/show/%s.json' % (self._API_BASE, video_id),
             video_id, 'Downloading API data',
             headers={
                 'Authorization': 'Bearer ' + bearer_token,
@@ -169,7 +171,8 @@ class TwitterCardIE(TwitterBaseIE):
             urls.append('https://twitter.com/i/videos/' + video_id)
 
         for u in urls:
-            webpage = self._download_webpage(u, video_id)
+            webpage = self._download_webpage(
+                u, video_id, headers={'Referer': 'https://twitter.com/'})
 
             iframe_url = self._html_search_regex(
                 r'<iframe[^>]+src="((?:https?:)?//(?:www\.youtube\.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
@@ -237,13 +240,13 @@ class TwitterCardIE(TwitterBaseIE):
             if ct0:
                 headers['csrf_token'] = ct0.value
             guest_token = self._download_json(
-                'https://api.twitter.com/1.1/guest/activate.json', video_id,
+                '%s/guest/activate.json' % self._API_BASE, video_id,
                 'Downloading guest token', data=b'',
                 headers=headers)['guest_token']
             headers['x-guest-token'] = guest_token
             self._set_cookie('api.twitter.com', 'gt', guest_token)
             config = self._download_json(
-                'https://api.twitter.com/1.1/videos/tweet/config/%s.json' % video_id,
+                '%s/videos/tweet/config/%s.json' % (self._API_BASE, video_id),
                 video_id, headers=headers)
             track = config['track']
             vmap_url = track.get('vmapUrl')
@@ -425,11 +428,22 @@ class TwitterIE(InfoExtractor):
         'params': {
             'skip_download': True,  # requires ffmpeg
         },
+    }, {
+        'url': 'https://twitter.com/foobar/status/1087791357756956680',
+        'info_dict': {
+            'id': '1087791357756956680',
+            'ext': 'mp4',
+            'title': 'Twitter - A new is coming.  Some of you got an opt-in to try it now. Check out the emoji button, quick keyboard shortcuts, upgraded trends, advanced search, and more. Let us know your thoughts!',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'description': 'md5:66d493500c013e3e2d434195746a7f78',
+            'uploader': 'Twitter',
+            'uploader_id': 'Twitter',
+            'duration': 61.567,
+        },
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        user_id = mobj.group('user_id')
         twid = mobj.group('id')
 
         webpage, urlh = self._download_webpage_handle(
@@ -438,8 +452,13 @@ class TwitterIE(InfoExtractor):
         if 'twitter.com/account/suspended' in urlh.geturl():
             raise ExtractorError('Account suspended by Twitter.', expected=True)
 
-        if user_id is None:
-            mobj = re.match(self._VALID_URL, urlh.geturl())
+        user_id = None
+
+        redirect_mobj = re.match(self._VALID_URL, urlh.geturl())
+        if redirect_mobj:
+            user_id = redirect_mobj.group('user_id')
+
+        if not user_id:
             user_id = mobj.group('user_id')
 
         username = remove_end(self._og_search_title(webpage), ' on Twitter')