[youtube] Fix extraction.

[youtube-dl] / youtube_dl / extractor / bitchute.py

diff --git a/youtube_dl/extractor/bitchute.py b/youtube_dl/extractor/bitchute.py
index 28016f18b23c77471177e57d5f750b5ff8e348cc..0c773e66e1c7349802b4bb8425e844163d18da75 100644 (file)
--- a/youtube_dl/extractor/bitchute.py
+++ b/youtube_dl/extractor/bitchute.py
@@ -5,7 +5,11 @@ import itertools
 import re
 
 from .common import InfoExtractor
-from ..utils import urlencode_postdata
+from ..utils import (
+    orderedSet,
+    unified_strdate,
+    urlencode_postdata,
+)
 
 
 class BitChuteIE(InfoExtractor):
@@ -20,6 +24,7 @@ class BitChuteIE(InfoExtractor):
             'description': 'md5:3f21f6fb5b1d17c3dee9cf6b5fe60b3a',
             'thumbnail': r're:^https?://.*\.jpg$',
             'uploader': 'Victoria X Rave',
+            'upload_date': '20170813',
         },
     }, {
         'url': 'https://www.bitchute.com/embed/lbb5G1hjPhw/',
@@ -33,18 +38,31 @@ class BitChuteIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'https://www.bitchute.com/video/%s' % video_id, video_id)
+            'https://www.bitchute.com/video/%s' % video_id, video_id, headers={
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.57 Safari/537.36',
+            })
 
-        title = self._search_regex(
+        title = self._html_search_regex(
             (r'<[^>]+\bid=["\']video-title[^>]+>([^<]+)', r'<title>([^<]+)'),
             webpage, 'title', default=None) or self._html_search_meta(
             'description', webpage, 'title',
             default=None) or self._og_search_description(webpage)
 
+        format_urls = []
+        for mobj in re.finditer(
+                r'addWebSeed\s*\(\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage):
+            format_urls.append(mobj.group('url'))
+        format_urls.extend(re.findall(r'as=(https?://[^&"\']+)', webpage))
+
         formats = [
-            {'url': mobj.group('url')}
-            for mobj in re.finditer(
-                r'addWebSeed\s*\(\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage)]
+            {'url': format_url}
+            for format_url in orderedSet(format_urls)]
+
+        if not formats:
+            formats = self._parse_html5_media_entries(
+                url, webpage, video_id)[0]['formats']
+
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         description = self._html_search_regex(
@@ -54,8 +72,13 @@ class BitChuteIE(InfoExtractor):
             webpage, default=None) or self._html_search_meta(
             'twitter:image:src', webpage, 'thumbnail')
         uploader = self._html_search_regex(
-            r'(?s)<p\b[^>]+\bclass=["\']video-author[^>]+>(.+?)</p>', webpage,
-            'uploader', fatal=False)
+            (r'(?s)<div class=["\']channel-banner.*?<p\b[^>]+\bclass=["\']name[^>]+>(.+?)</p>',
+             r'(?s)<p\b[^>]+\bclass=["\']video-author[^>]+>(.+?)</p>'),
+            webpage, 'uploader', fatal=False)
+
+        upload_date = unified_strdate(self._search_regex(
+            r'class=["\']video-publish-date[^>]+>[^<]+ at \d+:\d+ UTC on (.+?)\.',
+            webpage, 'upload date', fatal=False))
 
         return {
             'id': video_id,
@@ -63,6 +86,7 @@ class BitChuteIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
+            'upload_date': upload_date,
             'formats': formats,
         }
 
@@ -81,14 +105,15 @@ class BitChuteChannelIE(InfoExtractor):
 
     def _entries(self, channel_id):
         channel_url = 'https://www.bitchute.com/channel/%s/' % channel_id
-        for page_num in itertools.count(0):
+        offset = 0
+        for page_num in itertools.count(1):
             data = self._download_json(
                 '%sextend/' % channel_url, channel_id,
-                'Downloading channel page %d' % (page_num + 1),
+                'Downloading channel page %d' % page_num,
                 data=urlencode_postdata({
                     'csrfmiddlewaretoken': self._TOKEN,
                     'name': '',
-                    'offset': page_num * 25
+                    'offset': offset,
                 }), headers={
                     'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
                     'Referer': channel_url,
@@ -105,6 +130,7 @@ class BitChuteChannelIE(InfoExtractor):
                 html)
             if not video_ids:
                 break
+            offset += len(video_ids)
             for video_id in video_ids:
                 yield self.url_result(
                     'https://www.bitchute.com/video/%s' % video_id,