]> git.bitcoin.ninja Git - youtube-dl/blob - youtube_dl/extractor/bilibili.py
[downloader/http] Randomize HTTP chunk size
[youtube-dl] / youtube_dl / extractor / bilibili.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import hashlib
5 import re
6
7 from .common import InfoExtractor
8 from ..compat import (
9     compat_parse_qs,
10     compat_urlparse,
11 )
12 from ..utils import (
13     ExtractorError,
14     int_or_none,
15     float_or_none,
16     parse_iso8601,
17     smuggle_url,
18     strip_jsonp,
19     unified_timestamp,
20     unsmuggle_url,
21     urlencode_postdata,
22 )
23
24
25 class BiliBiliIE(InfoExtractor):
26     _VALID_URL = r'https?://(?:www\.|bangumi\.|)bilibili\.(?:tv|com)/(?:video/av|anime/(?P<anime_id>\d+)/play#)(?P<id>\d+)'
27
28     _TESTS = [{
29         'url': 'http://www.bilibili.tv/video/av1074402/',
30         'md5': '9fa226fe2b8a9a4d5a69b4c6a183417e',
31         'info_dict': {
32             'id': '1074402',
33             'ext': 'mp4',
34             'title': '【金坷垃】金泡沫',
35             'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
36             'duration': 308.315,
37             'timestamp': 1398012660,
38             'upload_date': '20140420',
39             'thumbnail': r're:^https?://.+\.jpg',
40             'uploader': '菊子桑',
41             'uploader_id': '156160',
42         },
43     }, {
44         # Tested in BiliBiliBangumiIE
45         'url': 'http://bangumi.bilibili.com/anime/1869/play#40062',
46         'only_matching': True,
47     }, {
48         'url': 'http://bangumi.bilibili.com/anime/5802/play#100643',
49         'md5': '3f721ad1e75030cc06faf73587cfec57',
50         'info_dict': {
51             'id': '100643',
52             'ext': 'mp4',
53             'title': 'CHAOS;CHILD',
54             'description': '如果你是神明,并且能够让妄想成为现实。那你会进行怎么样的妄想?是淫靡的世界?独裁社会?毁灭性的制裁?还是……2015年,涩谷。从6年前发生的大灾害“涩谷地震”之后复兴了的这个街区里新设立的私立高中...',
55         },
56         'skip': 'Geo-restricted to China',
57     }, {
58         # Title with double quotes
59         'url': 'http://www.bilibili.com/video/av8903802/',
60         'info_dict': {
61             'id': '8903802',
62             'ext': 'mp4',
63             'title': '阿滴英文|英文歌分享#6 "Closer',
64             'description': '滴妹今天唱Closer給你聽! 有史以来,被推最多次也是最久的歌曲,其实歌词跟我原本想像差蛮多的,不过还是好听! 微博@阿滴英文',
65             'uploader': '阿滴英文',
66             'uploader_id': '65880958',
67             'timestamp': 1488382620,
68             'upload_date': '20170301',
69         },
70         'params': {
71             'skip_download': True,  # Test metadata only
72         },
73     }]
74
75     _APP_KEY = '84956560bc028eb7'
76     _BILIBILI_KEY = '94aba54af9065f71de72f5508f1cd42e'
77
78     def _report_error(self, result):
79         if 'message' in result:
80             raise ExtractorError('%s said: %s' % (self.IE_NAME, result['message']), expected=True)
81         elif 'code' in result:
82             raise ExtractorError('%s returns error %d' % (self.IE_NAME, result['code']), expected=True)
83         else:
84             raise ExtractorError('Can\'t extract Bangumi episode ID')
85
86     def _real_extract(self, url):
87         url, smuggled_data = unsmuggle_url(url, {})
88
89         mobj = re.match(self._VALID_URL, url)
90         video_id = mobj.group('id')
91         anime_id = mobj.group('anime_id')
92         webpage = self._download_webpage(url, video_id)
93
94         if 'anime/' not in url:
95             cid = compat_parse_qs(self._search_regex(
96                 [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)',
97                  r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
98                 webpage, 'player parameters'))['cid'][0]
99         else:
100             if 'no_bangumi_tip' not in smuggled_data:
101                 self.to_screen('Downloading episode %s. To download all videos in anime %s, re-run youtube-dl with %s' % (
102                     video_id, anime_id, compat_urlparse.urljoin(url, '//bangumi.bilibili.com/anime/%s' % anime_id)))
103             headers = {
104                 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
105                 'Referer': url
106             }
107             headers.update(self.geo_verification_headers())
108
109             js = self._download_json(
110                 'http://bangumi.bilibili.com/web_api/get_source', video_id,
111                 data=urlencode_postdata({'episode_id': video_id}),
112                 headers=headers)
113             if 'result' not in js:
114                 self._report_error(js)
115             cid = js['result']['cid']
116
117         payload = 'appkey=%s&cid=%s&otype=json&quality=2&type=mp4' % (self._APP_KEY, cid)
118         sign = hashlib.md5((payload + self._BILIBILI_KEY).encode('utf-8')).hexdigest()
119
120         headers = {
121             'Referer': url
122         }
123         headers.update(self.geo_verification_headers())
124
125         video_info = self._download_json(
126             'http://interface.bilibili.com/playurl?%s&sign=%s' % (payload, sign),
127             video_id, note='Downloading video info page',
128             headers=headers)
129
130         if 'durl' not in video_info:
131             self._report_error(video_info)
132
133         entries = []
134
135         for idx, durl in enumerate(video_info['durl']):
136             formats = [{
137                 'url': durl['url'],
138                 'filesize': int_or_none(durl['size']),
139             }]
140             for backup_url in durl.get('backup_url', []):
141                 formats.append({
142                     'url': backup_url,
143                     # backup URLs have lower priorities
144                     'preference': -2 if 'hd.mp4' in backup_url else -3,
145                 })
146
147             for a_format in formats:
148                 a_format.setdefault('http_headers', {}).update({
149                     'Referer': url,
150                 })
151
152             self._sort_formats(formats)
153
154             entries.append({
155                 'id': '%s_part%s' % (video_id, idx),
156                 'duration': float_or_none(durl.get('length'), 1000),
157                 'formats': formats,
158             })
159
160         title = self._html_search_regex('<h1[^>]*>([^<]+)</h1>', webpage, 'title')
161         description = self._html_search_meta('description', webpage)
162         timestamp = unified_timestamp(self._html_search_regex(
163             r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', default=None))
164         thumbnail = self._html_search_meta(['og:image', 'thumbnailUrl'], webpage)
165
166         # TODO 'view_count' requires deobfuscating Javascript
167         info = {
168             'id': video_id,
169             'title': title,
170             'description': description,
171             'timestamp': timestamp,
172             'thumbnail': thumbnail,
173             'duration': float_or_none(video_info.get('timelength'), scale=1000),
174         }
175
176         uploader_mobj = re.search(
177             r'<a[^>]+href="(?:https?:)?//space\.bilibili\.com/(?P<id>\d+)"[^>]+title="(?P<name>[^"]+)"',
178             webpage)
179         if uploader_mobj:
180             info.update({
181                 'uploader': uploader_mobj.group('name'),
182                 'uploader_id': uploader_mobj.group('id'),
183             })
184
185         for entry in entries:
186             entry.update(info)
187
188         if len(entries) == 1:
189             return entries[0]
190         else:
191             for idx, entry in enumerate(entries):
192                 entry['id'] = '%s_part%d' % (video_id, (idx + 1))
193
194             return {
195                 '_type': 'multi_video',
196                 'id': video_id,
197                 'title': title,
198                 'description': description,
199                 'entries': entries,
200             }
201
202
203 class BiliBiliBangumiIE(InfoExtractor):
204     _VALID_URL = r'https?://bangumi\.bilibili\.com/anime/(?P<id>\d+)'
205
206     IE_NAME = 'bangumi.bilibili.com'
207     IE_DESC = 'BiliBili番剧'
208
209     _TESTS = [{
210         'url': 'http://bangumi.bilibili.com/anime/1869',
211         'info_dict': {
212             'id': '1869',
213             'title': '混沌武士',
214             'description': 'md5:6a9622b911565794c11f25f81d6a97d2',
215         },
216         'playlist_count': 26,
217     }, {
218         'url': 'http://bangumi.bilibili.com/anime/1869',
219         'info_dict': {
220             'id': '1869',
221             'title': '混沌武士',
222             'description': 'md5:6a9622b911565794c11f25f81d6a97d2',
223         },
224         'playlist': [{
225             'md5': '91da8621454dd58316851c27c68b0c13',
226             'info_dict': {
227                 'id': '40062',
228                 'ext': 'mp4',
229                 'title': '混沌武士',
230                 'description': '故事发生在日本的江户时代。风是一个小酒馆的打工女。一日,酒馆里来了一群恶霸,虽然他们的举动令风十分不满,但是毕竟风只是一届女流,无法对他们采取什么行动,只能在心里嘟哝。这时,酒家里又进来了个“不良份子...',
231                 'timestamp': 1414538739,
232                 'upload_date': '20141028',
233                 'episode': '疾风怒涛 Tempestuous Temperaments',
234                 'episode_number': 1,
235             },
236         }],
237         'params': {
238             'playlist_items': '1',
239         },
240     }]
241
242     @classmethod
243     def suitable(cls, url):
244         return False if BiliBiliIE.suitable(url) else super(BiliBiliBangumiIE, cls).suitable(url)
245
246     def _real_extract(self, url):
247         bangumi_id = self._match_id(url)
248
249         # Sometimes this API returns a JSONP response
250         season_info = self._download_json(
251             'http://bangumi.bilibili.com/jsonp/seasoninfo/%s.ver' % bangumi_id,
252             bangumi_id, transform_source=strip_jsonp)['result']
253
254         entries = [{
255             '_type': 'url_transparent',
256             'url': smuggle_url(episode['webplay_url'], {'no_bangumi_tip': 1}),
257             'ie_key': BiliBiliIE.ie_key(),
258             'timestamp': parse_iso8601(episode.get('update_time'), delimiter=' '),
259             'episode': episode.get('index_title'),
260             'episode_number': int_or_none(episode.get('index')),
261         } for episode in season_info['episodes']]
262
263         entries = sorted(entries, key=lambda entry: entry.get('episode_number'))
264
265         return self.playlist_result(
266             entries, bangumi_id,
267             season_info.get('bangumi_title'), season_info.get('evaluate'))