_ Git - youtube-dl/blob - youtube_dl/extractor/bilibili.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import hashlib
   5 import re
   6
   7 from .common import InfoExtractor
   8 from ..compat import compat_parse_qs
   9 from ..utils import (
  10     int_or_none,
  11     float_or_none,
  12     unified_timestamp,
  13     urlencode_postdata,
  14 )
  15
  16
  17 class BiliBiliIE(InfoExtractor):
  18     _VALID_URL = r'https?://(?:www\.|bangumi\.|)bilibili\.(?:tv|com)/(?:video/av|anime/v/)(?P<id>\d+)'
  19
  20     _TESTS = [{
  21         'url': 'http://www.bilibili.tv/video/av1074402/',
  22         'md5': '9fa226fe2b8a9a4d5a69b4c6a183417e',
  23         'info_dict': {
  24             'id': '1074402',
  25             'ext': 'mp4',
  26             'title': '【金坷垃】金泡沫',
  27             'description': 'md5:ce18c2a2d2193f0df2917d270f2e5923',
  28             'duration': 308.315,
  29             'timestamp': 1398012660,
  30             'upload_date': '20140420',
  31             'thumbnail': 're:^https?://.+\.jpg',
  32             'uploader': '菊子桑',
  33             'uploader_id': '156160',
  34         },
  35     }, {
  36         'url': 'http://www.bilibili.com/video/av1041170/',
  37         'info_dict': {
  38             'id': '1041170',
  39             'ext': 'mp4',
  40             'title': '【BD1080P】刀语【诸神&异域】',
  41             'description': '这是个神奇的故事~每个人不留弹幕不给走哦~切利哦！~',
  42             'duration': 3382.259,
  43             'timestamp': 1396530060,
  44             'upload_date': '20140403',
  45             'thumbnail': 're:^https?://.+\.jpg',
  46             'uploader': '枫叶逝去',
  47             'uploader_id': '520116',
  48         },
  49     }, {
  50         'url': 'http://www.bilibili.com/video/av4808130/',
  51         'info_dict': {
  52             'id': '4808130',
  53             'ext': 'mp4',
  54             'title': '【长篇】哆啦A梦443【钉铛】',
  55             'description': '(2016.05.27)来组合客人的脸吧&amp;amp;寻母六千里锭 抱歉，又轮到周日上班现在才到家 封面www.pixiv.net/member_illust.php?mode=medium&amp;amp;illust_id=56912929',
  56             'duration': 1493.995,
  57             'timestamp': 1464564180,
  58             'upload_date': '20160529',
  59             'thumbnail': 're:^https?://.+\.jpg',
  60             'uploader': '喜欢拉面',
  61             'uploader_id': '151066',
  62         },
  63     }, {
  64         # Missing upload time
  65         'url': 'http://www.bilibili.com/video/av1867637/',
  66         'info_dict': {
  67             'id': '1867637',
  68             'ext': 'mp4',
  69             'title': '【HDTV】【喜剧】岳父岳母真难当 （2014）【法国票房冠军】',
  70             'description': '一个信奉天主教的法国旧式传统资产阶级家庭中有四个女儿。三个女儿却分别找了阿拉伯、犹太、中国丈夫，老夫老妻唯独期盼剩下未嫁的小女儿能找一个信奉天主教的法国白人，结果没想到小女儿找了一位非裔黑人……【这次应该不会跳帧了】',
  71             'duration': 5760.0,
  72             'uploader': '黑夜为猫',
  73             'uploader_id': '610729',
  74             'thumbnail': 're:^https?://.+\.jpg',
  75         },
  76         'params': {
  77             # Just to test metadata extraction
  78             'skip_download': True,
  79         },
  80         'expected_warnings': ['upload time'],
  81     }, {
  82         'url': 'http://bangumi.bilibili.com/anime/v/40068',
  83         'md5': '08d539a0884f3deb7b698fb13ba69696',
  84         'info_dict': {
  85             'id': '40068',
  86             'ext': 'mp4',
  87             'duration': 1402.357,
  88             'title': '混沌武士 : 第7集 四面楚歌 A Risky Racket',
  89             'description': 'md5:6a9622b911565794c11f25f81d6a97d2',
  90             'thumbnail': 're:^http?://.+\.jpg',
  91         },
  92     }]
  93
  94     _APP_KEY = '6f90a59ac58a4123'
  95     _BILIBILI_KEY = '0bfd84cc3940035173f35e6777508326'
  96
  97     def _real_extract(self, url):
  98         video_id = self._match_id(url)
  99         webpage = self._download_webpage(url, video_id)
 100
 101         if 'anime/v' not in url:
 102             cid = compat_parse_qs(self._search_regex(
 103                 [r'EmbedPlayer\([^)]+,\s*"([^"]+)"\)',
 104                  r'<iframe[^>]+src="https://secure\.bilibili\.com/secure,([^"]+)"'],
 105                 webpage, 'player parameters'))['cid'][0]
 106         else:
 107             js = self._download_json(
 108                 'http://bangumi.bilibili.com/web_api/get_source', video_id,
 109                 data=urlencode_postdata({'episode_id': video_id}),
 110                 headers={'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8'})
 111             cid = js['result']['cid']
 112
 113         payload = 'appkey=%s&cid=%s&otype=json&quality=2&type=mp4' % (self._APP_KEY, cid)
 114         sign = hashlib.md5((payload + self._BILIBILI_KEY).encode('utf-8')).hexdigest()
 115
 116         video_info = self._download_json(
 117             'http://interface.bilibili.com/playurl?%s&sign=%s' % (payload, sign),
 118             video_id, note='Downloading video info page')
 119
 120         entries = []
 121
 122         for idx, durl in enumerate(video_info['durl']):
 123             formats = [{
 124                 'url': durl['url'],
 125                 'filesize': int_or_none(durl['size']),
 126             }]
 127             for backup_url in durl['backup_url']:
 128                 formats.append({
 129                     'url': backup_url,
 130                     # backup URLs have lower priorities
 131                     'preference': -2 if 'hd.mp4' in backup_url else -3,
 132                 })
 133
 134             self._sort_formats(formats)
 135
 136             entries.append({
 137                 'id': '%s_part%s' % (video_id, idx),
 138                 'duration': float_or_none(durl.get('length'), 1000),
 139                 'formats': formats,
 140             })
 141
 142         title = self._html_search_regex('<h1[^>]+title="([^"]+)">', webpage, 'title')
 143         description = self._html_search_meta('description', webpage)
 144         timestamp = unified_timestamp(self._html_search_regex(
 145             r'<time[^>]+datetime="([^"]+)"', webpage, 'upload time', fatal=False))
 146         thumbnail = self._html_search_meta(['og:image', 'thumbnailUrl'], webpage)
 147
 148         # TODO 'view_count' requires deobfuscating Javascript
 149         info = {
 150             'id': video_id,
 151             'title': title,
 152             'description': description,
 153             'timestamp': timestamp,
 154             'thumbnail': thumbnail,
 155             'duration': float_or_none(video_info.get('timelength'), scale=1000),
 156         }
 157
 158         uploader_mobj = re.search(
 159             r'<a[^>]+href="https?://space\.bilibili\.com/(?P<id>\d+)"[^>]+title="(?P<name>[^"]+)"',
 160             webpage)
 161         if uploader_mobj:
 162             info.update({
 163                 'uploader': uploader_mobj.group('name'),
 164                 'uploader_id': uploader_mobj.group('id'),
 165             })
 166
 167         for entry in entries:
 168             entry.update(info)
 169
 170         if len(entries) == 1:
 171             return entries[0]
 172         else:
 173             for idx, entry in enumerate(entries):
 174                 entry['id'] = '%s_part%d' % (video_id, (idx + 1))
 175
 176             return {
 177                 '_type': 'multi_video',
 178                 'id': video_id,
 179                 'title': title,
 180                 'description': description,
 181                 'entries': entries,
 182             }