]> git.bitcoin.ninja Git - youtube-dl/blob - youtube_dl/extractor/ximalaya.py
[extractor/common] Relax interaction count extraction in _json_ld
[youtube-dl] / youtube_dl / extractor / ximalaya.py
1 # coding: utf-8
2
3 from __future__ import unicode_literals
4
5 import itertools
6 import re
7
8 from .common import InfoExtractor
9
10
11 class XimalayaBaseIE(InfoExtractor):
12     _GEO_COUNTRIES = ['CN']
13
14
15 class XimalayaIE(XimalayaBaseIE):
16     IE_NAME = 'ximalaya'
17     IE_DESC = '喜马拉雅FM'
18     _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/(?P<uid>[0-9]+)/sound/(?P<id>[0-9]+)'
19     _USER_URL_FORMAT = '%s://www.ximalaya.com/zhubo/%i/'
20     _TESTS = [
21         {
22             'url': 'http://www.ximalaya.com/61425525/sound/47740352/',
23             'info_dict': {
24                 'id': '47740352',
25                 'ext': 'm4a',
26                 'uploader': '小彬彬爱听书',
27                 'uploader_id': 61425525,
28                 'uploader_url': 'http://www.ximalaya.com/zhubo/61425525/',
29                 'title': '261.唐诗三百首.卷八.送孟浩然之广陵.李白',
30                 'description': "contains:《送孟浩然之广陵》\n作者:李白\n故人西辞黄鹤楼,烟花三月下扬州。\n孤帆远影碧空尽,惟见长江天际流。",
31                 'thumbnails': [
32                     {
33                         'name': 'cover_url',
34                         'url': r're:^https?://.*\.jpg$',
35                     },
36                     {
37                         'name': 'cover_url_142',
38                         'url': r're:^https?://.*\.jpg$',
39                         'width': 180,
40                         'height': 180
41                     }
42                 ],
43                 'categories': ['renwen', '人文'],
44                 'duration': 93,
45                 'view_count': int,
46                 'like_count': int,
47             }
48         },
49         {
50             'url': 'http://m.ximalaya.com/61425525/sound/47740352/',
51             'info_dict': {
52                 'id': '47740352',
53                 'ext': 'm4a',
54                 'uploader': '小彬彬爱听书',
55                 'uploader_id': 61425525,
56                 'uploader_url': 'http://www.ximalaya.com/zhubo/61425525/',
57                 'title': '261.唐诗三百首.卷八.送孟浩然之广陵.李白',
58                 'description': "contains:《送孟浩然之广陵》\n作者:李白\n故人西辞黄鹤楼,烟花三月下扬州。\n孤帆远影碧空尽,惟见长江天际流。",
59                 'thumbnails': [
60                     {
61                         'name': 'cover_url',
62                         'url': r're:^https?://.*\.jpg$',
63                     },
64                     {
65                         'name': 'cover_url_142',
66                         'url': r're:^https?://.*\.jpg$',
67                         'width': 180,
68                         'height': 180
69                     }
70                 ],
71                 'categories': ['renwen', '人文'],
72                 'duration': 93,
73                 'view_count': int,
74                 'like_count': int,
75             }
76         },
77         {
78             'url': 'https://www.ximalaya.com/11045267/sound/15705996/',
79             'info_dict': {
80                 'id': '15705996',
81                 'ext': 'm4a',
82                 'uploader': '李延隆老师',
83                 'uploader_id': 11045267,
84                 'uploader_url': 'https://www.ximalaya.com/zhubo/11045267/',
85                 'title': 'Lesson 1 Excuse me!',
86                 'description': "contains:Listen to the tape then answer\xa0this question. Whose handbag is it?\n"
87                                "听录音,然后回答问题,这是谁的手袋?",
88                 'thumbnails': [
89                     {
90                         'name': 'cover_url',
91                         'url': r're:^https?://.*\.jpg$',
92                     },
93                     {
94                         'name': 'cover_url_142',
95                         'url': r're:^https?://.*\.jpg$',
96                         'width': 180,
97                         'height': 180
98                     }
99                 ],
100                 'categories': ['train', '外语'],
101                 'duration': 40,
102                 'view_count': int,
103                 'like_count': int,
104             }
105         },
106     ]
107
108     def _real_extract(self, url):
109
110         is_m = 'm.ximalaya' in url
111         scheme = 'https' if url.startswith('https') else 'http'
112
113         audio_id = self._match_id(url)
114         webpage = self._download_webpage(url, audio_id,
115                                          note='Download sound page for %s' % audio_id,
116                                          errnote='Unable to get sound page')
117
118         audio_info_file = '%s://m.ximalaya.com/tracks/%s.json' % (scheme, audio_id)
119         audio_info = self._download_json(audio_info_file, audio_id,
120                                          'Downloading info json %s' % audio_info_file,
121                                          'Unable to download info file')
122
123         formats = []
124         for bps, k in (('24k', 'play_path_32'), ('64k', 'play_path_64')):
125             if audio_info.get(k):
126                 formats.append({
127                     'format_id': bps,
128                     'url': audio_info[k],
129                 })
130
131         thumbnails = []
132         for k in audio_info.keys():
133             # cover pics kyes like: cover_url', 'cover_url_142'
134             if k.startswith('cover_url'):
135                 thumbnail = {'name': k, 'url': audio_info[k]}
136                 if k == 'cover_url_142':
137                     thumbnail['width'] = 180
138                     thumbnail['height'] = 180
139                 thumbnails.append(thumbnail)
140
141         audio_uploader_id = audio_info.get('uid')
142
143         if is_m:
144             audio_description = self._html_search_regex(r'(?s)<section\s+class=["\']content[^>]+>(.+?)</section>',
145                                                         webpage, 'audio_description', fatal=False)
146         else:
147             audio_description = self._html_search_regex(r'(?s)<div\s+class=["\']rich_intro[^>]*>(.+?</article>)',
148                                                         webpage, 'audio_description', fatal=False)
149
150         if not audio_description:
151             audio_description_file = '%s://www.ximalaya.com/sounds/%s/rich_intro' % (scheme, audio_id)
152             audio_description = self._download_webpage(audio_description_file, audio_id,
153                                                        note='Downloading description file %s' % audio_description_file,
154                                                        errnote='Unable to download descrip file',
155                                                        fatal=False)
156             audio_description = audio_description.strip() if audio_description else None
157
158         return {
159             'id': audio_id,
160             'uploader': audio_info.get('nickname'),
161             'uploader_id': audio_uploader_id,
162             'uploader_url': self._USER_URL_FORMAT % (scheme, audio_uploader_id) if audio_uploader_id else None,
163             'title': audio_info['title'],
164             'thumbnails': thumbnails,
165             'description': audio_description,
166             'categories': list(filter(None, (audio_info.get('category_name'), audio_info.get('category_title')))),
167             'duration': audio_info.get('duration'),
168             'view_count': audio_info.get('play_count'),
169             'like_count': audio_info.get('favorites_count'),
170             'formats': formats,
171         }
172
173
174 class XimalayaAlbumIE(XimalayaBaseIE):
175     IE_NAME = 'ximalaya:album'
176     IE_DESC = '喜马拉雅FM 专辑'
177     _VALID_URL = r'https?://(?:www\.|m\.)?ximalaya\.com/(?P<uid>[0-9]+)/album/(?P<id>[0-9]+)'
178     _TEMPLATE_URL = '%s://www.ximalaya.com/%s/album/%s/'
179     _BASE_URL_TEMPL = '%s://www.ximalaya.com%s'
180     _LIST_VIDEO_RE = r'<a[^>]+?href="(?P<url>/%s/sound/(?P<id>\d+)/?)"[^>]+?title="(?P<title>[^>]+)">'
181     _TESTS = [{
182         'url': 'http://www.ximalaya.com/61425525/album/5534601/',
183         'info_dict': {
184             'title': '唐诗三百首(含赏析)',
185             'id': '5534601',
186         },
187         'playlist_count': 312,
188     }, {
189         'url': 'http://m.ximalaya.com/61425525/album/5534601',
190         'info_dict': {
191             'title': '唐诗三百首(含赏析)',
192             'id': '5534601',
193         },
194         'playlist_count': 312,
195     },
196     ]
197
198     def _real_extract(self, url):
199         self.scheme = scheme = 'https' if url.startswith('https') else 'http'
200
201         mobj = re.match(self._VALID_URL, url)
202         uid, playlist_id = mobj.group('uid'), mobj.group('id')
203
204         webpage = self._download_webpage(self._TEMPLATE_URL % (scheme, uid, playlist_id), playlist_id,
205                                          note='Download album page for %s' % playlist_id,
206                                          errnote='Unable to get album info')
207
208         title = self._html_search_regex(r'detailContent_title[^>]*><h1(?:[^>]+)?>([^<]+)</h1>',
209                                         webpage, 'title', fatal=False)
210
211         return self.playlist_result(self._entries(webpage, playlist_id, uid), playlist_id, title)
212
213     def _entries(self, page, playlist_id, uid):
214         html = page
215         for page_num in itertools.count(1):
216             for entry in self._process_page(html, uid):
217                 yield entry
218
219             next_url = self._search_regex(r'<a\s+href=(["\'])(?P<more>[\S]+)\1[^>]+rel=(["\'])next\3',
220                                           html, 'list_next_url', default=None, group='more')
221             if not next_url:
222                 break
223
224             next_full_url = self._BASE_URL_TEMPL % (self.scheme, next_url)
225             html = self._download_webpage(next_full_url, playlist_id)
226
227     def _process_page(self, html, uid):
228         find_from = html.index('album_soundlist')
229         for mobj in re.finditer(self._LIST_VIDEO_RE % uid, html[find_from:]):
230             yield self.url_result(self._BASE_URL_TEMPL % (self.scheme, mobj.group('url')),
231                                   XimalayaIE.ie_key(),
232                                   mobj.group('id'),
233                                   mobj.group('title'))