[ted] Fix extraction for videos without nativeDownloads (closes #16756, closes #17085)
[youtube-dl] / youtube_dl / extractor / ted.py
1 from __future__ import unicode_literals
2
3 import json
4 import re
5
6 from .common import InfoExtractor
7
8 from ..compat import compat_str
9 from ..utils import (
10     int_or_none,
11     try_get,
12 )
13
14
15 class TEDIE(InfoExtractor):
16     IE_NAME = 'ted'
17     _VALID_URL = r'''(?x)
18         (?P<proto>https?://)
19         (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
20         (
21             (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
22             |
23             ((?P<type_talk>talks)) # We have a simple talk
24             |
25             (?P<type_watch>watch)/[^/]+/[^/]+
26         )
27         (/lang/(.*?))? # The url may contain the language
28         /(?P<name>[\w-]+) # Here goes the name and then ".html"
29         .*)$
30         '''
31     _TESTS = [{
32         'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
33         'md5': '0de43ac406aa3e4ea74b66c9c7789b13',
34         'info_dict': {
35             'id': '102',
36             'ext': 'mp4',
37             'title': 'The illusion of consciousness',
38             'description': ('Philosopher Dan Dennett makes a compelling '
39                             'argument that not only don\'t we understand our own '
40                             'consciousness, but that half the time our brains are '
41                             'actively fooling us.'),
42             'uploader': 'Dan Dennett',
43             'width': 853,
44             'duration': 1308,
45         }
46     }, {
47         'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
48         'md5': 'b899ac15e345fb39534d913f7606082b',
49         'info_dict': {
50             'id': 'tSVI8ta_P4w',
51             'ext': 'mp4',
52             'title': 'Vishal Sikka: The beauty and power of algorithms',
53             'thumbnail': r're:^https?://.+\.jpg',
54             'description': 'md5:6261fdfe3e02f4f579cbbfc00aff73f4',
55             'upload_date': '20140122',
56             'uploader_id': 'TEDInstitute',
57             'uploader': 'TED Institute',
58         },
59         'add_ie': ['Youtube'],
60     }, {
61         'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
62         'md5': '71b3ab2f4233012dce09d515c9c39ce2',
63         'info_dict': {
64             'id': '1972',
65             'ext': 'mp4',
66             'title': 'Be passionate. Be courageous. Be your best.',
67             'uploader': 'Gabby Giffords and Mark Kelly',
68             'description': 'md5:5174aed4d0f16021b704120360f72b92',
69             'duration': 1128,
70         },
71     }, {
72         'url': 'http://www.ted.com/playlists/who_are_the_hackers',
73         'info_dict': {
74             'id': '10',
75             'title': 'Who are the hackers?',
76         },
77         'playlist_mincount': 6,
78     }, {
79         # contains a youtube video
80         'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
81         'add_ie': ['Youtube'],
82         'info_dict': {
83             'id': '_ZG8HBuDjgc',
84             'ext': 'webm',
85             'title': 'Douglas Adams: Parrots the Universe and Everything',
86             'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
87             'uploader': 'University of California Television (UCTV)',
88             'uploader_id': 'UCtelevision',
89             'upload_date': '20080522',
90         },
91         'params': {
92             'skip_download': True,
93         },
94     }, {
95         # YouTube video
96         'url': 'http://www.ted.com/talks/jeffrey_kluger_the_sibling_bond',
97         'add_ie': ['Youtube'],
98         'info_dict': {
99             'id': 'aFBIPO-P7LM',
100             'ext': 'mp4',
101             'title': 'The hidden power of siblings: Jeff Kluger at TEDxAsheville',
102             'description': 'md5:3d7a4f50d95ca5dd67104e2a20f43fe1',
103             'uploader': 'TEDx Talks',
104             'uploader_id': 'TEDxTalks',
105             'upload_date': '20111216',
106         },
107         'params': {
108             'skip_download': True,
109         },
110     }, {
111         # no nativeDownloads
112         'url': 'https://www.ted.com/talks/tom_thum_the_orchestra_in_my_mouth',
113         'info_dict': {
114             'id': '1792',
115             'ext': 'mp4',
116             'title': 'The orchestra in my mouth',
117             'description': 'md5:5d1d78650e2f8dfcbb8ebee2951ac29a',
118             'uploader': 'Tom Thum',
119         },
120         'params': {
121             'skip_download': True,
122         },
123     }]
124
125     _NATIVE_FORMATS = {
126         'low': {'width': 320, 'height': 180},
127         'medium': {'width': 512, 'height': 288},
128         'high': {'width': 854, 'height': 480},
129     }
130
131     def _extract_info(self, webpage):
132         info_json = self._search_regex(
133             r'(?s)q\(\s*"\w+.init"\s*,\s*({.+})\)\s*</script>',
134             webpage, 'info json')
135         return json.loads(info_json)
136
137     def _real_extract(self, url):
138         m = re.match(self._VALID_URL, url, re.VERBOSE)
139         if m.group('type').startswith('embed'):
140             desktop_url = m.group('proto') + 'www' + m.group('urlmain')
141             return self.url_result(desktop_url, 'TED')
142         name = m.group('name')
143         if m.group('type_talk'):
144             return self._talk_info(url, name)
145         elif m.group('type_watch'):
146             return self._watch_info(url, name)
147         else:
148             return self._playlist_videos_info(url, name)
149
150     def _playlist_videos_info(self, url, name):
151         '''Returns the videos of the playlist'''
152
153         webpage = self._download_webpage(url, name,
154                                          'Downloading playlist webpage')
155         info = self._extract_info(webpage)
156
157         playlist_info = try_get(
158             info, lambda x: x['__INITIAL_DATA__']['playlist'],
159             dict) or info['playlist']
160
161         playlist_entries = [
162             self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
163             for talk in try_get(
164                 info, lambda x: x['__INITIAL_DATA__']['talks'],
165                 dict) or info['talks']
166         ]
167         return self.playlist_result(
168             playlist_entries,
169             playlist_id=compat_str(playlist_info['id']),
170             playlist_title=playlist_info['title'])
171
172     def _talk_info(self, url, video_name):
173         webpage = self._download_webpage(url, video_name)
174
175         info = self._extract_info(webpage)
176
177         talk_info = try_get(
178             info, lambda x: x['__INITIAL_DATA__']['talks'][0],
179             dict) or info['talks'][0]
180
181         title = talk_info['title'].strip()
182
183         external = talk_info.get('external')
184         if external:
185             service = external['service']
186             self.to_screen('Found video from %s' % service)
187             ext_url = None
188             if service.lower() == 'youtube':
189                 ext_url = external.get('code')
190             return {
191                 '_type': 'url',
192                 'url': ext_url or external['uri'],
193             }
194
195         native_downloads = try_get(
196             talk_info,
197             (lambda x: x['downloads']['nativeDownloads'],
198              lambda x: x['nativeDownloads']),
199             dict) or {}
200
201         formats = [{
202             'url': format_url,
203             'format_id': format_id,
204             'format': format_id,
205         } for (format_id, format_url) in native_downloads.items() if format_url is not None]
206         if formats:
207             for f in formats:
208                 finfo = self._NATIVE_FORMATS.get(f['format_id'])
209                 if finfo:
210                     f.update(finfo)
211
212         player_talk = talk_info['player_talks'][0]
213
214         resources_ = player_talk.get('resources') or talk_info.get('resources')
215
216         http_url = None
217         for format_id, resources in resources_.items():
218             if format_id == 'h264':
219                 for resource in resources:
220                     h264_url = resource.get('file')
221                     if not h264_url:
222                         continue
223                     bitrate = int_or_none(resource.get('bitrate'))
224                     formats.append({
225                         'url': h264_url,
226                         'format_id': '%s-%sk' % (format_id, bitrate),
227                         'tbr': bitrate,
228                     })
229                     if re.search(r'\d+k', h264_url):
230                         http_url = h264_url
231             elif format_id == 'rtmp':
232                 streamer = talk_info.get('streamer')
233                 if not streamer:
234                     continue
235                 for resource in resources:
236                     formats.append({
237                         'format_id': '%s-%s' % (format_id, resource.get('name')),
238                         'url': streamer,
239                         'play_path': resource['file'],
240                         'ext': 'flv',
241                         'width': int_or_none(resource.get('width')),
242                         'height': int_or_none(resource.get('height')),
243                         'tbr': int_or_none(resource.get('bitrate')),
244                     })
245             elif format_id == 'hls':
246                 formats.extend(self._extract_m3u8_formats(
247                     resources.get('stream'), video_name, 'mp4', m3u8_id=format_id, fatal=False))
248
249         m3u8_formats = list(filter(
250             lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none',
251             formats))
252         if http_url:
253             for m3u8_format in m3u8_formats:
254                 bitrate = self._search_regex(r'(\d+k)', m3u8_format['url'], 'bitrate', default=None)
255                 if not bitrate:
256                     continue
257                 f = m3u8_format.copy()
258                 f.update({
259                     'url': re.sub(r'\d+k', bitrate, http_url),
260                     'format_id': m3u8_format['format_id'].replace('hls', 'http'),
261                     'protocol': 'http',
262                 })
263                 formats.append(f)
264
265         audio_download = talk_info.get('audioDownload')
266         if audio_download:
267             formats.append({
268                 'url': audio_download,
269                 'format_id': 'audio',
270                 'vcodec': 'none',
271             })
272
273         self._sort_formats(formats)
274
275         video_id = compat_str(talk_info['id'])
276
277         return {
278             'id': video_id,
279             'title': title,
280             'uploader': player_talk.get('speaker') or talk_info.get('speaker'),
281             'thumbnail': player_talk.get('thumb') or talk_info.get('thumb'),
282             'description': self._og_search_description(webpage),
283             'subtitles': self._get_subtitles(video_id, talk_info),
284             'formats': formats,
285             'duration': talk_info.get('duration'),
286         }
287
288     def _get_subtitles(self, video_id, talk_info):
289         sub_lang_list = {}
290         for language in try_get(
291                 talk_info,
292                 (lambda x: x['downloads']['languages'],
293                  lambda x: x['languages']), list):
294             lang_code = language.get('languageCode') or language.get('ianaCode')
295             if not lang_code:
296                 continue
297             sub_lang_list[lang_code] = [
298                 {
299                     'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, lang_code, ext),
300                     'ext': ext,
301                 }
302                 for ext in ['ted', 'srt']
303             ]
304         return sub_lang_list
305
306     def _watch_info(self, url, name):
307         webpage = self._download_webpage(url, name)
308
309         config_json = self._html_search_regex(
310             r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*</script>',
311             webpage, 'config', default=None)
312         if not config_json:
313             embed_url = self._search_regex(
314                 r"<iframe[^>]+class='pages-video-embed__video__object'[^>]+src='([^']+)'", webpage, 'embed url')
315             return self.url_result(self._proto_relative_url(embed_url))
316         config = json.loads(config_json)['config']
317         video_url = config['video']['url']
318         thumbnail = config.get('image', {}).get('url')
319
320         title = self._html_search_regex(
321             r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
322         description = self._html_search_regex(
323             [
324                 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
325                 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
326             ],
327             webpage, 'description', fatal=False)
328
329         return {
330             'id': name,
331             'url': video_url,
332             'title': title,
333             'thumbnail': thumbnail,
334             'description': description,
335         }