_ Git - youtube-dl/blob - youtube_dl/extractor/ted.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4 import re
   5
   6 from .subtitles import SubtitlesInfoExtractor
   7
   8 from ..compat import (
   9     compat_str,
  10 )
  11
  12
  13 class TEDIE(SubtitlesInfoExtractor):
  14     _VALID_URL = r'''(?x)
  15         (?P<proto>https?://)
  16         (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
  17         (
  18             (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
  19             |
  20             ((?P<type_talk>talks)) # We have a simple talk
  21             |
  22             (?P<type_watch>watch)/[^/]+/[^/]+
  23         )
  24         (/lang/(.*?))? # The url may contain the language
  25         /(?P<name>[\w-]+) # Here goes the name and then ".html"
  26         .*)$
  27         '''
  28     _TESTS = [{
  29         'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
  30         'md5': 'fc94ac279feebbce69f21c0c6ee82810',
  31         'info_dict': {
  32             'id': '102',
  33             'ext': 'mp4',
  34             'title': 'The illusion of consciousness',
  35             'description': ('Philosopher Dan Dennett makes a compelling '
  36                             'argument that not only don\'t we understand our own '
  37                             'consciousness, but that half the time our brains are '
  38                             'actively fooling us.'),
  39             'uploader': 'Dan Dennett',
  40             'width': 854,
  41             'duration': 1308,
  42         }
  43     }, {
  44         'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
  45         'md5': '226f4fb9c62380d11b7995efa4c87994',
  46         'info_dict': {
  47             'id': 'vishal-sikka-the-beauty-and-power-of-algorithms',
  48             'ext': 'mp4',
  49             'title': 'Vishal Sikka: The beauty and power of algorithms',
  50             'thumbnail': 're:^https?://.+\.jpg',
  51             'description': 'Adaptive, intelligent, and consistent, algorithms are emerging as the ultimate app for everything from matching consumers to products to assessing medical diagnoses. Vishal Sikka shares his appreciation for the algorithm, charting both its inherent beauty and its growing power.',
  52         }
  53     }, {
  54         'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
  55         'info_dict': {
  56             'id': '1972',
  57             'ext': 'mp4',
  58             'title': 'Be passionate. Be courageous. Be your best.',
  59             'uploader': 'Gabby Giffords and Mark Kelly',
  60             'description': 'md5:5174aed4d0f16021b704120360f72b92',
  61             'duration': 1128,
  62         },
  63     }, {
  64         'url': 'http://www.ted.com/playlists/who_are_the_hackers',
  65         'info_dict': {
  66             'id': '10',
  67             'title': 'Who are the hackers?',
  68         },
  69         'playlist_mincount': 6,
  70     }, {
  71         # contains a youtube video
  72         'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
  73         'add_ie': ['Youtube'],
  74         'info_dict': {
  75             'id': '_ZG8HBuDjgc',
  76             'ext': 'mp4',
  77             'title': 'Douglas Adams: Parrots the Universe and Everything',
  78             'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
  79             'uploader': 'University of California Television (UCTV)',
  80             'uploader_id': 'UCtelevision',
  81             'upload_date': '20080522',
  82         },
  83         'params': {
  84             'skip_download': True,
  85         },
  86     }]
  87
  88     _NATIVE_FORMATS = {
  89         'low': {'preference': 1, 'width': 320, 'height': 180},
  90         'medium': {'preference': 2, 'width': 512, 'height': 288},
  91         'high': {'preference': 3, 'width': 854, 'height': 480},
  92     }
  93
  94     def _extract_info(self, webpage):
  95         info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
  96                                        webpage, 'info json')
  97         return json.loads(info_json)
  98
  99     def _real_extract(self, url):
 100         m = re.match(self._VALID_URL, url, re.VERBOSE)
 101         if m.group('type').startswith('embed'):
 102             desktop_url = m.group('proto') + 'www' + m.group('urlmain')
 103             return self.url_result(desktop_url, 'TED')
 104         name = m.group('name')
 105         if m.group('type_talk'):
 106             return self._talk_info(url, name)
 107         elif m.group('type_watch'):
 108             return self._watch_info(url, name)
 109         else:
 110             return self._playlist_videos_info(url, name)
 111
 112     def _playlist_videos_info(self, url, name):
 113         '''Returns the videos of the playlist'''
 114
 115         webpage = self._download_webpage(url, name,
 116                                          'Downloading playlist webpage')
 117         info = self._extract_info(webpage)
 118         playlist_info = info['playlist']
 119
 120         playlist_entries = [
 121             self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
 122             for talk in info['talks']
 123         ]
 124         return self.playlist_result(
 125             playlist_entries,
 126             playlist_id=compat_str(playlist_info['id']),
 127             playlist_title=playlist_info['title'])
 128
 129     def _talk_info(self, url, video_name):
 130         webpage = self._download_webpage(url, video_name)
 131         self.report_extraction(video_name)
 132
 133         talk_info = self._extract_info(webpage)['talks'][0]
 134
 135         if talk_info.get('external') is not None:
 136             self.to_screen('Found video from %s' % talk_info['external']['service'])
 137             if 'code' in talk_info['external']:
 138                 ext_url = talk_info['external']['code']
 139             else:
 140                 ext_url = talk_info['external']['uri']
 141             return {
 142                 '_type': 'url',
 143                 'url': ext_url,
 144             }
 145
 146         formats = [{
 147             'url': format_url,
 148             'format_id': format_id,
 149             'format': format_id,
 150         } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
 151         if formats:
 152             for f in formats:
 153                 finfo = self._NATIVE_FORMATS.get(f['format_id'])
 154                 if finfo:
 155                     f.update(finfo)
 156         else:
 157             # Use rtmp downloads
 158             formats = [{
 159                 'format_id': f['name'],
 160                 'url': talk_info['streamer'],
 161                 'play_path': f['file'],
 162                 'ext': 'flv',
 163                 'width': f['width'],
 164                 'height': f['height'],
 165                 'tbr': f['bitrate'],
 166             } for f in talk_info['resources']['rtmp']]
 167         self._sort_formats(formats)
 168
 169         video_id = compat_str(talk_info['id'])
 170         # subtitles
 171         video_subtitles = self.extract_subtitles(video_id, talk_info)
 172         if self._downloader.params.get('listsubtitles', False):
 173             self._list_available_subtitles(video_id, talk_info)
 174             return
 175
 176         thumbnail = talk_info['thumb']
 177         if not thumbnail.startswith('http'):
 178             thumbnail = 'http://' + thumbnail
 179         return {
 180             'id': video_id,
 181             'title': talk_info['title'].strip(),
 182             'uploader': talk_info['speaker'],
 183             'thumbnail': thumbnail,
 184             'description': self._og_search_description(webpage),
 185             'subtitles': video_subtitles,
 186             'formats': formats,
 187             'duration': talk_info.get('duration'),
 188         }
 189
 190     def _get_available_subtitles(self, video_id, talk_info):
 191         languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
 192         if languages:
 193             sub_lang_list = {}
 194             for l in languages:
 195                 url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
 196                 sub_lang_list[l] = url
 197             return sub_lang_list
 198         else:
 199             self._downloader.report_warning('video doesn\'t have subtitles')
 200             return {}
 201
 202     def _watch_info(self, url, name):
 203         webpage = self._download_webpage(url, name)
 204
 205         config_json = self._html_search_regex(
 206             r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*</script>',
 207             webpage, 'config')
 208         config = json.loads(config_json)['config']
 209         video_url = config['video']['url']
 210         thumbnail = config.get('image', {}).get('url')
 211
 212         title = self._html_search_regex(
 213             r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
 214         description = self._html_search_regex(
 215             [
 216                 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
 217                 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
 218             ],
 219             webpage, 'description', fatal=False)
 220
 221         return {
 222             'id': name,
 223             'url': video_url,
 224             'title': title,
 225             'thumbnail': thumbnail,
 226             'description': description,
 227         }