_ Git - youtube-dl/blob - youtube_dl/extractor/ted.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4 import re
   5
   6 from .common import InfoExtractor
   7
   8 from ..compat import (
   9     compat_str,
  10 )
  11
  12
  13 class TEDIE(InfoExtractor):
  14     _VALID_URL = r'''(?x)
  15         (?P<proto>https?://)
  16         (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
  17         (
  18             (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
  19             |
  20             ((?P<type_talk>talks)) # We have a simple talk
  21             |
  22             (?P<type_watch>watch)/[^/]+/[^/]+
  23         )
  24         (/lang/(.*?))? # The url may contain the language
  25         /(?P<name>[\w-]+) # Here goes the name and then ".html"
  26         .*)$
  27         '''
  28     _TESTS = [{
  29         'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
  30         'md5': 'fc94ac279feebbce69f21c0c6ee82810',
  31         'info_dict': {
  32             'id': '102',
  33             'ext': 'mp4',
  34             'title': 'The illusion of consciousness',
  35             'description': ('Philosopher Dan Dennett makes a compelling '
  36                             'argument that not only don\'t we understand our own '
  37                             'consciousness, but that half the time our brains are '
  38                             'actively fooling us.'),
  39             'uploader': 'Dan Dennett',
  40             'width': 854,
  41             'duration': 1308,
  42         }
  43     }, {
  44         'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
  45         'md5': '226f4fb9c62380d11b7995efa4c87994',
  46         'info_dict': {
  47             'id': 'vishal-sikka-the-beauty-and-power-of-algorithms',
  48             'ext': 'mp4',
  49             'title': 'Vishal Sikka: The beauty and power of algorithms',
  50             'thumbnail': 're:^https?://.+\.jpg',
  51             'description': 'Adaptive, intelligent, and consistent, algorithms are emerging as the ultimate app for everything from matching consumers to products to assessing medical diagnoses. Vishal Sikka shares his appreciation for the algorithm, charting both its inherent beauty and its growing power.',
  52         }
  53     }, {
  54         'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
  55         'info_dict': {
  56             'id': '1972',
  57             'ext': 'mp4',
  58             'title': 'Be passionate. Be courageous. Be your best.',
  59             'uploader': 'Gabby Giffords and Mark Kelly',
  60             'description': 'md5:5174aed4d0f16021b704120360f72b92',
  61             'duration': 1128,
  62         },
  63     }, {
  64         'url': 'http://www.ted.com/playlists/who_are_the_hackers',
  65         'info_dict': {
  66             'id': '10',
  67             'title': 'Who are the hackers?',
  68         },
  69         'playlist_mincount': 6,
  70     }, {
  71         # contains a youtube video
  72         'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
  73         'add_ie': ['Youtube'],
  74         'info_dict': {
  75             'id': '_ZG8HBuDjgc',
  76             'ext': 'mp4',
  77             'title': 'Douglas Adams: Parrots the Universe and Everything',
  78             'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
  79             'uploader': 'University of California Television (UCTV)',
  80             'uploader_id': 'UCtelevision',
  81             'upload_date': '20080522',
  82         },
  83         'params': {
  84             'skip_download': True,
  85         },
  86     }]
  87
  88     _NATIVE_FORMATS = {
  89         'low': {'preference': 1, 'width': 320, 'height': 180},
  90         'medium': {'preference': 2, 'width': 512, 'height': 288},
  91         'high': {'preference': 3, 'width': 854, 'height': 480},
  92     }
  93
  94     def _extract_info(self, webpage):
  95         info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
  96                                        webpage, 'info json')
  97         return json.loads(info_json)
  98
  99     def _real_extract(self, url):
 100         m = re.match(self._VALID_URL, url, re.VERBOSE)
 101         if m.group('type').startswith('embed'):
 102             desktop_url = m.group('proto') + 'www' + m.group('urlmain')
 103             return self.url_result(desktop_url, 'TED')
 104         name = m.group('name')
 105         if m.group('type_talk'):
 106             return self._talk_info(url, name)
 107         elif m.group('type_watch'):
 108             return self._watch_info(url, name)
 109         else:
 110             return self._playlist_videos_info(url, name)
 111
 112     def _playlist_videos_info(self, url, name):
 113         '''Returns the videos of the playlist'''
 114
 115         webpage = self._download_webpage(url, name,
 116                                          'Downloading playlist webpage')
 117         info = self._extract_info(webpage)
 118         playlist_info = info['playlist']
 119
 120         playlist_entries = [
 121             self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
 122             for talk in info['talks']
 123         ]
 124         return self.playlist_result(
 125             playlist_entries,
 126             playlist_id=compat_str(playlist_info['id']),
 127             playlist_title=playlist_info['title'])
 128
 129     def _talk_info(self, url, video_name):
 130         webpage = self._download_webpage(url, video_name)
 131         self.report_extraction(video_name)
 132
 133         talk_info = self._extract_info(webpage)['talks'][0]
 134
 135         if talk_info.get('external') is not None:
 136             self.to_screen('Found video from %s' % talk_info['external']['service'])
 137             return {
 138                 '_type': 'url',
 139                 'url': talk_info['external']['uri'],
 140             }
 141
 142         formats = [{
 143             'url': format_url,
 144             'format_id': format_id,
 145             'format': format_id,
 146         } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
 147         if formats:
 148             for f in formats:
 149                 finfo = self._NATIVE_FORMATS.get(f['format_id'])
 150                 if finfo:
 151                     f.update(finfo)
 152         else:
 153             # Use rtmp downloads
 154             formats = [{
 155                 'format_id': f['name'],
 156                 'url': talk_info['streamer'],
 157                 'play_path': f['file'],
 158                 'ext': 'flv',
 159                 'width': f['width'],
 160                 'height': f['height'],
 161                 'tbr': f['bitrate'],
 162             } for f in talk_info['resources']['rtmp']]
 163         self._sort_formats(formats)
 164
 165         video_id = compat_str(talk_info['id'])
 166         # subtitles
 167         video_subtitles = self.extract_subtitles(video_id, talk_info)
 168
 169         thumbnail = talk_info['thumb']
 170         if not thumbnail.startswith('http'):
 171             thumbnail = 'http://' + thumbnail
 172         return {
 173             'id': video_id,
 174             'title': talk_info['title'].strip(),
 175             'uploader': talk_info['speaker'],
 176             'thumbnail': thumbnail,
 177             'description': self._og_search_description(webpage),
 178             'subtitles': video_subtitles,
 179             'formats': formats,
 180             'duration': talk_info.get('duration'),
 181         }
 182
 183     def _get_subtitles(self, video_id, talk_info):
 184         languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
 185         if languages:
 186             sub_lang_list = {}
 187             for l in languages:
 188                 sub_lang_list[l] = [
 189                     {
 190                         'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, l, ext),
 191                         'ext': ext,
 192                     }
 193                     for ext in ['ted', 'srt']
 194                 ]
 195             return sub_lang_list
 196         else:
 197             self._downloader.report_warning('video doesn\'t have subtitles')
 198             return {}
 199
 200     def _watch_info(self, url, name):
 201         webpage = self._download_webpage(url, name)
 202
 203         config_json = self._html_search_regex(
 204             r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*</script>',
 205             webpage, 'config')
 206         config = json.loads(config_json)['config']
 207         video_url = config['video']['url']
 208         thumbnail = config.get('image', {}).get('url')
 209
 210         title = self._html_search_regex(
 211             r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
 212         description = self._html_search_regex(
 213             [
 214                 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
 215                 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
 216             ],
 217             webpage, 'description', fatal=False)
 218
 219         return {
 220             'id': name,
 221             'url': video_url,
 222             'title': title,
 223             'thumbnail': thumbnail,
 224             'description': description,
 225         }