_ Git - youtube-dl/blob - youtube_dl/extractor/ted.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4 import re
   5
   6 from .common import InfoExtractor
   7
   8 from ..compat import (
   9     compat_str,
  10 )
  11
  12
  13 class TEDIE(InfoExtractor):
  14     _VALID_URL = r'''(?x)
  15         (?P<proto>https?://)
  16         (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
  17         (
  18             (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
  19             |
  20             ((?P<type_talk>talks)) # We have a simple talk
  21             |
  22             (?P<type_watch>watch)/[^/]+/[^/]+
  23         )
  24         (/lang/(.*?))? # The url may contain the language
  25         /(?P<name>[\w-]+) # Here goes the name and then ".html"
  26         .*)$
  27         '''
  28     _TESTS = [{
  29         'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
  30         'md5': 'fc94ac279feebbce69f21c0c6ee82810',
  31         'info_dict': {
  32             'id': '102',
  33             'ext': 'mp4',
  34             'title': 'The illusion of consciousness',
  35             'description': ('Philosopher Dan Dennett makes a compelling '
  36                             'argument that not only don\'t we understand our own '
  37                             'consciousness, but that half the time our brains are '
  38                             'actively fooling us.'),
  39             'uploader': 'Dan Dennett',
  40             'width': 854,
  41             'duration': 1308,
  42         }
  43     }, {
  44         'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
  45         'md5': '226f4fb9c62380d11b7995efa4c87994',
  46         'info_dict': {
  47             'id': 'vishal-sikka-the-beauty-and-power-of-algorithms',
  48             'ext': 'mp4',
  49             'title': 'Vishal Sikka: The beauty and power of algorithms',
  50             'thumbnail': 're:^https?://.+\.jpg',
  51             'description': 'Adaptive, intelligent, and consistent, algorithms are emerging as the ultimate app for everything from matching consumers to products to assessing medical diagnoses. Vishal Sikka shares his appreciation for the algorithm, charting both its inherent beauty and its growing power.',
  52         }
  53     }, {
  54         'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
  55         'info_dict': {
  56             'id': '1972',
  57             'ext': 'mp4',
  58             'title': 'Be passionate. Be courageous. Be your best.',
  59             'uploader': 'Gabby Giffords and Mark Kelly',
  60             'description': 'md5:5174aed4d0f16021b704120360f72b92',
  61             'duration': 1128,
  62         },
  63     }, {
  64         'url': 'http://www.ted.com/playlists/who_are_the_hackers',
  65         'info_dict': {
  66             'id': '10',
  67             'title': 'Who are the hackers?',
  68         },
  69         'playlist_mincount': 6,
  70     }, {
  71         # contains a youtube video
  72         'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
  73         'add_ie': ['Youtube'],
  74         'info_dict': {
  75             'id': '_ZG8HBuDjgc',
  76             'ext': 'mp4',
  77             'title': 'Douglas Adams: Parrots the Universe and Everything',
  78             'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
  79             'uploader': 'University of California Television (UCTV)',
  80             'uploader_id': 'UCtelevision',
  81             'upload_date': '20080522',
  82         },
  83         'params': {
  84             'skip_download': True,
  85         },
  86     }]
  87
  88     _NATIVE_FORMATS = {
  89         'low': {'preference': 1, 'width': 320, 'height': 180},
  90         'medium': {'preference': 2, 'width': 512, 'height': 288},
  91         'high': {'preference': 3, 'width': 854, 'height': 480},
  92     }
  93
  94     def _extract_info(self, webpage):
  95         info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
  96                                        webpage, 'info json')
  97         return json.loads(info_json)
  98
  99     def _real_extract(self, url):
 100         m = re.match(self._VALID_URL, url, re.VERBOSE)
 101         if m.group('type').startswith('embed'):
 102             desktop_url = m.group('proto') + 'www' + m.group('urlmain')
 103             return self.url_result(desktop_url, 'TED')
 104         name = m.group('name')
 105         if m.group('type_talk'):
 106             return self._talk_info(url, name)
 107         elif m.group('type_watch'):
 108             return self._watch_info(url, name)
 109         else:
 110             return self._playlist_videos_info(url, name)
 111
 112     def _playlist_videos_info(self, url, name):
 113         '''Returns the videos of the playlist'''
 114
 115         webpage = self._download_webpage(url, name,
 116                                          'Downloading playlist webpage')
 117         info = self._extract_info(webpage)
 118         playlist_info = info['playlist']
 119
 120         playlist_entries = [
 121             self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
 122             for talk in info['talks']
 123         ]
 124         return self.playlist_result(
 125             playlist_entries,
 126             playlist_id=compat_str(playlist_info['id']),
 127             playlist_title=playlist_info['title'])
 128
 129     def _talk_info(self, url, video_name):
 130         webpage = self._download_webpage(url, video_name)
 131         self.report_extraction(video_name)
 132
 133         talk_info = self._extract_info(webpage)['talks'][0]
 134
 135         if talk_info.get('external') is not None:
 136             self.to_screen('Found video from %s' % talk_info['external']['service'])
 137             return {
 138                 '_type': 'url',
 139                 'url': talk_info['external']['uri'],
 140             }
 141
 142         formats = [{
 143             'url': format_url,
 144             'format_id': format_id,
 145             'format': format_id,
 146         } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
 147         if formats:
 148             for f in formats:
 149                 finfo = self._NATIVE_FORMATS.get(f['format_id'])
 150                 if finfo:
 151                     f.update(finfo)
 152         else:
 153             # Use rtmp downloads
 154             formats = [{
 155                 'format_id': f['name'],
 156                 'url': talk_info['streamer'],
 157                 'play_path': f['file'],
 158                 'ext': 'flv',
 159                 'width': f['width'],
 160                 'height': f['height'],
 161                 'tbr': f['bitrate'],
 162             } for f in talk_info['resources']['rtmp']]
 163         self._sort_formats(formats)
 164
 165         video_id = compat_str(talk_info['id'])
 166
 167         thumbnail = talk_info['thumb']
 168         if not thumbnail.startswith('http'):
 169             thumbnail = 'http://' + thumbnail
 170         return {
 171             'id': video_id,
 172             'title': talk_info['title'].strip(),
 173             'uploader': talk_info['speaker'],
 174             'thumbnail': thumbnail,
 175             'description': self._og_search_description(webpage),
 176             'subtitles': self._get_subtitles(video_id, talk_info),
 177             'formats': formats,
 178             'duration': talk_info.get('duration'),
 179         }
 180
 181     def _get_subtitles(self, video_id, talk_info):
 182         languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
 183         if languages:
 184             sub_lang_list = {}
 185             for l in languages:
 186                 sub_lang_list[l] = [
 187                     {
 188                         'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, l, ext),
 189                         'ext': ext,
 190                     }
 191                     for ext in ['ted', 'srt']
 192                 ]
 193             return sub_lang_list
 194         else:
 195             return {}
 196
 197     def _watch_info(self, url, name):
 198         webpage = self._download_webpage(url, name)
 199
 200         config_json = self._html_search_regex(
 201             r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*</script>',
 202             webpage, 'config')
 203         config = json.loads(config_json)['config']
 204         video_url = config['video']['url']
 205         thumbnail = config.get('image', {}).get('url')
 206
 207         title = self._html_search_regex(
 208             r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
 209         description = self._html_search_regex(
 210             [
 211                 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
 212                 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
 213             ],
 214             webpage, 'description', fatal=False)
 215
 216         return {
 217             'id': name,
 218             'url': video_url,
 219             'title': title,
 220             'thumbnail': thumbnail,
 221             'description': description,
 222         }