]> git.bitcoin.ninja Git - youtube-dl/blob - youtube_dl/extractor/twitch.py
68f50487b8a13b1cda9e2bc966a53a546145c531
[youtube-dl] / youtube_dl / extractor / twitch.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import itertools
5 import re
6 import random
7
8 from .common import InfoExtractor
9 from ..compat import (
10     compat_parse_qs,
11     compat_str,
12     compat_urllib_parse_urlencode,
13     compat_urllib_parse_urlparse,
14     compat_urlparse,
15 )
16 from ..utils import (
17     ExtractorError,
18     int_or_none,
19     orderedSet,
20     parse_duration,
21     parse_iso8601,
22     sanitized_Request,
23     urlencode_postdata,
24 )
25
26
27 class TwitchBaseIE(InfoExtractor):
28     _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
29
30     _API_BASE = 'https://api.twitch.tv'
31     _USHER_BASE = 'http://usher.twitch.tv'
32     _LOGIN_URL = 'http://www.twitch.tv/login'
33     _NETRC_MACHINE = 'twitch'
34
35     def _handle_error(self, response):
36         if not isinstance(response, dict):
37             return
38         error = response.get('error')
39         if error:
40             raise ExtractorError(
41                 '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
42                 expected=True)
43
44     def _download_json(self, url, video_id, note='Downloading JSON metadata'):
45         headers = {
46             'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
47             'X-Requested-With': 'XMLHttpRequest',
48         }
49         for cookie in self._downloader.cookiejar:
50             if cookie.name == 'api_token':
51                 headers['Twitch-Api-Token'] = cookie.value
52         request = sanitized_Request(url, headers=headers)
53         response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
54         self._handle_error(response)
55         return response
56
57     def _real_initialize(self):
58         self._login()
59
60     def _login(self):
61         (username, password) = self._get_login_info()
62         if username is None:
63             return
64
65         login_page, handle = self._download_webpage_handle(
66             self._LOGIN_URL, None, 'Downloading login page')
67
68         login_form = self._hidden_inputs(login_page)
69
70         login_form.update({
71             'username': username,
72             'password': password,
73         })
74
75         redirect_url = handle.geturl()
76
77         post_url = self._search_regex(
78             r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
79             'post url', default=redirect_url, group='url')
80
81         if not post_url.startswith('http'):
82             post_url = compat_urlparse.urljoin(redirect_url, post_url)
83
84         request = sanitized_Request(
85             post_url, urlencode_postdata(login_form))
86         request.add_header('Referer', redirect_url)
87         response = self._download_webpage(
88             request, None, 'Logging in as %s' % username)
89
90         error_message = self._search_regex(
91             r'<div[^>]+class="subwindow_notice"[^>]*>([^<]+)</div>',
92             response, 'error message', default=None)
93         if error_message:
94             raise ExtractorError(
95                 'Unable to login. Twitch said: %s' % error_message, expected=True)
96
97         if '>Reset your password<' in response:
98             self.report_warning('Twitch asks you to reset your password, go to https://secure.twitch.tv/reset/submit')
99
100     def _prefer_source(self, formats):
101         try:
102             source = next(f for f in formats if f['format_id'] == 'Source')
103             source['preference'] = 10
104         except StopIteration:
105             pass  # No Source stream present
106         self._sort_formats(formats)
107
108
109 class TwitchItemBaseIE(TwitchBaseIE):
110     def _download_info(self, item, item_id):
111         return self._extract_info(self._download_json(
112             '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
113             'Downloading %s info JSON' % self._ITEM_TYPE))
114
115     def _extract_media(self, item_id):
116         info = self._download_info(self._ITEM_SHORTCUT, item_id)
117         response = self._download_json(
118             '%s/api/videos/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
119             'Downloading %s playlist JSON' % self._ITEM_TYPE)
120         entries = []
121         chunks = response['chunks']
122         qualities = list(chunks.keys())
123         for num, fragment in enumerate(zip(*chunks.values()), start=1):
124             formats = []
125             for fmt_num, fragment_fmt in enumerate(fragment):
126                 format_id = qualities[fmt_num]
127                 fmt = {
128                     'url': fragment_fmt['url'],
129                     'format_id': format_id,
130                     'quality': 1 if format_id == 'live' else 0,
131                 }
132                 m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
133                 if m:
134                     fmt['height'] = int(m.group('height'))
135                 formats.append(fmt)
136             self._sort_formats(formats)
137             entry = dict(info)
138             entry['id'] = '%s_%d' % (entry['id'], num)
139             entry['title'] = '%s part %d' % (entry['title'], num)
140             entry['formats'] = formats
141             entries.append(entry)
142         return self.playlist_result(entries, info['id'], info['title'])
143
144     def _extract_info(self, info):
145         return {
146             'id': info['_id'],
147             'title': info.get('title') or 'Untitled Broadcast',
148             'description': info.get('description'),
149             'duration': int_or_none(info.get('length')),
150             'thumbnail': info.get('preview'),
151             'uploader': info.get('channel', {}).get('display_name'),
152             'uploader_id': info.get('channel', {}).get('name'),
153             'timestamp': parse_iso8601(info.get('recorded_at')),
154             'view_count': int_or_none(info.get('views')),
155         }
156
157     def _real_extract(self, url):
158         return self._extract_media(self._match_id(url))
159
160
161 class TwitchVideoIE(TwitchItemBaseIE):
162     IE_NAME = 'twitch:video'
163     _VALID_URL = r'%s/[^/]+/b/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
164     _ITEM_TYPE = 'video'
165     _ITEM_SHORTCUT = 'a'
166
167     _TEST = {
168         'url': 'http://www.twitch.tv/riotgames/b/577357806',
169         'info_dict': {
170             'id': 'a577357806',
171             'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
172         },
173         'playlist_mincount': 12,
174         'skip': 'HTTP Error 404: Not Found',
175     }
176
177
178 class TwitchChapterIE(TwitchItemBaseIE):
179     IE_NAME = 'twitch:chapter'
180     _VALID_URL = r'%s/[^/]+/c/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
181     _ITEM_TYPE = 'chapter'
182     _ITEM_SHORTCUT = 'c'
183
184     _TESTS = [{
185         'url': 'http://www.twitch.tv/acracingleague/c/5285812',
186         'info_dict': {
187             'id': 'c5285812',
188             'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
189         },
190         'playlist_mincount': 3,
191         'skip': 'HTTP Error 404: Not Found',
192     }, {
193         'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
194         'only_matching': True,
195     }]
196
197
198 class TwitchVodIE(TwitchItemBaseIE):
199     IE_NAME = 'twitch:vod'
200     _VALID_URL = r'%s/[^/]+/v/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
201     _ITEM_TYPE = 'vod'
202     _ITEM_SHORTCUT = 'v'
203
204     _TESTS = [{
205         'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
206         'info_dict': {
207             'id': 'v6528877',
208             'ext': 'mp4',
209             'title': 'LCK Summer Split - Week 6 Day 1',
210             'thumbnail': 're:^https?://.*\.jpg$',
211             'duration': 17208,
212             'timestamp': 1435131709,
213             'upload_date': '20150624',
214             'uploader': 'Riot Games',
215             'uploader_id': 'riotgames',
216             'view_count': int,
217             'start_time': 310,
218         },
219         'params': {
220             # m3u8 download
221             'skip_download': True,
222         },
223     }, {
224         # Untitled broadcast (title is None)
225         'url': 'http://www.twitch.tv/belkao_o/v/11230755',
226         'info_dict': {
227             'id': 'v11230755',
228             'ext': 'mp4',
229             'title': 'Untitled Broadcast',
230             'thumbnail': 're:^https?://.*\.jpg$',
231             'duration': 1638,
232             'timestamp': 1439746708,
233             'upload_date': '20150816',
234             'uploader': 'BelkAO_o',
235             'uploader_id': 'belkao_o',
236             'view_count': int,
237         },
238         'params': {
239             # m3u8 download
240             'skip_download': True,
241         },
242     }]
243
244     def _real_extract(self, url):
245         item_id = self._match_id(url)
246
247         info = self._download_info(self._ITEM_SHORTCUT, item_id)
248         access_token = self._download_json(
249             '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
250             'Downloading %s access token' % self._ITEM_TYPE)
251
252         formats = self._extract_m3u8_formats(
253             '%s/vod/%s?%s' % (
254                 self._USHER_BASE, item_id,
255                 compat_urllib_parse_urlencode({
256                     'allow_source': 'true',
257                     'allow_audio_only': 'true',
258                     'allow_spectre': 'true',
259                     'player': 'twitchweb',
260                     'nauth': access_token['token'],
261                     'nauthsig': access_token['sig'],
262                 })),
263             item_id, 'mp4')
264
265         self._prefer_source(formats)
266         info['formats'] = formats
267
268         parsed_url = compat_urllib_parse_urlparse(url)
269         query = compat_parse_qs(parsed_url.query)
270         if 't' in query:
271             info['start_time'] = parse_duration(query['t'][0])
272
273         return info
274
275
276 class TwitchPlaylistBaseIE(TwitchBaseIE):
277     _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
278     _PAGE_LIMIT = 100
279
280     def _extract_playlist(self, channel_id):
281         info = self._download_json(
282             '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
283             channel_id, 'Downloading channel info JSON')
284         channel_name = info.get('display_name') or info.get('name')
285         entries = []
286         offset = 0
287         limit = self._PAGE_LIMIT
288         broken_paging_detected = False
289         counter_override = None
290         for counter in itertools.count(1):
291             response = self._download_json(
292                 self._PLAYLIST_URL % (channel_id, offset, limit),
293                 channel_id,
294                 'Downloading %s videos JSON page %s'
295                 % (self._PLAYLIST_TYPE, counter_override or counter))
296             page_entries = self._extract_playlist_page(response)
297             if not page_entries:
298                 break
299             total = int_or_none(response.get('_total'))
300             # Since the beginning of March 2016 twitch's paging mechanism
301             # is completely broken on the twitch side. It simply ignores
302             # a limit and returns the whole offset number of videos.
303             # Working around by just requesting all videos at once.
304             # Upd: pagination bug was fixed by twitch on 15.03.2016.
305             if not broken_paging_detected and total and len(page_entries) > limit:
306                 self.report_warning(
307                     'Twitch pagination is broken on twitch side, requesting all videos at once',
308                     channel_id)
309                 broken_paging_detected = True
310                 offset = total
311                 counter_override = '(all at once)'
312                 continue
313             entries.extend(page_entries)
314             if broken_paging_detected or total and len(page_entries) >= total:
315                 break
316             offset += limit
317         return self.playlist_result(
318             [self.url_result(entry) for entry in orderedSet(entries)],
319             channel_id, channel_name)
320
321     def _extract_playlist_page(self, response):
322         videos = response.get('videos')
323         return [video['url'] for video in videos] if videos else []
324
325     def _real_extract(self, url):
326         return self._extract_playlist(self._match_id(url))
327
328
329 class TwitchProfileIE(TwitchPlaylistBaseIE):
330     IE_NAME = 'twitch:profile'
331     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
332     _PLAYLIST_TYPE = 'profile'
333
334     _TEST = {
335         'url': 'http://www.twitch.tv/vanillatv/profile',
336         'info_dict': {
337             'id': 'vanillatv',
338             'title': 'VanillaTV',
339         },
340         'playlist_mincount': 412,
341     }
342
343
344 class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
345     IE_NAME = 'twitch:past_broadcasts'
346     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
347     _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
348     _PLAYLIST_TYPE = 'past broadcasts'
349
350     _TEST = {
351         'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
352         'info_dict': {
353             'id': 'spamfish',
354             'title': 'Spamfish',
355         },
356         'playlist_mincount': 54,
357     }
358
359
360 class TwitchBookmarksIE(TwitchPlaylistBaseIE):
361     IE_NAME = 'twitch:bookmarks'
362     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
363     _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
364     _PLAYLIST_TYPE = 'bookmarks'
365
366     _TEST = {
367         'url': 'http://www.twitch.tv/ognos/profile/bookmarks',
368         'info_dict': {
369             'id': 'ognos',
370             'title': 'Ognos',
371         },
372         'playlist_mincount': 3,
373         'skip': 'HTTP Error 404: Not Found',
374     }
375
376     def _extract_playlist_page(self, response):
377         entries = []
378         for bookmark in response.get('bookmarks', []):
379             video = bookmark.get('video')
380             if not video:
381                 continue
382             entries.append(video['url'])
383         return entries
384
385
386 class TwitchStreamIE(TwitchBaseIE):
387     IE_NAME = 'twitch:stream'
388     _VALID_URL = r'%s/(?P<id>[^/#?]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
389
390     _TESTS = [{
391         'url': 'http://www.twitch.tv/shroomztv',
392         'info_dict': {
393             'id': '12772022048',
394             'display_id': 'shroomztv',
395             'ext': 'mp4',
396             'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
397             'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
398             'is_live': True,
399             'timestamp': 1421928037,
400             'upload_date': '20150122',
401             'uploader': 'ShroomzTV',
402             'uploader_id': 'shroomztv',
403             'view_count': int,
404         },
405         'params': {
406             # m3u8 download
407             'skip_download': True,
408         },
409     }, {
410         'url': 'http://www.twitch.tv/miracle_doto#profile-0',
411         'only_matching': True,
412     }]
413
414     def _real_extract(self, url):
415         channel_id = self._match_id(url)
416
417         stream = self._download_json(
418             '%s/kraken/streams/%s' % (self._API_BASE, channel_id), channel_id,
419             'Downloading stream JSON').get('stream')
420
421         # Fallback on profile extraction if stream is offline
422         if not stream:
423             return self.url_result(
424                 'http://www.twitch.tv/%s/profile' % channel_id,
425                 'TwitchProfile', channel_id)
426
427         # Channel name may be typed if different case than the original channel name
428         # (e.g. http://www.twitch.tv/TWITCHPLAYSPOKEMON) that will lead to constructing
429         # an invalid m3u8 URL. Working around by use of original channel name from stream
430         # JSON and fallback to lowercase if it's not available.
431         channel_id = stream.get('channel', {}).get('name') or channel_id.lower()
432
433         access_token = self._download_json(
434             '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
435             'Downloading channel access token')
436
437         query = {
438             'allow_source': 'true',
439             'allow_audio_only': 'true',
440             'p': random.randint(1000000, 10000000),
441             'player': 'twitchweb',
442             'segment_preference': '4',
443             'sig': access_token['sig'].encode('utf-8'),
444             'token': access_token['token'].encode('utf-8'),
445         }
446         formats = self._extract_m3u8_formats(
447             '%s/api/channel/hls/%s.m3u8?%s'
448             % (self._USHER_BASE, channel_id, compat_urllib_parse_urlencode(query)),
449             channel_id, 'mp4')
450         self._prefer_source(formats)
451
452         view_count = stream.get('viewers')
453         timestamp = parse_iso8601(stream.get('created_at'))
454
455         channel = stream['channel']
456         title = self._live_title(channel.get('display_name') or channel.get('name'))
457         description = channel.get('status')
458
459         thumbnails = []
460         for thumbnail_key, thumbnail_url in stream['preview'].items():
461             m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
462             if not m:
463                 continue
464             thumbnails.append({
465                 'url': thumbnail_url,
466                 'width': int(m.group('width')),
467                 'height': int(m.group('height')),
468             })
469
470         return {
471             'id': compat_str(stream['_id']),
472             'display_id': channel_id,
473             'title': title,
474             'description': description,
475             'thumbnails': thumbnails,
476             'uploader': channel.get('display_name'),
477             'uploader_id': channel.get('name'),
478             'timestamp': timestamp,
479             'view_count': view_count,
480             'formats': formats,
481             'is_live': True,
482         }