Merge branch 'douyutv' of https://github.com/bonfy/youtube-dl into bonfy-douyutv

[youtube-dl] / youtube_dl / extractor / twitch.py

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 340cadcf5f2a00f855d4cbb47c1b46ba09ec502a..cbdaf9c7ab0d8c878011b10f8c59d71c851007c5 100644 (file)
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -23,6 +23,7 @@ class TwitchBaseIE(InfoExtractor):
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/user/login'
+    _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
         if not isinstance(response, dict):
@@ -34,7 +35,15 @@ class TwitchBaseIE(InfoExtractor):
                 expected=True)
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata'):
-        response = super(TwitchBaseIE, self)._download_json(url, video_id, note)
+        headers = {
+            'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
+            'X-Requested-With': 'XMLHttpRequest',
+        }
+        for cookie in self._downloader.cookiejar:
+            if cookie.name == 'api_token':
+                headers['Twitch-Api-Token'] = cookie.value
+        request = compat_urllib_request.Request(url, headers=headers)
+        response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
         self._handle_error(response)
         return response
 
@@ -76,6 +85,14 @@ class TwitchBaseIE(InfoExtractor):
             raise ExtractorError(
                 'Unable to login: %s' % m.group('msg').strip(), expected=True)
 
+    def _prefer_source(self, formats):
+        try:
+            source = next(f for f in formats if f['format_id'] == 'Source')
+            source['preference'] = 10
+        except StopIteration:
+            pass  # No Source stream present
+        self._sort_formats(formats)
+
 
 class TwitchItemBaseIE(TwitchBaseIE):
     def _download_info(self, item, item_id):
@@ -200,6 +217,7 @@ class TwitchVodIE(TwitchItemBaseIE):
             '%s/vod/%s?nauth=%s&nauthsig=%s'
             % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
             item_id, 'mp4')
+        self._prefer_source(formats)
         info['formats'] = formats
         return info
 
@@ -220,12 +238,18 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
             response = self._download_json(
                 self._PLAYLIST_URL % (channel_id, offset, limit),
                 channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
-            videos = response['videos']
-            if not videos:
+            page_entries = self._extract_playlist_page(response)
+            if not page_entries:
                 break
-            entries.extend([self.url_result(video['url']) for video in videos])
+            entries.extend(page_entries)
             offset += limit
-        return self.playlist_result(entries, channel_id, channel_name)
+        return self.playlist_result(
+            [self.url_result(entry) for entry in set(entries)],
+            channel_id, channel_name)
+
+    def _extract_playlist_page(self, response):
+        videos = response.get('videos')
+        return [video['url'] for video in videos] if videos else []
 
     def _real_extract(self, url):
         return self._extract_playlist(self._match_id(url))
@@ -262,6 +286,31 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
     }
 
 
+class TwitchBookmarksIE(TwitchPlaylistBaseIE):
+    IE_NAME = 'twitch:bookmarks'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
+    _PLAYLIST_TYPE = 'bookmarks'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/ognos/profile/bookmarks',
+        'info_dict': {
+            'id': 'ognos',
+            'title': 'Ognos',
+        },
+        'playlist_mincount': 3,
+    }
+
+    def _extract_playlist_page(self, response):
+        entries = []
+        for bookmark in response.get('bookmarks', []):
+            video = bookmark.get('video')
+            if not video:
+                continue
+            entries.append(video['url'])
+        return entries
+
+
 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'
     _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
@@ -309,14 +358,14 @@ class TwitchStreamIE(TwitchBaseIE):
             'p': random.randint(1000000, 10000000),
             'player': 'twitchweb',
             'segment_preference': '4',
-            'sig': access_token['sig'],
-            'token': access_token['token'],
+            'sig': access_token['sig'].encode('utf-8'),
+            'token': access_token['token'].encode('utf-8'),
         }
-
         formats = self._extract_m3u8_formats(
             '%s/api/channel/hls/%s.m3u8?%s'
-            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
+            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query)),
             channel_id, 'mp4')
+        self._prefer_source(formats)
 
         view_count = stream.get('viewers')
         timestamp = parse_iso8601(stream.get('created_at'))