]> git.bitcoin.ninja Git - youtube-dl/blob - youtube_dl/extractor/ustream.py
[pornhub:uservideos] Add support for multipage videos (Closes #9006)
[youtube-dl] / youtube_dl / extractor / ustream.py
1 from __future__ import unicode_literals
2
3 import re
4
5 from .common import InfoExtractor
6 from ..compat import (
7     compat_urlparse,
8 )
9 from ..utils import (
10     ExtractorError,
11     int_or_none,
12     float_or_none,
13 )
14
15
16 class UstreamIE(InfoExtractor):
17     _VALID_URL = r'https?://www\.ustream\.tv/(?P<type>recorded|embed|embed/recorded)/(?P<id>\d+)'
18     IE_NAME = 'ustream'
19     _TESTS = [{
20         'url': 'http://www.ustream.tv/recorded/20274954',
21         'md5': '088f151799e8f572f84eb62f17d73e5c',
22         'info_dict': {
23             'id': '20274954',
24             'ext': 'flv',
25             'title': 'Young Americans for Liberty February 7, 2012 2:28 AM',
26             'description': 'Young Americans for Liberty February 7, 2012 2:28 AM',
27             'timestamp': 1328577035,
28             'upload_date': '20120207',
29             'uploader': 'yaliberty',
30             'uploader_id': '6780869',
31         },
32     }, {
33         # From http://sportscanada.tv/canadagames/index.php/week2/figure-skating/444
34         # Title and uploader available only from params JSON
35         'url': 'http://www.ustream.tv/embed/recorded/59307601?ub=ff0000&lc=ff0000&oc=ffffff&uc=ffffff&v=3&wmode=direct',
36         'md5': '5a2abf40babeac9812ed20ae12d34e10',
37         'info_dict': {
38             'id': '59307601',
39             'ext': 'flv',
40             'title': '-CG11- Canada Games Figure Skating',
41             'uploader': 'sportscanadatv',
42         },
43         'skip': 'This Pro Broadcaster has chosen to remove this video from the ustream.tv site.',
44     }]
45
46     def _real_extract(self, url):
47         m = re.match(self._VALID_URL, url)
48         video_id = m.group('id')
49
50         # some sites use this embed format (see: https://github.com/rg3/youtube-dl/issues/2990)
51         if m.group('type') == 'embed/recorded':
52             video_id = m.group('id')
53             desktop_url = 'http://www.ustream.tv/recorded/' + video_id
54             return self.url_result(desktop_url, 'Ustream')
55         if m.group('type') == 'embed':
56             video_id = m.group('id')
57             webpage = self._download_webpage(url, video_id)
58             desktop_video_id = self._html_search_regex(
59                 r'ContentVideoIds=\["([^"]*?)"\]', webpage, 'desktop_video_id')
60             desktop_url = 'http://www.ustream.tv/recorded/' + desktop_video_id
61             return self.url_result(desktop_url, 'Ustream')
62
63         params = self._download_json(
64             'https://api.ustream.tv/videos/%s.json' % video_id, video_id)
65
66         error = params.get('error')
67         if error:
68             raise ExtractorError(
69                 '%s returned error: %s' % (self.IE_NAME, error), expected=True)
70
71         video = params['video']
72
73         title = video['title']
74         filesize = float_or_none(video.get('file_size'))
75
76         formats = [{
77             'id': video_id,
78             'url': video_url,
79             'ext': format_id,
80             'filesize': filesize,
81         } for format_id, video_url in video['media_urls'].items()]
82         self._sort_formats(formats)
83
84         description = video.get('description')
85         timestamp = int_or_none(video.get('created_at'))
86         duration = float_or_none(video.get('length'))
87         view_count = int_or_none(video.get('views'))
88
89         uploader = video.get('owner', {}).get('username')
90         uploader_id = video.get('owner', {}).get('id')
91
92         thumbnails = [{
93             'id': thumbnail_id,
94             'url': thumbnail_url,
95         } for thumbnail_id, thumbnail_url in video.get('thumbnail', {}).items()]
96
97         return {
98             'id': video_id,
99             'title': title,
100             'description': description,
101             'thumbnails': thumbnails,
102             'timestamp': timestamp,
103             'duration': duration,
104             'view_count': view_count,
105             'uploader': uploader,
106             'uploader_id': uploader_id,
107             'formats': formats,
108         }
109
110
111 class UstreamChannelIE(InfoExtractor):
112     _VALID_URL = r'https?://www\.ustream\.tv/channel/(?P<slug>.+)'
113     IE_NAME = 'ustream:channel'
114     _TEST = {
115         'url': 'http://www.ustream.tv/channel/channeljapan',
116         'info_dict': {
117             'id': '10874166',
118         },
119         'playlist_mincount': 17,
120     }
121
122     def _real_extract(self, url):
123         m = re.match(self._VALID_URL, url)
124         display_id = m.group('slug')
125         webpage = self._download_webpage(url, display_id)
126         channel_id = self._html_search_meta('ustream:channel_id', webpage)
127
128         BASE = 'http://www.ustream.tv'
129         next_url = '/ajax/socialstream/videos/%s/1.json' % channel_id
130         video_ids = []
131         while next_url:
132             reply = self._download_json(
133                 compat_urlparse.urljoin(BASE, next_url), display_id,
134                 note='Downloading video information (next: %d)' % (len(video_ids) + 1))
135             video_ids.extend(re.findall(r'data-content-id="(\d.*)"', reply['data']))
136             next_url = reply['nextUrl']
137
138         entries = [
139             self.url_result('http://www.ustream.tv/recorded/' + vid, 'Ustream')
140             for vid in video_ids]
141         return {
142             '_type': 'playlist',
143             'id': channel_id,
144             'display_id': display_id,
145             'entries': entries,
146         }