]> git.bitcoin.ninja Git - youtube-dl/blob - youtube_dl/extractor/itv.py
[pornhub] Improve locked videos detection (closes #22449, closes #22780)
[youtube-dl] / youtube_dl / extractor / itv.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import uuid
5 import xml.etree.ElementTree as etree
6 import json
7 import re
8
9 from .common import InfoExtractor
10 from .brightcove import BrightcoveNewIE
11 from ..compat import (
12     compat_str,
13     compat_etree_register_namespace,
14 )
15 from ..utils import (
16     determine_ext,
17     ExtractorError,
18     extract_attributes,
19     int_or_none,
20     merge_dicts,
21     parse_duration,
22     smuggle_url,
23     url_or_none,
24     xpath_with_ns,
25     xpath_element,
26     xpath_text,
27 )
28
29
30 class ITVIE(InfoExtractor):
31     _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
32     _GEO_COUNTRIES = ['GB']
33     _TESTS = [{
34         'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',
35         'info_dict': {
36             'id': '2a2936a0053',
37             'ext': 'flv',
38             'title': 'Home Movie',
39         },
40         'params': {
41             # rtmp download
42             'skip_download': True,
43         },
44     }, {
45         # unavailable via data-playlist-url
46         'url': 'https://www.itv.com/hub/through-the-keyhole/2a2271a0033',
47         'only_matching': True,
48     }, {
49         # InvalidVodcrid
50         'url': 'https://www.itv.com/hub/james-martins-saturday-morning/2a5159a0034',
51         'only_matching': True,
52     }, {
53         # ContentUnavailable
54         'url': 'https://www.itv.com/hub/whos-doing-the-dishes/2a2898a0024',
55         'only_matching': True,
56     }]
57
58     def _real_extract(self, url):
59         video_id = self._match_id(url)
60         webpage = self._download_webpage(url, video_id)
61         params = extract_attributes(self._search_regex(
62             r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))
63
64         ns_map = {
65             'soapenv': 'http://schemas.xmlsoap.org/soap/envelope/',
66             'tem': 'http://tempuri.org/',
67             'itv': 'http://schemas.datacontract.org/2004/07/Itv.BB.Mercury.Common.Types',
68             'com': 'http://schemas.itv.com/2009/05/Common',
69         }
70         for ns, full_ns in ns_map.items():
71             compat_etree_register_namespace(ns, full_ns)
72
73         def _add_ns(name):
74             return xpath_with_ns(name, ns_map)
75
76         def _add_sub_element(element, name):
77             return etree.SubElement(element, _add_ns(name))
78
79         production_id = (
80             params.get('data-video-autoplay-id')
81             or '%s#001' % (
82                 params.get('data-video-episode-id')
83                 or video_id.replace('a', '/')))
84
85         req_env = etree.Element(_add_ns('soapenv:Envelope'))
86         _add_sub_element(req_env, 'soapenv:Header')
87         body = _add_sub_element(req_env, 'soapenv:Body')
88         get_playlist = _add_sub_element(body, ('tem:GetPlaylist'))
89         request = _add_sub_element(get_playlist, 'tem:request')
90         _add_sub_element(request, 'itv:ProductionId').text = production_id
91         _add_sub_element(request, 'itv:RequestGuid').text = compat_str(uuid.uuid4()).upper()
92         vodcrid = _add_sub_element(request, 'itv:Vodcrid')
93         _add_sub_element(vodcrid, 'com:Id')
94         _add_sub_element(request, 'itv:Partition')
95         user_info = _add_sub_element(get_playlist, 'tem:userInfo')
96         _add_sub_element(user_info, 'itv:Broadcaster').text = 'Itv'
97         _add_sub_element(user_info, 'itv:DM')
98         _add_sub_element(user_info, 'itv:RevenueScienceValue')
99         _add_sub_element(user_info, 'itv:SessionId')
100         _add_sub_element(user_info, 'itv:SsoToken')
101         _add_sub_element(user_info, 'itv:UserToken')
102         site_info = _add_sub_element(get_playlist, 'tem:siteInfo')
103         _add_sub_element(site_info, 'itv:AdvertisingRestriction').text = 'None'
104         _add_sub_element(site_info, 'itv:AdvertisingSite').text = 'ITV'
105         _add_sub_element(site_info, 'itv:AdvertisingType').text = 'Any'
106         _add_sub_element(site_info, 'itv:Area').text = 'ITVPLAYER.VIDEO'
107         _add_sub_element(site_info, 'itv:Category')
108         _add_sub_element(site_info, 'itv:Platform').text = 'DotCom'
109         _add_sub_element(site_info, 'itv:Site').text = 'ItvCom'
110         device_info = _add_sub_element(get_playlist, 'tem:deviceInfo')
111         _add_sub_element(device_info, 'itv:ScreenSize').text = 'Big'
112         player_info = _add_sub_element(get_playlist, 'tem:playerInfo')
113         _add_sub_element(player_info, 'itv:Version').text = '2'
114
115         headers = self.geo_verification_headers()
116         headers.update({
117             'Content-Type': 'text/xml; charset=utf-8',
118             'SOAPAction': 'http://tempuri.org/PlaylistService/GetPlaylist',
119         })
120
121         info = self._search_json_ld(webpage, video_id, default={})
122         formats = []
123         subtitles = {}
124
125         def extract_subtitle(sub_url):
126             ext = determine_ext(sub_url, 'ttml')
127             subtitles.setdefault('en', []).append({
128                 'url': sub_url,
129                 'ext': 'ttml' if ext == 'xml' else ext,
130             })
131
132         resp_env = self._download_xml(
133             params['data-playlist-url'], video_id,
134             headers=headers, data=etree.tostring(req_env), fatal=False)
135         if resp_env:
136             playlist = xpath_element(resp_env, './/Playlist')
137             if playlist is None:
138                 fault_code = xpath_text(resp_env, './/faultcode')
139                 fault_string = xpath_text(resp_env, './/faultstring')
140                 if fault_code == 'InvalidGeoRegion':
141                     self.raise_geo_restricted(
142                         msg=fault_string, countries=self._GEO_COUNTRIES)
143                 elif fault_code not in (
144                         'InvalidEntity', 'InvalidVodcrid', 'ContentUnavailable'):
145                     raise ExtractorError(
146                         '%s said: %s' % (self.IE_NAME, fault_string), expected=True)
147                 info.update({
148                     'title': self._og_search_title(webpage),
149                     'episode_title': params.get('data-video-episode'),
150                     'series': params.get('data-video-title'),
151                 })
152             else:
153                 title = xpath_text(playlist, 'EpisodeTitle', default=None)
154                 info.update({
155                     'title': title,
156                     'episode_title': title,
157                     'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
158                     'series': xpath_text(playlist, 'ProgrammeTitle'),
159                     'duration': parse_duration(xpath_text(playlist, 'Duration')),
160                 })
161                 video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
162                 media_files = xpath_element(video_element, 'MediaFiles', fatal=True)
163                 rtmp_url = media_files.attrib['base']
164
165                 for media_file in media_files.findall('MediaFile'):
166                     play_path = xpath_text(media_file, 'URL')
167                     if not play_path:
168                         continue
169                     tbr = int_or_none(media_file.get('bitrate'), 1000)
170                     f = {
171                         'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
172                         'play_path': play_path,
173                         # Providing this swfVfy allows to avoid truncated downloads
174                         'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf',
175                         'page_url': url,
176                         'tbr': tbr,
177                         'ext': 'flv',
178                     }
179                     app = self._search_regex(
180                         'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None)
181                     if app:
182                         f.update({
183                             'url': rtmp_url.split('?', 1)[0],
184                             'app': app,
185                         })
186                     else:
187                         f['url'] = rtmp_url
188                     formats.append(f)
189
190                 for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):
191                     if caption_url.text:
192                         extract_subtitle(caption_url.text)
193
194         ios_playlist_url = params.get('data-video-playlist') or params.get('data-video-id')
195         hmac = params.get('data-video-hmac')
196         if ios_playlist_url and hmac and re.match(r'https?://', ios_playlist_url):
197             headers = self.geo_verification_headers()
198             headers.update({
199                 'Accept': 'application/vnd.itv.vod.playlist.v2+json',
200                 'Content-Type': 'application/json',
201                 'hmac': hmac.upper(),
202             })
203             ios_playlist = self._download_json(
204                 ios_playlist_url, video_id, data=json.dumps({
205                     'user': {
206                         'itvUserId': '',
207                         'entitlements': [],
208                         'token': ''
209                     },
210                     'device': {
211                         'manufacturer': 'Safari',
212                         'model': '5',
213                         'os': {
214                             'name': 'Windows NT',
215                             'version': '6.1',
216                             'type': 'desktop'
217                         }
218                     },
219                     'client': {
220                         'version': '4.1',
221                         'id': 'browser'
222                     },
223                     'variantAvailability': {
224                         'featureset': {
225                             'min': ['hls', 'aes', 'outband-webvtt'],
226                             'max': ['hls', 'aes', 'outband-webvtt']
227                         },
228                         'platformTag': 'dotcom'
229                     }
230                 }).encode(), headers=headers, fatal=False)
231             if ios_playlist:
232                 video_data = ios_playlist.get('Playlist', {}).get('Video', {})
233                 ios_base_url = video_data.get('Base')
234                 for media_file in video_data.get('MediaFiles', []):
235                     href = media_file.get('Href')
236                     if not href:
237                         continue
238                     if ios_base_url:
239                         href = ios_base_url + href
240                     ext = determine_ext(href)
241                     if ext == 'm3u8':
242                         formats.extend(self._extract_m3u8_formats(
243                             href, video_id, 'mp4', entry_protocol='m3u8_native',
244                             m3u8_id='hls', fatal=False))
245                     else:
246                         formats.append({
247                             'url': href,
248                         })
249                 subs = video_data.get('Subtitles')
250                 if isinstance(subs, list):
251                     for sub in subs:
252                         if not isinstance(sub, dict):
253                             continue
254                         href = url_or_none(sub.get('Href'))
255                         if href:
256                             extract_subtitle(href)
257                 if not info.get('duration'):
258                     info['duration'] = parse_duration(video_data.get('Duration'))
259
260         self._sort_formats(formats)
261
262         info.update({
263             'id': video_id,
264             'formats': formats,
265             'subtitles': subtitles,
266         })
267
268         webpage_info = self._search_json_ld(webpage, video_id, default={})
269         if not webpage_info.get('title'):
270             webpage_info['title'] = self._html_search_regex(
271                 r'(?s)<h\d+[^>]+\bclass=["\'][^>]*episode-title["\'][^>]*>([^<]+)<',
272                 webpage, 'title', default=None) or self._og_search_title(
273                 webpage, default=None) or self._html_search_meta(
274                 'twitter:title', webpage, 'title',
275                 default=None) or webpage_info['episode']
276
277         return merge_dicts(info, webpage_info)
278
279
280 class ITVBTCCIE(InfoExtractor):
281     _VALID_URL = r'https?://(?:www\.)?itv\.com/btcc/(?:[^/]+/)*(?P<id>[^/?#&]+)'
282     _TEST = {
283         'url': 'http://www.itv.com/btcc/races/btcc-2018-all-the-action-from-brands-hatch',
284         'info_dict': {
285             'id': 'btcc-2018-all-the-action-from-brands-hatch',
286             'title': 'BTCC 2018: All the action from Brands Hatch',
287         },
288         'playlist_mincount': 9,
289     }
290     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1582188683001/HkiHLnNRx_default/index.html?videoId=%s'
291
292     def _real_extract(self, url):
293         playlist_id = self._match_id(url)
294
295         webpage = self._download_webpage(url, playlist_id)
296
297         entries = [
298             self.url_result(
299                 smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {
300                     # ITV does not like some GB IP ranges, so here are some
301                     # IP blocks it accepts
302                     'geo_ip_blocks': [
303                         '193.113.0.0/16', '54.36.162.0/23', '159.65.16.0/21'
304                     ],
305                     'referrer': url,
306                 }),
307                 ie=BrightcoveNewIE.ie_key(), video_id=video_id)
308             for video_id in re.findall(r'data-video-id=["\'](\d+)', webpage)]
309
310         title = self._og_search_title(webpage, fatal=False)
311
312         return self.playlist_result(entries, playlist_id, title)