_ Git - youtube-dl/blob - youtube_dl/extractor/itv.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import uuid
   5 import xml.etree.ElementTree as etree
   6 import json
   7 import re
   8
   9 from .common import InfoExtractor
  10 from .brightcove import BrightcoveNewIE
  11 from ..compat import (
  12     compat_str,
  13     compat_etree_register_namespace,
  14 )
  15 from ..utils import (
  16     determine_ext,
  17     ExtractorError,
  18     extract_attributes,
  19     int_or_none,
  20     merge_dicts,
  21     parse_duration,
  22     smuggle_url,
  23     xpath_with_ns,
  24     xpath_element,
  25     xpath_text,
  26 )
  27
  28
  29 class ITVIE(InfoExtractor):
  30     _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
  31     _GEO_COUNTRIES = ['GB']
  32     _TESTS = [{
  33         'url': 'http://www.itv.com/hub/mr-bean-animated-series/2a2936a0053',
  34         'info_dict': {
  35             'id': '2a2936a0053',
  36             'ext': 'flv',
  37             'title': 'Home Movie',
  38         },
  39         'params': {
  40             # rtmp download
  41             'skip_download': True,
  42         },
  43     }, {
  44         # unavailable via data-playlist-url
  45         'url': 'https://www.itv.com/hub/through-the-keyhole/2a2271a0033',
  46         'only_matching': True,
  47     }, {
  48         # InvalidVodcrid
  49         'url': 'https://www.itv.com/hub/james-martins-saturday-morning/2a5159a0034',
  50         'only_matching': True,
  51     }, {
  52         # ContentUnavailable
  53         'url': 'https://www.itv.com/hub/whos-doing-the-dishes/2a2898a0024',
  54         'only_matching': True,
  55     }]
  56
  57     def _real_extract(self, url):
  58         video_id = self._match_id(url)
  59         webpage = self._download_webpage(url, video_id)
  60         params = extract_attributes(self._search_regex(
  61             r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))
  62
  63         ns_map = {
  64             'soapenv': 'http://schemas.xmlsoap.org/soap/envelope/',
  65             'tem': 'http://tempuri.org/',
  66             'itv': 'http://schemas.datacontract.org/2004/07/Itv.BB.Mercury.Common.Types',
  67             'com': 'http://schemas.itv.com/2009/05/Common',
  68         }
  69         for ns, full_ns in ns_map.items():
  70             compat_etree_register_namespace(ns, full_ns)
  71
  72         def _add_ns(name):
  73             return xpath_with_ns(name, ns_map)
  74
  75         def _add_sub_element(element, name):
  76             return etree.SubElement(element, _add_ns(name))
  77
  78         production_id = (
  79             params.get('data-video-autoplay-id') or
  80             '%s#001' % (
  81                 params.get('data-video-episode-id') or
  82                 video_id.replace('a', '/')))
  83
  84         req_env = etree.Element(_add_ns('soapenv:Envelope'))
  85         _add_sub_element(req_env, 'soapenv:Header')
  86         body = _add_sub_element(req_env, 'soapenv:Body')
  87         get_playlist = _add_sub_element(body, ('tem:GetPlaylist'))
  88         request = _add_sub_element(get_playlist, 'tem:request')
  89         _add_sub_element(request, 'itv:ProductionId').text = production_id
  90         _add_sub_element(request, 'itv:RequestGuid').text = compat_str(uuid.uuid4()).upper()
  91         vodcrid = _add_sub_element(request, 'itv:Vodcrid')
  92         _add_sub_element(vodcrid, 'com:Id')
  93         _add_sub_element(request, 'itv:Partition')
  94         user_info = _add_sub_element(get_playlist, 'tem:userInfo')
  95         _add_sub_element(user_info, 'itv:Broadcaster').text = 'Itv'
  96         _add_sub_element(user_info, 'itv:DM')
  97         _add_sub_element(user_info, 'itv:RevenueScienceValue')
  98         _add_sub_element(user_info, 'itv:SessionId')
  99         _add_sub_element(user_info, 'itv:SsoToken')
 100         _add_sub_element(user_info, 'itv:UserToken')
 101         site_info = _add_sub_element(get_playlist, 'tem:siteInfo')
 102         _add_sub_element(site_info, 'itv:AdvertisingRestriction').text = 'None'
 103         _add_sub_element(site_info, 'itv:AdvertisingSite').text = 'ITV'
 104         _add_sub_element(site_info, 'itv:AdvertisingType').text = 'Any'
 105         _add_sub_element(site_info, 'itv:Area').text = 'ITVPLAYER.VIDEO'
 106         _add_sub_element(site_info, 'itv:Category')
 107         _add_sub_element(site_info, 'itv:Platform').text = 'DotCom'
 108         _add_sub_element(site_info, 'itv:Site').text = 'ItvCom'
 109         device_info = _add_sub_element(get_playlist, 'tem:deviceInfo')
 110         _add_sub_element(device_info, 'itv:ScreenSize').text = 'Big'
 111         player_info = _add_sub_element(get_playlist, 'tem:playerInfo')
 112         _add_sub_element(player_info, 'itv:Version').text = '2'
 113
 114         headers = self.geo_verification_headers()
 115         headers.update({
 116             'Content-Type': 'text/xml; charset=utf-8',
 117             'SOAPAction': 'http://tempuri.org/PlaylistService/GetPlaylist',
 118         })
 119
 120         info = self._search_json_ld(webpage, video_id, default={})
 121         formats = []
 122         subtitles = {}
 123
 124         def extract_subtitle(sub_url):
 125             ext = determine_ext(sub_url, 'ttml')
 126             subtitles.setdefault('en', []).append({
 127                 'url': sub_url,
 128                 'ext': 'ttml' if ext == 'xml' else ext,
 129             })
 130
 131         resp_env = self._download_xml(
 132             params['data-playlist-url'], video_id,
 133             headers=headers, data=etree.tostring(req_env), fatal=False)
 134         if resp_env:
 135             playlist = xpath_element(resp_env, './/Playlist')
 136             if playlist is None:
 137                 fault_code = xpath_text(resp_env, './/faultcode')
 138                 fault_string = xpath_text(resp_env, './/faultstring')
 139                 if fault_code == 'InvalidGeoRegion':
 140                     self.raise_geo_restricted(
 141                         msg=fault_string, countries=self._GEO_COUNTRIES)
 142                 elif fault_code not in (
 143                         'InvalidEntity', 'InvalidVodcrid', 'ContentUnavailable'):
 144                     raise ExtractorError(
 145                         '%s said: %s' % (self.IE_NAME, fault_string), expected=True)
 146                 info.update({
 147                     'title': self._og_search_title(webpage),
 148                     'episode_title': params.get('data-video-episode'),
 149                     'series': params.get('data-video-title'),
 150                 })
 151             else:
 152                 title = xpath_text(playlist, 'EpisodeTitle', default=None)
 153                 info.update({
 154                     'title': title,
 155                     'episode_title': title,
 156                     'episode_number': int_or_none(xpath_text(playlist, 'EpisodeNumber')),
 157                     'series': xpath_text(playlist, 'ProgrammeTitle'),
 158                     'duration': parse_duration(xpath_text(playlist, 'Duration')),
 159                 })
 160                 video_element = xpath_element(playlist, 'VideoEntries/Video', fatal=True)
 161                 media_files = xpath_element(video_element, 'MediaFiles', fatal=True)
 162                 rtmp_url = media_files.attrib['base']
 163
 164                 for media_file in media_files.findall('MediaFile'):
 165                     play_path = xpath_text(media_file, 'URL')
 166                     if not play_path:
 167                         continue
 168                     tbr = int_or_none(media_file.get('bitrate'), 1000)
 169                     f = {
 170                         'format_id': 'rtmp' + ('-%d' % tbr if tbr else ''),
 171                         'play_path': play_path,
 172                         # Providing this swfVfy allows to avoid truncated downloads
 173                         'player_url': 'http://www.itv.com/mercury/Mercury_VideoPlayer.swf',
 174                         'page_url': url,
 175                         'tbr': tbr,
 176                         'ext': 'flv',
 177                     }
 178                     app = self._search_regex(
 179                         'rtmpe?://[^/]+/(.+)$', rtmp_url, 'app', default=None)
 180                     if app:
 181                         f.update({
 182                             'url': rtmp_url.split('?', 1)[0],
 183                             'app': app,
 184                         })
 185                     else:
 186                         f['url'] = rtmp_url
 187                     formats.append(f)
 188
 189                 for caption_url in video_element.findall('ClosedCaptioningURIs/URL'):
 190                     if caption_url.text:
 191                         extract_subtitle(caption_url.text)
 192
 193         ios_playlist_url = params.get('data-video-playlist') or params.get('data-video-id')
 194         hmac = params.get('data-video-hmac')
 195         if ios_playlist_url and hmac and re.match(r'https?://', ios_playlist_url):
 196             headers = self.geo_verification_headers()
 197             headers.update({
 198                 'Accept': 'application/vnd.itv.vod.playlist.v2+json',
 199                 'Content-Type': 'application/json',
 200                 'hmac': hmac.upper(),
 201             })
 202             ios_playlist = self._download_json(
 203                 ios_playlist_url, video_id, data=json.dumps({
 204                     'user': {
 205                         'itvUserId': '',
 206                         'entitlements': [],
 207                         'token': ''
 208                     },
 209                     'device': {
 210                         'manufacturer': 'Safari',
 211                         'model': '5',
 212                         'os': {
 213                             'name': 'Windows NT',
 214                             'version': '6.1',
 215                             'type': 'desktop'
 216                         }
 217                     },
 218                     'client': {
 219                         'version': '4.1',
 220                         'id': 'browser'
 221                     },
 222                     'variantAvailability': {
 223                         'featureset': {
 224                             'min': ['hls', 'aes', 'outband-webvtt'],
 225                             'max': ['hls', 'aes', 'outband-webvtt']
 226                         },
 227                         'platformTag': 'dotcom'
 228                     }
 229                 }).encode(), headers=headers, fatal=False)
 230             if ios_playlist:
 231                 video_data = ios_playlist.get('Playlist', {}).get('Video', {})
 232                 ios_base_url = video_data.get('Base')
 233                 for media_file in video_data.get('MediaFiles', []):
 234                     href = media_file.get('Href')
 235                     if not href:
 236                         continue
 237                     if ios_base_url:
 238                         href = ios_base_url + href
 239                     ext = determine_ext(href)
 240                     if ext == 'm3u8':
 241                         formats.extend(self._extract_m3u8_formats(
 242                             href, video_id, 'mp4', entry_protocol='m3u8_native',
 243                             m3u8_id='hls', fatal=False))
 244                     else:
 245                         formats.append({
 246                             'url': href,
 247                         })
 248                 subs = video_data.get('Subtitles')
 249                 if isinstance(subs, list):
 250                     for sub in subs:
 251                         if not isinstance(sub, dict):
 252                             continue
 253                         href = sub.get('Href')
 254                         if isinstance(href, compat_str):
 255                             extract_subtitle(href)
 256                 if not info.get('duration'):
 257                     info['duration'] = parse_duration(video_data.get('Duration'))
 258
 259         self._sort_formats(formats)
 260
 261         info.update({
 262             'id': video_id,
 263             'formats': formats,
 264             'subtitles': subtitles,
 265         })
 266
 267         webpage_info = self._search_json_ld(webpage, video_id, default={})
 268         if not webpage_info.get('title'):
 269             webpage_info['title'] = self._html_search_regex(
 270                 r'(?s)<h\d+[^>]+\bclass=["\'][^>]*episode-title["\'][^>]*>([^<]+)<',
 271                 webpage, 'title', default=None) or self._og_search_title(
 272                 webpage, default=None) or self._html_search_meta(
 273                 'twitter:title', webpage, 'title',
 274                 default=None) or webpage_info['episode']
 275
 276         return merge_dicts(info, webpage_info)
 277
 278
 279 class ITVBTCCIE(InfoExtractor):
 280     _VALID_URL = r'https?://(?:www\.)?itv\.com/btcc/(?:[^/]+/)*(?P<id>[^/?#&]+)'
 281     _TEST = {
 282         'url': 'http://www.itv.com/btcc/races/btcc-2018-all-the-action-from-brands-hatch',
 283         'info_dict': {
 284             'id': 'btcc-2018-all-the-action-from-brands-hatch',
 285             'title': 'BTCC 2018: All the action from Brands Hatch',
 286         },
 287         'playlist_mincount': 9,
 288     }
 289     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1582188683001/HkiHLnNRx_default/index.html?videoId=%s'
 290
 291     def _real_extract(self, url):
 292         playlist_id = self._match_id(url)
 293
 294         webpage = self._download_webpage(url, playlist_id)
 295
 296         entries = [
 297             self.url_result(
 298                 smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {
 299                     # ITV does not like some GB IP ranges, so here are some
 300                     # IP blocks it accepts
 301                     'geo_ip_blocks': [
 302                         '193.113.0.0/16', '54.36.162.0/23', '159.65.16.0/21'
 303                     ],
 304                     'referrer': url,
 305                 }),
 306                 ie=BrightcoveNewIE.ie_key(), video_id=video_id)
 307             for video_id in re.findall(r'data-video-id=["\'](\d+)', webpage)]
 308
 309         title = self._og_search_title(webpage, fatal=False)
 310
 311         return self.playlist_result(entries, playlist_id, title)