]> git.bitcoin.ninja Git - youtube-dl/blob - youtube_dl/extractor/dreisat.py
[downloader/http] Randomize HTTP chunk size
[youtube-dl] / youtube_dl / extractor / dreisat.py
1 from __future__ import unicode_literals
2
3 import re
4
5 from .common import InfoExtractor
6 from ..utils import (
7     int_or_none,
8     unified_strdate,
9     xpath_text,
10     determine_ext,
11     qualities,
12     float_or_none,
13     ExtractorError,
14 )
15
16
17 class DreiSatIE(InfoExtractor):
18     IE_NAME = '3sat'
19     _VALID_URL = r'(?:https?://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php|mediathek\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
20     _TESTS = [
21         {
22             'url': 'http://www.3sat.de/mediathek/index.php?mode=play&obj=45918',
23             'md5': 'be37228896d30a88f315b638900a026e',
24             'info_dict': {
25                 'id': '45918',
26                 'ext': 'mp4',
27                 'title': 'Waidmannsheil',
28                 'description': 'md5:cce00ca1d70e21425e72c86a98a56817',
29                 'uploader': 'SCHWEIZWEIT',
30                 'uploader_id': '100000210',
31                 'upload_date': '20140913'
32             },
33             'params': {
34                 'skip_download': True,  # m3u8 downloads
35             }
36         },
37         {
38             'url': 'http://www.3sat.de/mediathek/mediathek.php?mode=play&obj=51066',
39             'only_matching': True,
40         },
41     ]
42
43     def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
44         param_groups = {}
45         for param_group in smil.findall(self._xpath_ns('./head/paramGroup', namespace)):
46             group_id = param_group.attrib.get(self._xpath_ns('id', 'http://www.w3.org/XML/1998/namespace'))
47             params = {}
48             for param in param_group:
49                 params[param.get('name')] = param.get('value')
50             param_groups[group_id] = params
51
52         formats = []
53         for video in smil.findall(self._xpath_ns('.//video', namespace)):
54             src = video.get('src')
55             if not src:
56                 continue
57             bitrate = float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
58             group_id = video.get('paramGroup')
59             param_group = param_groups[group_id]
60             for proto in param_group['protocols'].split(','):
61                 formats.append({
62                     'url': '%s://%s' % (proto, param_group['host']),
63                     'app': param_group['app'],
64                     'play_path': src,
65                     'ext': 'flv',
66                     'format_id': '%s-%d' % (proto, bitrate),
67                     'tbr': bitrate,
68                 })
69         self._sort_formats(formats)
70         return formats
71
72     def extract_from_xml_url(self, video_id, xml_url):
73         doc = self._download_xml(
74             xml_url, video_id,
75             note='Downloading video info',
76             errnote='Failed to download video info')
77
78         status_code = doc.find('./status/statuscode')
79         if status_code is not None and status_code.text != 'ok':
80             code = status_code.text
81             if code == 'notVisibleAnymore':
82                 message = 'Video %s is not available' % video_id
83             else:
84                 message = '%s returned error: %s' % (self.IE_NAME, code)
85             raise ExtractorError(message, expected=True)
86
87         title = doc.find('.//information/title').text
88         description = xpath_text(doc, './/information/detail', 'description')
89         duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration'))
90         uploader = xpath_text(doc, './/details/originChannelTitle', 'uploader')
91         uploader_id = xpath_text(doc, './/details/originChannelId', 'uploader id')
92         upload_date = unified_strdate(xpath_text(doc, './/details/airtime', 'upload date'))
93
94         def xml_to_thumbnails(fnode):
95             thumbnails = []
96             for node in fnode:
97                 thumbnail_url = node.text
98                 if not thumbnail_url:
99                     continue
100                 thumbnail = {
101                     'url': thumbnail_url,
102                 }
103                 if 'key' in node.attrib:
104                     m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
105                     if m:
106                         thumbnail['width'] = int(m.group(1))
107                         thumbnail['height'] = int(m.group(2))
108                 thumbnails.append(thumbnail)
109             return thumbnails
110
111         thumbnails = xml_to_thumbnails(doc.findall('.//teaserimages/teaserimage'))
112
113         format_nodes = doc.findall('.//formitaeten/formitaet')
114         quality = qualities(['veryhigh', 'high', 'med', 'low'])
115
116         def get_quality(elem):
117             return quality(xpath_text(elem, 'quality'))
118         format_nodes.sort(key=get_quality)
119         format_ids = []
120         formats = []
121         for fnode in format_nodes:
122             video_url = fnode.find('url').text
123             is_available = 'http://www.metafilegenerator' not in video_url
124             if not is_available:
125                 continue
126             format_id = fnode.attrib['basetype']
127             quality = xpath_text(fnode, './quality', 'quality')
128             format_m = re.match(r'''(?x)
129                 (?P<vcodec>[^_]+)_(?P<acodec>[^_]+)_(?P<container>[^_]+)_
130                 (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+)
131             ''', format_id)
132
133             ext = determine_ext(video_url, None) or format_m.group('container')
134             if ext not in ('smil', 'f4m', 'm3u8'):
135                 format_id = format_id + '-' + quality
136             if format_id in format_ids:
137                 continue
138
139             if ext == 'meta':
140                 continue
141             elif ext == 'smil':
142                 formats.extend(self._extract_smil_formats(
143                     video_url, video_id, fatal=False))
144             elif ext == 'm3u8':
145                 # the certificates are misconfigured (see
146                 # https://github.com/rg3/youtube-dl/issues/8665)
147                 if video_url.startswith('https://'):
148                     continue
149                 formats.extend(self._extract_m3u8_formats(
150                     video_url, video_id, 'mp4', m3u8_id=format_id, fatal=False))
151             elif ext == 'f4m':
152                 formats.extend(self._extract_f4m_formats(
153                     video_url, video_id, f4m_id=format_id, fatal=False))
154             else:
155                 proto = format_m.group('proto').lower()
156
157                 abr = int_or_none(xpath_text(fnode, './audioBitrate', 'abr'), 1000)
158                 vbr = int_or_none(xpath_text(fnode, './videoBitrate', 'vbr'), 1000)
159
160                 width = int_or_none(xpath_text(fnode, './width', 'width'))
161                 height = int_or_none(xpath_text(fnode, './height', 'height'))
162
163                 filesize = int_or_none(xpath_text(fnode, './filesize', 'filesize'))
164
165                 format_note = ''
166                 if not format_note:
167                     format_note = None
168
169                 formats.append({
170                     'format_id': format_id,
171                     'url': video_url,
172                     'ext': ext,
173                     'acodec': format_m.group('acodec'),
174                     'vcodec': format_m.group('vcodec'),
175                     'abr': abr,
176                     'vbr': vbr,
177                     'width': width,
178                     'height': height,
179                     'filesize': filesize,
180                     'format_note': format_note,
181                     'protocol': proto,
182                     '_available': is_available,
183                 })
184             format_ids.append(format_id)
185
186         self._sort_formats(formats)
187
188         return {
189             'id': video_id,
190             'title': title,
191             'description': description,
192             'duration': duration,
193             'thumbnails': thumbnails,
194             'uploader': uploader,
195             'uploader_id': uploader_id,
196             'upload_date': upload_date,
197             'formats': formats,
198         }
199
200     def _real_extract(self, url):
201         mobj = re.match(self._VALID_URL, url)
202         video_id = mobj.group('id')
203         details_url = 'http://www.3sat.de/mediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
204         return self.extract_from_xml_url(video_id, details_url)