]> git.bitcoin.ninja Git - youtube-dl/blob - youtube_dl/extractor/nrk.py
[pornhub:uservideos] Add support for multipage videos (Closes #9006)
[youtube-dl] / youtube_dl / extractor / nrk.py
1 # encoding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import (
8     compat_urlparse,
9     compat_urllib_parse_unquote,
10 )
11 from ..utils import (
12     determine_ext,
13     ExtractorError,
14     float_or_none,
15     parse_duration,
16     unified_strdate,
17 )
18
19
20 class NRKIE(InfoExtractor):
21     _VALID_URL = r'(?:nrk:|https?://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
22
23     _TESTS = [
24         {
25             'url': 'http://www.nrk.no/video/PS*150533',
26             'md5': 'bccd850baebefe23b56d708a113229c2',
27             'info_dict': {
28                 'id': '150533',
29                 'ext': 'flv',
30                 'title': 'Dompap og andre fugler i Piip-Show',
31                 'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
32                 'duration': 263,
33             }
34         },
35         {
36             'url': 'http://www.nrk.no/video/PS*154915',
37             'md5': '0b1493ba1aae7d9579a5ad5531bc395a',
38             'info_dict': {
39                 'id': '154915',
40                 'ext': 'flv',
41                 'title': 'Slik høres internett ut når du er blind',
42                 'description': 'md5:a621f5cc1bd75c8d5104cb048c6b8568',
43                 'duration': 20,
44             }
45         },
46     ]
47
48     def _real_extract(self, url):
49         video_id = self._match_id(url)
50
51         data = self._download_json(
52             'http://v8.psapi.nrk.no/mediaelement/%s' % video_id,
53             video_id, 'Downloading media JSON')
54
55         media_url = data.get('mediaUrl')
56
57         if not media_url:
58             if data['usageRights']['isGeoBlocked']:
59                 raise ExtractorError(
60                     'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
61                     expected=True)
62
63         if determine_ext(media_url) == 'f4m':
64             formats = self._extract_f4m_formats(
65                 media_url + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81', video_id, f4m_id='hds')
66             self._sort_formats(formats)
67         else:
68             formats = [{
69                 'url': media_url,
70                 'ext': 'flv',
71             }]
72
73         duration = parse_duration(data.get('duration'))
74
75         images = data.get('images')
76         if images:
77             thumbnails = images['webImages']
78             thumbnails.sort(key=lambda image: image['pixelWidth'])
79             thumbnail = thumbnails[-1]['imageUrl']
80         else:
81             thumbnail = None
82
83         return {
84             'id': video_id,
85             'title': data['title'],
86             'description': data['description'],
87             'duration': duration,
88             'thumbnail': thumbnail,
89             'formats': formats,
90         }
91
92
93 class NRKPlaylistIE(InfoExtractor):
94     _VALID_URL = r'https?://(?:www\.)?nrk\.no/(?!video|skole)(?:[^/]+/)+(?P<id>[^/]+)'
95
96     _TESTS = [{
97         'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
98         'info_dict': {
99             'id': 'gjenopplev-den-historiske-solformorkelsen-1.12270763',
100             'title': 'Gjenopplev den historiske solformørkelsen',
101             'description': 'md5:c2df8ea3bac5654a26fc2834a542feed',
102         },
103         'playlist_count': 2,
104     }, {
105         'url': 'http://www.nrk.no/kultur/bok/rivertonprisen-til-karin-fossum-1.12266449',
106         'info_dict': {
107             'id': 'rivertonprisen-til-karin-fossum-1.12266449',
108             'title': 'Rivertonprisen til Karin Fossum',
109             'description': 'Første kvinne på 15 år til å vinne krimlitteraturprisen.',
110         },
111         'playlist_count': 5,
112     }]
113
114     def _real_extract(self, url):
115         playlist_id = self._match_id(url)
116
117         webpage = self._download_webpage(url, playlist_id)
118
119         entries = [
120             self.url_result('nrk:%s' % video_id, 'NRK')
121             for video_id in re.findall(
122                 r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="([^"]+)"',
123                 webpage)
124         ]
125
126         playlist_title = self._og_search_title(webpage)
127         playlist_description = self._og_search_description(webpage)
128
129         return self.playlist_result(
130             entries, playlist_id, playlist_title, playlist_description)
131
132
133 class NRKSkoleIE(InfoExtractor):
134     IE_DESC = 'NRK Skole'
135     _VALID_URL = r'https?://(?:www\.)?nrk\.no/skole/klippdetalj?.*\btopic=(?P<id>[^/?#&]+)'
136
137     _TESTS = [{
138         'url': 'http://nrk.no/skole/klippdetalj?topic=nrk:klipp/616532',
139         'md5': '04cd85877cc1913bce73c5d28a47e00f',
140         'info_dict': {
141             'id': '6021',
142             'ext': 'flv',
143             'title': 'Genetikk og eneggede tvillinger',
144             'description': 'md5:3aca25dcf38ec30f0363428d2b265f8d',
145             'duration': 399,
146         },
147     }, {
148         'url': 'http://www.nrk.no/skole/klippdetalj?topic=nrk%3Aklipp%2F616532#embed',
149         'only_matching': True,
150     }, {
151         'url': 'http://www.nrk.no/skole/klippdetalj?topic=urn:x-mediadb:21379',
152         'only_matching': True,
153     }]
154
155     def _real_extract(self, url):
156         video_id = compat_urllib_parse_unquote(self._match_id(url))
157
158         webpage = self._download_webpage(url, video_id)
159
160         nrk_id = self._search_regex(r'data-nrk-id=["\'](\d+)', webpage, 'nrk id')
161         return self.url_result('nrk:%s' % nrk_id)
162
163
164 class NRKTVIE(InfoExtractor):
165     IE_DESC = 'NRK TV and NRK Radio'
166     _VALID_URL = r'(?P<baseurl>https?://(?:tv|radio)\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
167
168     _TESTS = [
169         {
170             'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
171             'info_dict': {
172                 'id': 'MUHH48000314',
173                 'ext': 'mp4',
174                 'title': '20 spørsmål',
175                 'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
176                 'upload_date': '20140523',
177                 'duration': 1741.52,
178             },
179             'params': {
180                 # m3u8 download
181                 'skip_download': True,
182             },
183         },
184         {
185             'url': 'https://tv.nrk.no/program/mdfp15000514',
186             'info_dict': {
187                 'id': 'mdfp15000514',
188                 'ext': 'mp4',
189                 'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting',
190                 'description': 'md5:654c12511f035aed1e42bdf5db3b206a',
191                 'upload_date': '20140524',
192                 'duration': 4605.08,
193             },
194             'params': {
195                 # m3u8 download
196                 'skip_download': True,
197             },
198         },
199         {
200             # single playlist video
201             'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
202             'md5': 'adbd1dbd813edaf532b0a253780719c2',
203             'info_dict': {
204                 'id': 'MSPO40010515-part2',
205                 'ext': 'flv',
206                 'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
207                 'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
208                 'upload_date': '20150106',
209             },
210             'skip': 'Only works from Norway',
211         },
212         {
213             'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
214             'playlist': [
215                 {
216                     'md5': '9480285eff92d64f06e02a5367970a7a',
217                     'info_dict': {
218                         'id': 'MSPO40010515-part1',
219                         'ext': 'flv',
220                         'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 1:2)',
221                         'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
222                         'upload_date': '20150106',
223                     },
224                 },
225                 {
226                     'md5': 'adbd1dbd813edaf532b0a253780719c2',
227                     'info_dict': {
228                         'id': 'MSPO40010515-part2',
229                         'ext': 'flv',
230                         'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
231                         'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
232                         'upload_date': '20150106',
233                     },
234                 },
235             ],
236             'info_dict': {
237                 'id': 'MSPO40010515',
238                 'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn',
239                 'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
240                 'upload_date': '20150106',
241                 'duration': 6947.5199999999995,
242             },
243             'skip': 'Only works from Norway',
244         },
245         {
246             'url': 'https://radio.nrk.no/serie/dagsnytt/NPUB21019315/12-07-2015#',
247             'only_matching': True,
248         }
249     ]
250
251     def _extract_f4m(self, manifest_url, video_id):
252         return self._extract_f4m_formats(
253             manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id, f4m_id='hds')
254
255     def _real_extract(self, url):
256         mobj = re.match(self._VALID_URL, url)
257         video_id = mobj.group('id')
258         part_id = mobj.group('part_id')
259         base_url = mobj.group('baseurl')
260
261         webpage = self._download_webpage(url, video_id)
262
263         title = self._html_search_meta(
264             'title', webpage, 'title')
265         description = self._html_search_meta(
266             'description', webpage, 'description')
267
268         thumbnail = self._html_search_regex(
269             r'data-posterimage="([^"]+)"',
270             webpage, 'thumbnail', fatal=False)
271         upload_date = unified_strdate(self._html_search_meta(
272             'rightsfrom', webpage, 'upload date', fatal=False))
273         duration = float_or_none(self._html_search_regex(
274             r'data-duration="([^"]+)"',
275             webpage, 'duration', fatal=False))
276
277         # playlist
278         parts = re.findall(
279             r'<a href="#del=(\d+)"[^>]+data-argument="([^"]+)">([^<]+)</a>', webpage)
280         if parts:
281             entries = []
282             for current_part_id, stream_url, part_title in parts:
283                 if part_id and current_part_id != part_id:
284                     continue
285                 video_part_id = '%s-part%s' % (video_id, current_part_id)
286                 formats = self._extract_f4m(stream_url, video_part_id)
287                 entries.append({
288                     'id': video_part_id,
289                     'title': part_title,
290                     'description': description,
291                     'thumbnail': thumbnail,
292                     'upload_date': upload_date,
293                     'formats': formats,
294                 })
295             if part_id:
296                 if entries:
297                     return entries[0]
298             else:
299                 playlist = self.playlist_result(entries, video_id, title, description)
300                 playlist.update({
301                     'thumbnail': thumbnail,
302                     'upload_date': upload_date,
303                     'duration': duration,
304                 })
305                 return playlist
306
307         formats = []
308
309         f4m_url = re.search(r'data-media="([^"]+)"', webpage)
310         if f4m_url:
311             formats.extend(self._extract_f4m(f4m_url.group(1), video_id))
312
313         m3u8_url = re.search(r'data-hls-media="([^"]+)"', webpage)
314         if m3u8_url:
315             formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4', m3u8_id='hls'))
316         self._sort_formats(formats)
317
318         subtitles_url = self._html_search_regex(
319             r'data-subtitlesurl\s*=\s*(["\'])(?P<url>.+?)\1',
320             webpage, 'subtitle URL', default=None, group='url')
321         subtitles = {}
322         if subtitles_url:
323             subtitles['no'] = [{
324                 'ext': 'ttml',
325                 'url': compat_urlparse.urljoin(base_url, subtitles_url),
326             }]
327
328         return {
329             'id': video_id,
330             'title': title,
331             'description': description,
332             'thumbnail': thumbnail,
333             'upload_date': upload_date,
334             'duration': duration,
335             'formats': formats,
336             'subtitles': subtitles,
337         }