]> git.bitcoin.ninja Git - youtube-dl/blob - youtube_dl/extractor/condenast.py
[pornhub] Improve locked videos detection (closes #22449, closes #22780)
[youtube-dl] / youtube_dl / extractor / condenast.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import (
8     compat_urllib_parse_urlparse,
9     compat_urlparse,
10 )
11 from ..utils import (
12     determine_ext,
13     extract_attributes,
14     int_or_none,
15     js_to_json,
16     mimetype2ext,
17     orderedSet,
18     parse_iso8601,
19 )
20
21
22 class CondeNastIE(InfoExtractor):
23     """
24     Condé Nast is a media group, some of its sites use a custom HTML5 player
25     that works the same in all of them.
26     """
27
28     # The keys are the supported sites and the values are the name to be shown
29     # to the user and in the extractor description.
30     _SITES = {
31         'allure': 'Allure',
32         'architecturaldigest': 'Architectural Digest',
33         'arstechnica': 'Ars Technica',
34         'bonappetit': 'Bon Appétit',
35         'brides': 'Brides',
36         'cnevids': 'Condé Nast',
37         'cntraveler': 'Condé Nast Traveler',
38         'details': 'Details',
39         'epicurious': 'Epicurious',
40         'glamour': 'Glamour',
41         'golfdigest': 'Golf Digest',
42         'gq': 'GQ',
43         'newyorker': 'The New Yorker',
44         'self': 'SELF',
45         'teenvogue': 'Teen Vogue',
46         'vanityfair': 'Vanity Fair',
47         'vogue': 'Vogue',
48         'wired': 'WIRED',
49         'wmagazine': 'W Magazine',
50     }
51
52     _VALID_URL = r'''(?x)https?://(?:video|www|player(?:-backend)?)\.(?:%s)\.com/
53         (?:
54             (?:
55                 embed(?:js)?|
56                 (?:script|inline)/video
57             )/(?P<id>[0-9a-f]{24})(?:/(?P<player_id>[0-9a-f]{24}))?(?:.+?\btarget=(?P<target>[^&]+))?|
58             (?P<type>watch|series|video)/(?P<display_id>[^/?#]+)
59         )''' % '|'.join(_SITES.keys())
60     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
61
62     EMBED_URL = r'(?:https?:)?//player(?:-backend)?\.(?:%s)\.com/(?:embed(?:js)?|(?:script|inline)/video)/.+?' % '|'.join(_SITES.keys())
63
64     _TESTS = [{
65         'url': 'http://video.wired.com/watch/3d-printed-speakers-lit-with-led',
66         'md5': '1921f713ed48aabd715691f774c451f7',
67         'info_dict': {
68             'id': '5171b343c2b4c00dd0c1ccb3',
69             'ext': 'mp4',
70             'title': '3D Printed Speakers Lit With LED',
71             'description': 'Check out these beautiful 3D printed LED speakers.  You can\'t actually buy them, but LumiGeek is working on a board that will let you make you\'re own.',
72             'uploader': 'wired',
73             'upload_date': '20130314',
74             'timestamp': 1363219200,
75         }
76     }, {
77         'url': 'http://video.gq.com/watch/the-closer-with-keith-olbermann-the-only-true-surprise-trump-s-an-idiot?c=series',
78         'info_dict': {
79             'id': '58d1865bfd2e6126e2000015',
80             'ext': 'mp4',
81             'title': 'The Only True Surprise? Trump’s an Idiot',
82             'uploader': 'gq',
83             'upload_date': '20170321',
84             'timestamp': 1490126427,
85         },
86     }, {
87         # JS embed
88         'url': 'http://player.cnevids.com/embedjs/55f9cf8b61646d1acf00000c/5511d76261646d5566020000.js',
89         'md5': 'f1a6f9cafb7083bab74a710f65d08999',
90         'info_dict': {
91             'id': '55f9cf8b61646d1acf00000c',
92             'ext': 'mp4',
93             'title': '3D printed TSA Travel Sentry keys really do open TSA locks',
94             'uploader': 'arstechnica',
95             'upload_date': '20150916',
96             'timestamp': 1442434955,
97         }
98     }, {
99         'url': 'https://player.cnevids.com/inline/video/59138decb57ac36b83000005.js?target=js-cne-player',
100         'only_matching': True,
101     }, {
102         'url': 'http://player-backend.cnevids.com/script/video/59138decb57ac36b83000005.js',
103         'only_matching': True,
104     }]
105
106     def _extract_series(self, url, webpage):
107         title = self._html_search_regex(
108             r'(?s)<div class="cne-series-info">.*?<h1>(.+?)</h1>',
109             webpage, 'series title')
110         url_object = compat_urllib_parse_urlparse(url)
111         base_url = '%s://%s' % (url_object.scheme, url_object.netloc)
112         m_paths = re.finditer(
113             r'(?s)<p class="cne-thumb-title">.*?<a href="(/watch/.+?)["\?]', webpage)
114         paths = orderedSet(m.group(1) for m in m_paths)
115         build_url = lambda path: compat_urlparse.urljoin(base_url, path)
116         entries = [self.url_result(build_url(path), 'CondeNast') for path in paths]
117         return self.playlist_result(entries, playlist_title=title)
118
119     def _extract_video_params(self, webpage, display_id):
120         query = self._parse_json(
121             self._search_regex(
122                 r'(?s)var\s+params\s*=\s*({.+?})[;,]', webpage, 'player params',
123                 default='{}'),
124             display_id, transform_source=js_to_json, fatal=False)
125         if query:
126             query['videoId'] = self._search_regex(
127                 r'(?:data-video-id=|currentVideoId\s*=\s*)["\']([\da-f]+)',
128                 webpage, 'video id', default=None)
129         else:
130             params = extract_attributes(self._search_regex(
131                 r'(<[^>]+data-js="video-player"[^>]+>)',
132                 webpage, 'player params element'))
133             query.update({
134                 'videoId': params['data-video'],
135                 'playerId': params['data-player'],
136                 'target': params['id'],
137             })
138         return query
139
140     def _extract_video(self, params):
141         video_id = params['videoId']
142
143         video_info = None
144
145         # New API path
146         query = params.copy()
147         query['embedType'] = 'inline'
148         info_page = self._download_json(
149             'http://player.cnevids.com/embed-api.json', video_id,
150             'Downloading embed info', fatal=False, query=query)
151
152         # Old fallbacks
153         if not info_page:
154             if params.get('playerId'):
155                 info_page = self._download_json(
156                     'http://player.cnevids.com/player/video.js', video_id,
157                     'Downloading video info', fatal=False, query=params)
158         if info_page:
159             video_info = info_page.get('video')
160         if not video_info:
161             info_page = self._download_webpage(
162                 'http://player.cnevids.com/player/loader.js',
163                 video_id, 'Downloading loader info', query=params)
164         if not video_info:
165             info_page = self._download_webpage(
166                 'https://player.cnevids.com/inline/video/%s.js' % video_id,
167                 video_id, 'Downloading inline info', query={
168                     'target': params.get('target', 'embedplayer')
169                 })
170
171         if not video_info:
172             video_info = self._parse_json(
173                 self._search_regex(
174                     r'(?s)var\s+config\s*=\s*({.+?});', info_page, 'config'),
175                 video_id, transform_source=js_to_json)['video']
176
177         title = video_info['title']
178
179         formats = []
180         for fdata in video_info['sources']:
181             src = fdata.get('src')
182             if not src:
183                 continue
184             ext = mimetype2ext(fdata.get('type')) or determine_ext(src)
185             if ext == 'm3u8':
186                 formats.extend(self._extract_m3u8_formats(
187                     src, video_id, 'mp4', entry_protocol='m3u8_native',
188                     m3u8_id='hls', fatal=False))
189                 continue
190             quality = fdata.get('quality')
191             formats.append({
192                 'format_id': ext + ('-%s' % quality if quality else ''),
193                 'url': src,
194                 'ext': ext,
195                 'quality': 1 if quality == 'high' else 0,
196             })
197         self._sort_formats(formats)
198
199         return {
200             'id': video_id,
201             'formats': formats,
202             'title': title,
203             'thumbnail': video_info.get('poster_frame'),
204             'uploader': video_info.get('brand'),
205             'duration': int_or_none(video_info.get('duration')),
206             'tags': video_info.get('tags'),
207             'series': video_info.get('series_title'),
208             'season': video_info.get('season_title'),
209             'timestamp': parse_iso8601(video_info.get('premiere_date')),
210             'categories': video_info.get('categories'),
211         }
212
213     def _real_extract(self, url):
214         video_id, player_id, target, url_type, display_id = re.match(self._VALID_URL, url).groups()
215
216         if video_id:
217             return self._extract_video({
218                 'videoId': video_id,
219                 'playerId': player_id,
220                 'target': target,
221             })
222
223         webpage = self._download_webpage(url, display_id)
224
225         if url_type == 'series':
226             return self._extract_series(url, webpage)
227         else:
228             params = self._extract_video_params(webpage, display_id)
229             info = self._search_json_ld(
230                 webpage, display_id, fatal=False)
231             info.update(self._extract_video(params))
232             return info