2 from __future__ import unicode_literals
9 from .common import InfoExtractor
10 from ..compat import (
13 compat_urllib_request,
15 from .openload import PhantomJSwrapper
27 class PornHubBaseIE(InfoExtractor):
28 def _download_webpage_handle(self, *args, **kwargs):
29 def dl(*args, **kwargs):
30 return super(PornHubBaseIE, self)._download_webpage_handle(*args, **kwargs)
32 webpage, urlh = dl(*args, **kwargs)
34 if any(re.search(p, webpage) for p in (
35 r'<body\b[^>]+\bonload=["\']go\(\)',
36 r'document\.cookie\s*=\s*["\']RNKEY=',
37 r'document\.location\.reload\(true\)')):
38 url_or_request = args[0]
39 url = (url_or_request.get_full_url()
40 if isinstance(url_or_request, compat_urllib_request.Request)
42 phantom = PhantomJSwrapper(self, required_version='2.0')
43 phantom.get(url, html=webpage)
44 webpage, urlh = dl(*args, **kwargs)
49 class PornHubIE(PornHubBaseIE):
50 IE_DESC = 'PornHub and Thumbzilla'
54 (?:[^/]+\.)?(?P<host>pornhub\.(?:com|net))/(?:(?:view_video\.php|video/show)\?viewkey=|embed/)|
55 (?:www\.)?thumbzilla\.com/video/
60 'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
61 'md5': '1e19b41231a02eba417839222ac9d58e',
65 'title': 'Seductive Indian beauty strips down and fingers her pink pussy',
67 'upload_date': '20130628',
79 'url': 'http://www.pornhub.com/view_video.php?viewkey=1331683002',
84 'uploader': 'Unknown',
85 'upload_date': '20150213',
96 'skip_download': True,
100 'url': 'https://www.pornhub.com/view_video.php?viewkey=ph5af5fef7c2aa7',
102 'id': 'ph5af5fef7c2aa7',
104 'title': 'BFFS - Cute Teen Girls Share Cock On the Floor',
109 'dislike_count': int,
110 'comment_count': int,
121 'skip_download': True,
124 'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
125 'only_matching': True,
127 # removed at the request of cam4.com
128 'url': 'http://fr.pornhub.com/view_video.php?viewkey=ph55ca2f9760862',
129 'only_matching': True,
131 # removed at the request of the copyright owner
132 'url': 'http://www.pornhub.com/view_video.php?viewkey=788152859',
133 'only_matching': True,
135 # removed by uploader
136 'url': 'http://www.pornhub.com/view_video.php?viewkey=ph572716d15a111',
137 'only_matching': True,
140 'url': 'http://www.pornhub.com/view_video.php?viewkey=ph56fd731fce6b7',
141 'only_matching': True,
143 'url': 'https://www.thumbzilla.com/video/ph56c6114abd99a/horny-girlfriend-sex',
144 'only_matching': True,
146 'url': 'http://www.pornhub.com/video/show?viewkey=648719015',
147 'only_matching': True,
149 'url': 'https://www.pornhub.net/view_video.php?viewkey=203640933',
150 'only_matching': True,
154 def _extract_urls(webpage):
156 r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?pornhub\.(?:com|net)/embed/[\da-z]+)',
159 def _extract_count(self, pattern, webpage, name):
160 return str_to_int(self._search_regex(
161 pattern, webpage, '%s count' % name, fatal=False))
163 def _real_extract(self, url):
164 mobj = re.match(self._VALID_URL, url)
165 host = mobj.group('host') or 'pornhub.com'
166 video_id = mobj.group('id')
168 self._set_cookie(host, 'age_verified', '1')
170 def dl_webpage(platform):
171 self._set_cookie(host, 'platform', platform)
172 return self._download_webpage(
173 'https://www.%s/view_video.php?viewkey=%s' % (host, video_id),
174 video_id, 'Downloading %s webpage' % platform)
176 webpage = dl_webpage('pc')
178 error_msg = self._html_search_regex(
179 r'(?s)<div[^>]+class=(["\'])(?:(?!\1).)*\b(?:removed|userMessageSection)\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</div>',
180 webpage, 'error message', default=None, group='error')
182 error_msg = re.sub(r'\s+', ' ', error_msg)
183 raise ExtractorError(
184 'PornHub said: %s' % error_msg,
185 expected=True, video_id=video_id)
187 # video_title from flashvars contains whitespace instead of non-ASCII (see
188 # http://www.pornhub.com/view_video.php?viewkey=1331683002), not relying
190 title = self._html_search_meta(
191 'twitter:title', webpage, default=None) or self._search_regex(
192 (r'<h1[^>]+class=["\']title["\'][^>]*>(?P<title>[^<]+)',
193 r'<div[^>]+data-video-title=(["\'])(?P<title>.+?)\1',
194 r'shareTitle\s*=\s*(["\'])(?P<title>.+?)\1'),
195 webpage, 'title', group='title')
198 video_urls_set = set()
201 flashvars = self._parse_json(
203 r'var\s+flashvars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'),
206 subtitle_url = url_or_none(flashvars.get('closedCaptionsFile'))
208 subtitles.setdefault('en', []).append({
212 thumbnail = flashvars.get('image_url')
213 duration = int_or_none(flashvars.get('video_duration'))
214 media_definitions = flashvars.get('mediaDefinitions')
215 if isinstance(media_definitions, list):
216 for definition in media_definitions:
217 if not isinstance(definition, dict):
219 video_url = definition.get('videoUrl')
220 if not video_url or not isinstance(video_url, compat_str):
222 if video_url in video_urls_set:
224 video_urls_set.add(video_url)
226 (video_url, int_or_none(definition.get('quality'))))
228 thumbnail, duration = [None] * 2
230 def extract_js_vars(webpage, pattern, fatal=True):
231 assignments = self._search_regex(
232 pattern, webpage, 'encoded url', fatal=fatal)
236 assignments = assignments.split(';')
240 def parse_js_value(inp):
241 inp = re.sub(r'/\*(?:(?!\*/).)*?\*/', '', inp)
243 inps = inp.split('+')
244 return functools.reduce(
245 operator.concat, map(parse_js_value, inps))
249 return remove_quotes(inp)
251 for assn in assignments:
255 assn = re.sub(r'var\s+', '', assn)
256 vname, value = assn.split('=', 1)
257 js_vars[vname] = parse_js_value(value)
260 def add_video_url(video_url):
261 v_url = url_or_none(video_url)
264 if v_url in video_urls_set:
266 video_urls.append((v_url, None))
267 video_urls_set.add(v_url)
270 FORMAT_PREFIXES = ('media', 'quality')
271 js_vars = extract_js_vars(
272 webpage, r'(var\s+(?:%s)_.+)' % '|'.join(FORMAT_PREFIXES),
275 for key, format_url in js_vars.items():
276 if any(key.startswith(p) for p in FORMAT_PREFIXES):
277 add_video_url(format_url)
280 js_vars = extract_js_vars(
281 dl_webpage('tv'), r'(var.+?mediastring.+?)</script>')
282 add_video_url(js_vars['mediastring'])
284 for mobj in re.finditer(
285 r'<a[^>]+\bclass=["\']downloadBtn\b[^>]+\bhref=(["\'])(?P<url>(?:(?!\1).)+)\1',
287 video_url = mobj.group('url')
288 if video_url not in video_urls_set:
289 video_urls.append((video_url, None))
290 video_urls_set.add(video_url)
294 for video_url, height in video_urls:
296 upload_date = self._search_regex(
297 r'/(\d{6}/\d{2})/', video_url, 'upload data', default=None)
299 upload_date = upload_date.replace('/', '')
300 ext = determine_ext(video_url)
302 formats.extend(self._extract_mpd_formats(
303 video_url, video_id, mpd_id='dash', fatal=False))
306 formats.extend(self._extract_m3u8_formats(
307 video_url, video_id, 'mp4', entry_protocol='m3u8_native',
308 m3u8_id='hls', fatal=False))
311 mobj = re.search(r'(?P<height>\d+)[pP]?_(?P<tbr>\d+)[kK]', video_url)
314 height = int(mobj.group('height'))
315 tbr = int(mobj.group('tbr'))
318 'format_id': '%dp' % height if height else None,
322 self._sort_formats(formats)
324 video_uploader = self._html_search_regex(
325 r'(?s)From: .+?<(?:a\b[^>]+\bhref=["\']/(?:(?:user|channel)s|model|pornstar)/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<',
326 webpage, 'uploader', fatal=False)
328 view_count = self._extract_count(
329 r'<span class="count">([\d,\.]+)</span> views', webpage, 'view')
330 like_count = self._extract_count(
331 r'<span class="votesUp">([\d,\.]+)</span>', webpage, 'like')
332 dislike_count = self._extract_count(
333 r'<span class="votesDown">([\d,\.]+)</span>', webpage, 'dislike')
334 comment_count = self._extract_count(
335 r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
337 def extract_list(meta_key):
338 div = self._search_regex(
339 r'(?s)<div[^>]+\bclass=["\'].*?\b%sWrapper[^>]*>(.+?)</div>'
340 % meta_key, webpage, meta_key, default=None)
342 return re.findall(r'<a[^>]+\bhref=[^>]+>([^<]+)', div)
346 'uploader': video_uploader,
347 'upload_date': upload_date,
349 'thumbnail': thumbnail,
350 'duration': duration,
351 'view_count': view_count,
352 'like_count': like_count,
353 'dislike_count': dislike_count,
354 'comment_count': comment_count,
357 'tags': extract_list('tags'),
358 'categories': extract_list('categories'),
359 'subtitles': subtitles,
363 class PornHubPlaylistBaseIE(PornHubBaseIE):
364 def _extract_entries(self, webpage, host):
365 # Only process container div with main playlist content skipping
366 # drop-down menu that uses similar pattern for videos (see
367 # https://github.com/ytdl-org/youtube-dl/issues/11594).
368 container = self._search_regex(
369 r'(?s)(<div[^>]+class=["\']container.+)', webpage,
370 'container', default=webpage)
374 'http://www.%s/%s' % (host, video_url),
375 PornHubIE.ie_key(), video_title=title)
376 for video_url, title in orderedSet(re.findall(
377 r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"[^>]*\s+title="([^"]+)"',
381 def _real_extract(self, url):
382 mobj = re.match(self._VALID_URL, url)
383 host = mobj.group('host')
384 playlist_id = mobj.group('id')
386 webpage = self._download_webpage(url, playlist_id)
388 entries = self._extract_entries(webpage, host)
390 playlist = self._parse_json(
392 r'(?:playlistObject|PLAYLIST_VIEW)\s*=\s*({.+?});', webpage,
393 'playlist', default='{}'),
394 playlist_id, fatal=False)
395 title = playlist.get('title') or self._search_regex(
396 r'>Videos\s+in\s+(.+?)\s+[Pp]laylist<', webpage, 'title', fatal=False)
398 return self.playlist_result(
399 entries, playlist_id, title, playlist.get('description'))
402 class PornHubUserIE(PornHubPlaylistBaseIE):
403 _VALID_URL = r'(?P<url>https?://(?:[^/]+\.)?pornhub\.(?:com|net)/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/?#&]+))(?:[?#&]|/(?!videos)|$)'
405 'url': 'https://www.pornhub.com/model/zoe_ph',
406 'playlist_mincount': 118,
408 'url': 'https://www.pornhub.com/pornstar/liz-vicious',
412 'playlist_mincount': 118,
414 'url': 'https://www.pornhub.com/users/russianveet69',
415 'only_matching': True,
417 'url': 'https://www.pornhub.com/channels/povd',
418 'only_matching': True,
420 'url': 'https://www.pornhub.com/model/zoe_ph?abc=1',
421 'only_matching': True,
424 def _real_extract(self, url):
425 mobj = re.match(self._VALID_URL, url)
426 user_id = mobj.group('id')
427 return self.url_result(
428 '%s/videos' % mobj.group('url'), ie=PornHubPagedVideoListIE.ie_key(),
432 class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
434 def _has_more(webpage):
437 <li[^>]+\bclass=["\']page_next|
438 <link[^>]+\brel=["\']next|
439 <button[^>]+\bid=["\']moreDataBtn
440 ''', webpage) is not None
442 def _real_extract(self, url):
443 mobj = re.match(self._VALID_URL, url)
444 host = mobj.group('host')
445 item_id = mobj.group('id')
447 page = int_or_none(self._search_regex(
448 r'\bpage=(\d+)', url, 'page', default=None))
451 for page_num in (page, ) if page is not None else itertools.count(1):
453 webpage = self._download_webpage(
454 url, item_id, 'Downloading page %d' % page_num,
455 query={'page': page_num})
456 except ExtractorError as e:
457 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
460 page_entries = self._extract_entries(webpage, host)
463 entries.extend(page_entries)
464 if not self._has_more(webpage):
467 return self.playlist_result(orderedSet(entries), item_id)
470 class PornHubPagedVideoListIE(PornHubPagedPlaylistBaseIE):
471 _VALID_URL = r'https?://(?:[^/]+\.)?(?P<host>pornhub\.(?:com|net))/(?P<id>(?:[^/]+/)*[^/?#&]+)'
473 'url': 'https://www.pornhub.com/model/zoe_ph/videos',
474 'only_matching': True,
476 'url': 'http://www.pornhub.com/users/rushandlia/videos',
477 'only_matching': True,
479 'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos',
481 'id': 'pornstar/jenny-blighe/videos',
483 'playlist_mincount': 149,
485 'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos?page=3',
487 'id': 'pornstar/jenny-blighe/videos',
489 'playlist_mincount': 40,
491 # default sorting as Top Rated Videos
492 'url': 'https://www.pornhub.com/channels/povd/videos',
494 'id': 'channels/povd/videos',
496 'playlist_mincount': 293,
499 'url': 'https://www.pornhub.com/channels/povd/videos?o=ra',
500 'only_matching': True,
503 'url': 'https://www.pornhub.com/channels/povd/videos?o=da',
504 'only_matching': True,
507 'url': 'https://www.pornhub.com/channels/povd/videos?o=vi',
508 'only_matching': True,
510 'url': 'http://www.pornhub.com/users/zoe_ph/videos/public',
511 'only_matching': True,
514 'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos?o=mv',
515 'only_matching': True,
518 'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos?o=tr',
519 'only_matching': True,
522 'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos?o=lg',
523 'only_matching': True,
526 'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos?o=cm',
527 'only_matching': True,
529 'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos/paid',
530 'only_matching': True,
532 'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos/fanonly',
533 'only_matching': True,
535 'url': 'https://www.pornhub.com/video',
536 'only_matching': True,
538 'url': 'https://www.pornhub.com/video?page=3',
539 'only_matching': True,
541 'url': 'https://www.pornhub.com/video/search?search=123',
542 'only_matching': True,
544 'url': 'https://www.pornhub.com/categories/teen',
545 'only_matching': True,
547 'url': 'https://www.pornhub.com/categories/teen?page=3',
548 'only_matching': True,
550 'url': 'https://www.pornhub.com/hd',
551 'only_matching': True,
553 'url': 'https://www.pornhub.com/hd?page=3',
554 'only_matching': True,
556 'url': 'https://www.pornhub.com/described-video',
557 'only_matching': True,
559 'url': 'https://www.pornhub.com/described-video?page=2',
560 'only_matching': True,
562 'url': 'https://www.pornhub.com/video/incategories/60fps-1/hd-porn',
563 'only_matching': True,
565 'url': 'https://www.pornhub.com/playlist/44121572',
567 'id': 'playlist/44121572',
569 'playlist_mincount': 132,
571 'url': 'https://www.pornhub.com/playlist/4667351',
572 'only_matching': True,
574 'url': 'https://de.pornhub.com/playlist/4667351',
575 'only_matching': True,
579 def suitable(cls, url):
581 if PornHubIE.suitable(url) or PornHubUserIE.suitable(url) or PornHubUserVideosUploadIE.suitable(url)
582 else super(PornHubPagedVideoListIE, cls).suitable(url))
585 class PornHubUserVideosUploadIE(PornHubPagedPlaylistBaseIE):
586 _VALID_URL = r'(?P<url>https?://(?:[^/]+\.)?(?P<host>pornhub\.(?:com|net))/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/]+)/videos/upload)'
588 'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos/upload',
590 'id': 'jenny-blighe',
592 'playlist_mincount': 129,
594 'url': 'https://www.pornhub.com/model/zoe_ph/videos/upload',
595 'only_matching': True,