2 from __future__ import unicode_literals
9 from .common import InfoExtractor
10 from ..compat import (
13 compat_urllib_request,
15 from .openload import PhantomJSwrapper
29 class PornHubBaseIE(InfoExtractor):
30 def _download_webpage_handle(self, *args, **kwargs):
31 def dl(*args, **kwargs):
32 return super(PornHubBaseIE, self)._download_webpage_handle(*args, **kwargs)
34 webpage, urlh = dl(*args, **kwargs)
36 if any(re.search(p, webpage) for p in (
37 r'<body\b[^>]+\bonload=["\']go\(\)',
38 r'document\.cookie\s*=\s*["\']RNKEY=',
39 r'document\.location\.reload\(true\)')):
40 url_or_request = args[0]
41 url = (url_or_request.get_full_url()
42 if isinstance(url_or_request, compat_urllib_request.Request)
44 phantom = PhantomJSwrapper(self, required_version='2.0')
45 phantom.get(url, html=webpage)
46 webpage, urlh = dl(*args, **kwargs)
51 class PornHubIE(PornHubBaseIE):
52 IE_DESC = 'PornHub and Thumbzilla'
56 (?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net))/(?:(?:view_video\.php|video/show)\?viewkey=|embed/)|
57 (?:www\.)?thumbzilla\.com/video/
62 'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
63 'md5': 'a6391306d050e4547f62b3f485dd9ba9',
67 'title': 'Seductive Indian beauty strips down and fingers her pink pussy',
69 'upload_date': '20130628',
70 'timestamp': 1372447216,
82 'url': 'http://www.pornhub.com/view_video.php?viewkey=1331683002',
87 'upload_date': '20150213',
88 'timestamp': 1423804862,
99 'skip_download': True,
103 'url': 'https://www.pornhub.com/view_video.php?viewkey=ph5af5fef7c2aa7',
105 'id': 'ph5af5fef7c2aa7',
107 'title': 'BFFS - Cute Teen Girls Share Cock On the Floor',
112 'dislike_count': int,
113 'comment_count': int,
124 'skip_download': True,
126 'skip': 'This video has been disabled',
128 'url': 'http://www.pornhub.com/view_video.php?viewkey=ph557bbb6676d2d',
129 'only_matching': True,
131 # removed at the request of cam4.com
132 'url': 'http://fr.pornhub.com/view_video.php?viewkey=ph55ca2f9760862',
133 'only_matching': True,
135 # removed at the request of the copyright owner
136 'url': 'http://www.pornhub.com/view_video.php?viewkey=788152859',
137 'only_matching': True,
139 # removed by uploader
140 'url': 'http://www.pornhub.com/view_video.php?viewkey=ph572716d15a111',
141 'only_matching': True,
144 'url': 'http://www.pornhub.com/view_video.php?viewkey=ph56fd731fce6b7',
145 'only_matching': True,
147 'url': 'https://www.thumbzilla.com/video/ph56c6114abd99a/horny-girlfriend-sex',
148 'only_matching': True,
150 'url': 'http://www.pornhub.com/video/show?viewkey=648719015',
151 'only_matching': True,
153 'url': 'https://www.pornhub.net/view_video.php?viewkey=203640933',
154 'only_matching': True,
156 'url': 'https://www.pornhubpremium.com/view_video.php?viewkey=ph5e4acdae54a82',
157 'only_matching': True,
161 def _extract_urls(webpage):
163 r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?pornhub\.(?:com|net)/embed/[\da-z]+)',
166 def _extract_count(self, pattern, webpage, name):
167 return str_to_int(self._search_regex(
168 pattern, webpage, '%s count' % name, fatal=False))
170 def _real_extract(self, url):
171 mobj = re.match(self._VALID_URL, url)
172 host = mobj.group('host') or 'pornhub.com'
173 video_id = mobj.group('id')
175 if 'premium' in host:
176 if not self._downloader.params.get('cookiefile'):
177 raise ExtractorError(
178 'PornHub Premium requires authentication.'
179 ' You may want to use --cookies.',
182 self._set_cookie(host, 'age_verified', '1')
184 def dl_webpage(platform):
185 self._set_cookie(host, 'platform', platform)
186 return self._download_webpage(
187 'https://www.%s/view_video.php?viewkey=%s' % (host, video_id),
188 video_id, 'Downloading %s webpage' % platform)
190 webpage = dl_webpage('pc')
192 error_msg = self._html_search_regex(
193 r'(?s)<div[^>]+class=(["\'])(?:(?!\1).)*\b(?:removed|userMessageSection)\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</div>',
194 webpage, 'error message', default=None, group='error')
196 error_msg = re.sub(r'\s+', ' ', error_msg)
197 raise ExtractorError(
198 'PornHub said: %s' % error_msg,
199 expected=True, video_id=video_id)
201 # video_title from flashvars contains whitespace instead of non-ASCII (see
202 # http://www.pornhub.com/view_video.php?viewkey=1331683002), not relying
204 title = self._html_search_meta(
205 'twitter:title', webpage, default=None) or self._html_search_regex(
206 (r'(?s)<h1[^>]+class=["\']title["\'][^>]*>(?P<title>.+?)</h1>',
207 r'<div[^>]+data-video-title=(["\'])(?P<title>(?:(?!\1).)+)\1',
208 r'shareTitle["\']\s*[=:]\s*(["\'])(?P<title>(?:(?!\1).)+)\1'),
209 webpage, 'title', group='title')
212 video_urls_set = set()
215 flashvars = self._parse_json(
217 r'var\s+flashvars_\d+\s*=\s*({.+?});', webpage, 'flashvars', default='{}'),
220 subtitle_url = url_or_none(flashvars.get('closedCaptionsFile'))
222 subtitles.setdefault('en', []).append({
226 thumbnail = flashvars.get('image_url')
227 duration = int_or_none(flashvars.get('video_duration'))
228 media_definitions = flashvars.get('mediaDefinitions')
229 if isinstance(media_definitions, list):
230 for definition in media_definitions:
231 if not isinstance(definition, dict):
233 video_url = definition.get('videoUrl')
234 if not video_url or not isinstance(video_url, compat_str):
236 if video_url in video_urls_set:
238 video_urls_set.add(video_url)
240 (video_url, int_or_none(definition.get('quality'))))
242 thumbnail, duration = [None] * 2
244 def extract_js_vars(webpage, pattern, default=NO_DEFAULT):
245 assignments = self._search_regex(
246 pattern, webpage, 'encoded url', default=default)
250 assignments = assignments.split(';')
254 def parse_js_value(inp):
255 inp = re.sub(r'/\*(?:(?!\*/).)*?\*/', '', inp)
257 inps = inp.split('+')
258 return functools.reduce(
259 operator.concat, map(parse_js_value, inps))
263 return remove_quotes(inp)
265 for assn in assignments:
269 assn = re.sub(r'var\s+', '', assn)
270 vname, value = assn.split('=', 1)
271 js_vars[vname] = parse_js_value(value)
274 def add_video_url(video_url):
275 v_url = url_or_none(video_url)
278 if v_url in video_urls_set:
280 video_urls.append((v_url, None))
281 video_urls_set.add(v_url)
284 FORMAT_PREFIXES = ('media', 'quality')
285 js_vars = extract_js_vars(
286 webpage, r'(var\s+(?:%s)_.+)' % '|'.join(FORMAT_PREFIXES),
289 for key, format_url in js_vars.items():
290 if any(key.startswith(p) for p in FORMAT_PREFIXES):
291 add_video_url(format_url)
292 if not video_urls and re.search(
293 r'<[^>]+\bid=["\']lockedPlayer', webpage):
294 raise ExtractorError(
295 'Video %s is locked' % video_id, expected=True)
298 js_vars = extract_js_vars(
299 dl_webpage('tv'), r'(var.+?mediastring.+?)</script>')
300 add_video_url(js_vars['mediastring'])
302 for mobj in re.finditer(
303 r'<a[^>]+\bclass=["\']downloadBtn\b[^>]+\bhref=(["\'])(?P<url>(?:(?!\1).)+)\1',
305 video_url = mobj.group('url')
306 if video_url not in video_urls_set:
307 video_urls.append((video_url, None))
308 video_urls_set.add(video_url)
312 for video_url, height in video_urls:
314 upload_date = self._search_regex(
315 r'/(\d{6}/\d{2})/', video_url, 'upload data', default=None)
317 upload_date = upload_date.replace('/', '')
318 ext = determine_ext(video_url)
320 formats.extend(self._extract_mpd_formats(
321 video_url, video_id, mpd_id='dash', fatal=False))
324 formats.extend(self._extract_m3u8_formats(
325 video_url, video_id, 'mp4', entry_protocol='m3u8_native',
326 m3u8_id='hls', fatal=False))
329 mobj = re.search(r'(?P<height>\d+)[pP]?_(?P<tbr>\d+)[kK]', video_url)
332 height = int(mobj.group('height'))
333 tbr = int(mobj.group('tbr'))
336 'format_id': '%dp' % height if height else None,
340 self._sort_formats(formats)
342 video_uploader = self._html_search_regex(
343 r'(?s)From: .+?<(?:a\b[^>]+\bhref=["\']/(?:(?:user|channel)s|model|pornstar)/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<',
344 webpage, 'uploader', default=None)
346 view_count = self._extract_count(
347 r'<span class="count">([\d,\.]+)</span> [Vv]iews', webpage, 'view')
348 like_count = self._extract_count(
349 r'<span class="votesUp">([\d,\.]+)</span>', webpage, 'like')
350 dislike_count = self._extract_count(
351 r'<span class="votesDown">([\d,\.]+)</span>', webpage, 'dislike')
352 comment_count = self._extract_count(
353 r'All Comments\s*<span>\(([\d,.]+)\)', webpage, 'comment')
355 def extract_list(meta_key):
356 div = self._search_regex(
357 r'(?s)<div[^>]+\bclass=["\'].*?\b%sWrapper[^>]*>(.+?)</div>'
358 % meta_key, webpage, meta_key, default=None)
360 return re.findall(r'<a[^>]+\bhref=[^>]+>([^<]+)', div)
362 info = self._search_json_ld(webpage, video_id, default={})
363 # description provided in JSON-LD is irrelevant
364 info['description'] = None
368 'uploader': video_uploader,
369 'upload_date': upload_date,
371 'thumbnail': thumbnail,
372 'duration': duration,
373 'view_count': view_count,
374 'like_count': like_count,
375 'dislike_count': dislike_count,
376 'comment_count': comment_count,
379 'tags': extract_list('tags'),
380 'categories': extract_list('categories'),
381 'subtitles': subtitles,
385 class PornHubPlaylistBaseIE(PornHubBaseIE):
386 def _extract_entries(self, webpage, host):
387 # Only process container div with main playlist content skipping
388 # drop-down menu that uses similar pattern for videos (see
389 # https://github.com/ytdl-org/youtube-dl/issues/11594).
390 container = self._search_regex(
391 r'(?s)(<div[^>]+class=["\']container.+)', webpage,
392 'container', default=webpage)
396 'http://www.%s/%s' % (host, video_url),
397 PornHubIE.ie_key(), video_title=title)
398 for video_url, title in orderedSet(re.findall(
399 r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"[^>]*\s+title="([^"]+)"',
403 def _real_extract(self, url):
404 mobj = re.match(self._VALID_URL, url)
405 host = mobj.group('host')
406 playlist_id = mobj.group('id')
408 webpage = self._download_webpage(url, playlist_id)
410 entries = self._extract_entries(webpage, host)
412 playlist = self._parse_json(
414 r'(?:playlistObject|PLAYLIST_VIEW)\s*=\s*({.+?});', webpage,
415 'playlist', default='{}'),
416 playlist_id, fatal=False)
417 title = playlist.get('title') or self._search_regex(
418 r'>Videos\s+in\s+(.+?)\s+[Pp]laylist<', webpage, 'title', fatal=False)
420 return self.playlist_result(
421 entries, playlist_id, title, playlist.get('description'))
424 class PornHubUserIE(PornHubPlaylistBaseIE):
425 _VALID_URL = r'(?P<url>https?://(?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net))/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/?#&]+))(?:[?#&]|/(?!videos)|$)'
427 'url': 'https://www.pornhub.com/model/zoe_ph',
428 'playlist_mincount': 118,
430 'url': 'https://www.pornhub.com/pornstar/liz-vicious',
434 'playlist_mincount': 118,
436 'url': 'https://www.pornhub.com/users/russianveet69',
437 'only_matching': True,
439 'url': 'https://www.pornhub.com/channels/povd',
440 'only_matching': True,
442 'url': 'https://www.pornhub.com/model/zoe_ph?abc=1',
443 'only_matching': True,
446 def _real_extract(self, url):
447 mobj = re.match(self._VALID_URL, url)
448 user_id = mobj.group('id')
449 return self.url_result(
450 '%s/videos' % mobj.group('url'), ie=PornHubPagedVideoListIE.ie_key(),
454 class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
456 def _has_more(webpage):
459 <li[^>]+\bclass=["\']page_next|
460 <link[^>]+\brel=["\']next|
461 <button[^>]+\bid=["\']moreDataBtn
462 ''', webpage) is not None
464 def _real_extract(self, url):
465 mobj = re.match(self._VALID_URL, url)
466 host = mobj.group('host')
467 item_id = mobj.group('id')
469 page = int_or_none(self._search_regex(
470 r'\bpage=(\d+)', url, 'page', default=None))
473 for page_num in (page, ) if page is not None else itertools.count(1):
475 webpage = self._download_webpage(
476 url, item_id, 'Downloading page %d' % page_num,
477 query={'page': page_num})
478 except ExtractorError as e:
479 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
482 page_entries = self._extract_entries(webpage, host)
485 entries.extend(page_entries)
486 if not self._has_more(webpage):
489 return self.playlist_result(orderedSet(entries), item_id)
492 class PornHubPagedVideoListIE(PornHubPagedPlaylistBaseIE):
493 _VALID_URL = r'https?://(?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net))/(?P<id>(?:[^/]+/)*[^/?#&]+)'
495 'url': 'https://www.pornhub.com/model/zoe_ph/videos',
496 'only_matching': True,
498 'url': 'http://www.pornhub.com/users/rushandlia/videos',
499 'only_matching': True,
501 'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos',
503 'id': 'pornstar/jenny-blighe/videos',
505 'playlist_mincount': 149,
507 'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos?page=3',
509 'id': 'pornstar/jenny-blighe/videos',
511 'playlist_mincount': 40,
513 # default sorting as Top Rated Videos
514 'url': 'https://www.pornhub.com/channels/povd/videos',
516 'id': 'channels/povd/videos',
518 'playlist_mincount': 293,
521 'url': 'https://www.pornhub.com/channels/povd/videos?o=ra',
522 'only_matching': True,
525 'url': 'https://www.pornhub.com/channels/povd/videos?o=da',
526 'only_matching': True,
529 'url': 'https://www.pornhub.com/channels/povd/videos?o=vi',
530 'only_matching': True,
532 'url': 'http://www.pornhub.com/users/zoe_ph/videos/public',
533 'only_matching': True,
536 'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos?o=mv',
537 'only_matching': True,
540 'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos?o=tr',
541 'only_matching': True,
544 'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos?o=lg',
545 'only_matching': True,
548 'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos?o=cm',
549 'only_matching': True,
551 'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos/paid',
552 'only_matching': True,
554 'url': 'https://www.pornhub.com/pornstar/liz-vicious/videos/fanonly',
555 'only_matching': True,
557 'url': 'https://www.pornhub.com/video',
558 'only_matching': True,
560 'url': 'https://www.pornhub.com/video?page=3',
561 'only_matching': True,
563 'url': 'https://www.pornhub.com/video/search?search=123',
564 'only_matching': True,
566 'url': 'https://www.pornhub.com/categories/teen',
567 'only_matching': True,
569 'url': 'https://www.pornhub.com/categories/teen?page=3',
570 'only_matching': True,
572 'url': 'https://www.pornhub.com/hd',
573 'only_matching': True,
575 'url': 'https://www.pornhub.com/hd?page=3',
576 'only_matching': True,
578 'url': 'https://www.pornhub.com/described-video',
579 'only_matching': True,
581 'url': 'https://www.pornhub.com/described-video?page=2',
582 'only_matching': True,
584 'url': 'https://www.pornhub.com/video/incategories/60fps-1/hd-porn',
585 'only_matching': True,
587 'url': 'https://www.pornhub.com/playlist/44121572',
589 'id': 'playlist/44121572',
591 'playlist_mincount': 132,
593 'url': 'https://www.pornhub.com/playlist/4667351',
594 'only_matching': True,
596 'url': 'https://de.pornhub.com/playlist/4667351',
597 'only_matching': True,
601 def suitable(cls, url):
603 if PornHubIE.suitable(url) or PornHubUserIE.suitable(url) or PornHubUserVideosUploadIE.suitable(url)
604 else super(PornHubPagedVideoListIE, cls).suitable(url))
607 class PornHubUserVideosUploadIE(PornHubPagedPlaylistBaseIE):
608 _VALID_URL = r'(?P<url>https?://(?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net))/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/]+)/videos/upload)'
610 'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos/upload',
612 'id': 'jenny-blighe',
614 'playlist_mincount': 129,
616 'url': 'https://www.pornhub.com/model/zoe_ph/videos/upload',
617 'only_matching': True,