X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Fpatreon.py;h=761a4b1de3d194686339d4d8eb15a5f1bb2b50a6;hb=a6211d237b4e7051ca018cc09440502561fedaa7;hp=4efb18d6c831d79d8cd1ba9a80a1a9bd1740703d;hpb=c3f0b12b0f1db168b6d30df68e33a9dba077728e;p=youtube-dl diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py index 4efb18d6c..761a4b1de 100644 --- a/youtube_dl/extractor/patreon.py +++ b/youtube_dl/extractor/patreon.py @@ -1,88 +1,75 @@ -# encoding: utf-8 +# coding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor from ..utils import ( - ExtractorError, - compat_html_parser, - #compat_urllib_request, - #compat_urllib_parse, + clean_html, + determine_ext, + int_or_none, + KNOWN_EXTENSIONS, + mimetype2ext, + parse_iso8601, + str_or_none, + try_get, ) -class PatreonHTMLParser(compat_html_parser.HTMLParser): - _PREFIX = 'http://www.patreon.com' - _ATTACH_TAGS = 5 * ['div'] - _ATTACH_CLASSES = [ - 'fancyboxhidden', 'box photo double', 'boxwrapper double', - 'hiddendisplay shareinfo', 'attach' - ] - _INFO_TAGS = 4 * ['div'] - _INFO_CLASSES = [ - 'fancyboxhidden', 'box photo double', 'boxwrapper double', - 'hiddendisplay shareinfo' - ] - - def get_creation_info(self, html_data): - self.tag_stack = [] - self.attrs_stack = [] - self.creation_info = {} - self.feed(html_data) - - def handle_starttag(self, tag, attrs): - self.tag_stack.append(tag.lower()) - self.attrs_stack.append(dict(attrs)) - - def handle_endtag(self, tag): - self.tag_stack.pop() - self.attrs_stack.pop() - - def handle_data(self, data): - # Check first if this is a creation attachment - if self.tag_stack[-6:-1] == self._ATTACH_TAGS: - attrs_classes = [ - x.get('class', '').lower() for x in self.attrs_stack[-6:-1] - ] - if attrs_classes == self._ATTACH_CLASSES: - if self.tag_stack[-1] == 'a': - url = self._PREFIX + self.attrs_stack[-1].get('href') - self.creation_info['url'] = url - if '.' in data: - self.creation_info['ext'] = data.rsplit('.')[-1] - # Next, check if this is within the div containing the creation info - if self.tag_stack[-5:-1] == self._INFO_TAGS: - attrs_classes = [ - x.get('class', '').lower() for x in self.attrs_stack[-5:-1] - ] - if attrs_classes == self._INFO_CLASSES: - if self.attrs_stack[-1].get('class') == 'utitle': - self.creation_info['title'] = data.strip() - - class PatreonIE(InfoExtractor): - IE_NAME = 'patreon' - _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(.+)' - _TESTS = [ - { - 'url': 'http://www.patreon.com/creation?hid=743933', - 'md5': 'e25505eec1053a6e6813b8ed369875cc', - 'name': 'Patreon', - 'info_dict': { - 'id': '743933', - 'ext': 'mp3', - 'title': 'Episode 166: David Smalley of Dogma Debate', - 'uploader': 'Cognitive Dissonance Podcast', - }, + _VALID_URL = r'https?://(?:www\.)?patreon\.com/(?:creation\?hid=|posts/(?:[\w-]+-)?)(?P\d+)' + _TESTS = [{ + 'url': 'http://www.patreon.com/creation?hid=743933', + 'md5': 'e25505eec1053a6e6813b8ed369875cc', + 'info_dict': { + 'id': '743933', + 'ext': 'mp3', + 'title': 'Episode 166: David Smalley of Dogma Debate', + 'description': 'md5:713b08b772cd6271b9f3906683cfacdf', + 'uploader': 'Cognitive Dissonance Podcast', + 'thumbnail': 're:^https?://.*$', + 'timestamp': 1406473987, + 'upload_date': '20140727', + 'uploader_id': '87145', }, - ] + }, { + 'url': 'http://www.patreon.com/creation?hid=754133', + 'md5': '3eb09345bf44bf60451b8b0b81759d0a', + 'info_dict': { + 'id': '754133', + 'ext': 'mp3', + 'title': 'CD 167 Extra', + 'uploader': 'Cognitive Dissonance Podcast', + 'thumbnail': 're:^https?://.*$', + }, + 'skip': 'Patron-only content', + }, { + 'url': 'https://www.patreon.com/creation?hid=1682498', + 'info_dict': { + 'id': 'SU4fj_aEMVw', + 'ext': 'mp4', + 'title': 'I\'m on Patreon!', + 'uploader': 'TraciJHines', + 'thumbnail': 're:^https?://.*$', + 'upload_date': '20150211', + 'description': 'md5:c5a706b1f687817a3de09db1eb93acd4', + 'uploader_id': 'TraciJHines', + }, + 'params': { + 'noplaylist': True, + 'skip_download': True, + } + }, { + 'url': 'https://www.patreon.com/posts/episode-166-of-743933', + 'only_matching': True, + }, { + 'url': 'https://www.patreon.com/posts/743933', + 'only_matching': True, + }] # Currently Patreon exposes download URL via hidden CSS, so login is not # needed. Keeping this commented for when this inevitably changes. ''' def _login(self): - (username, password) = self._get_login_info() + username, password = self._get_login_info() if username is None: return @@ -92,11 +79,11 @@ class PatreonIE(InfoExtractor): 'password': password, } - request = compat_urllib_request.Request( + request = sanitized_Request( 'https://www.patreon.com/processLogin', - compat_urllib_parse.urlencode(login_form).encode('utf-8') + compat_urllib_parse_urlencode(login_form).encode('utf-8') ) - login_page = self._download_webpage(request, None, note='Logging in as %s' % username) + login_page = self._download_webpage(request, None, note='Logging in') if re.search(r'onLoginFailed', login_page): raise ExtractorError('Unable to login, incorrect username and/or password', expected=True) @@ -106,22 +93,64 @@ class PatreonIE(InfoExtractor): ''' def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group(1) - - info_page = self._download_webpage(url, video_id) - - ret = {'id': video_id} - try: - ret['uploader'] = re.search( - r'(.+) is creating', info_page - ).group(1) - except AttributeError: - pass + video_id = self._match_id(url) + post = self._download_json( + 'https://www.patreon.com/api/posts/' + video_id, video_id, query={ + 'fields[media]': 'download_url,mimetype,size_bytes', + 'fields[post]': 'comment_count,content,embed,image,like_count,post_file,published_at,title', + 'fields[user]': 'full_name,url', + 'json-api-use-default-includes': 'false', + 'include': 'media,user', + }) + attributes = post['data']['attributes'] + title = attributes['title'].strip() + image = attributes.get('image') or {} + info = { + 'id': video_id, + 'title': title, + 'description': clean_html(attributes.get('content')), + 'thumbnail': image.get('large_url') or image.get('url'), + 'timestamp': parse_iso8601(attributes.get('published_at')), + 'like_count': int_or_none(attributes.get('like_count')), + 'comment_count': int_or_none(attributes.get('comment_count')), + } - parser = PatreonHTMLParser() - parser.get_creation_info(info_page) - if not parser.creation_info.get('url'): - raise ExtractorError('Unable to retrieve creation URL') - ret.update(parser.creation_info) - return ret + for i in post.get('included', []): + i_type = i.get('type') + if i_type == 'media': + media_attributes = i.get('attributes') or {} + download_url = media_attributes.get('download_url') + ext = mimetype2ext(media_attributes.get('mimetype')) + if download_url and ext in KNOWN_EXTENSIONS: + info.update({ + 'ext': ext, + 'filesize': int_or_none(media_attributes.get('size_bytes')), + 'url': download_url, + }) + elif i_type == 'user': + user_attributes = i.get('attributes') + if user_attributes: + info.update({ + 'uploader': user_attributes.get('full_name'), + 'uploader_id': str_or_none(i.get('id')), + 'uploader_url': user_attributes.get('url'), + }) + + if not info.get('url'): + embed_url = try_get(attributes, lambda x: x['embed']['url']) + if embed_url: + info.update({ + '_type': 'url', + 'url': embed_url, + }) + + if not info.get('url'): + post_file = attributes['post_file'] + ext = determine_ext(post_file.get('name')) + if ext in KNOWN_EXTENSIONS: + info.update({ + 'ext': ext, + 'url': post_file['url'], + }) + + return info