X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Fpatreon.py;h=9eb0276795356af12571e98982f35edec490d0ba;hb=68217024e83c8e7965f2800e9ff7a9575f049b5c;hp=4ddff42e65985ee682ab1a2c91ff5abf46ffd18a;hpb=6994e70651180cfe7c5d097fee517de78429c18d;p=youtube-dl diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py index 4ddff42e6..9eb027679 100644 --- a/youtube_dl/extractor/patreon.py +++ b/youtube_dl/extractor/patreon.py @@ -1,81 +1,13 @@ -# encoding: utf-8 +# coding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor -from ..utils import ( - ExtractorError, - compat_html_parser, - #compat_urllib_request, - #compat_urllib_parse, -) - - -class PatreonHTMLParser(compat_html_parser.HTMLParser): - _PREFIX = 'http://www.patreon.com' - _ATTACH_TAGS = 5 * ['div'] - _ATTACH_CLASSES = [ - 'fancyboxhidden', 'box photo', 'boxwrapper', - 'hiddendisplay shareinfo', 'attach' - ] - _INFO_TAGS = 4 * ['div'] - _INFO_CLASSES = [ - 'fancyboxhidden', 'box photo', 'boxwrapper', - 'hiddendisplay shareinfo' - ] - - def _match(self, attrs_classes, desired): - if attrs_classes == desired: - return True - elif len(attrs_classes) == len(desired): - return all( - x.startswith(y) - for x, y in zip(attrs_classes, desired) - ) - return False - - def get_creation_info(self, html_data): - self.tag_stack = [] - self.attrs_stack = [] - self.creation_info = {} - self.feed(html_data) - - def handle_starttag(self, tag, attrs): - self.tag_stack.append(tag.lower()) - self.attrs_stack.append(dict(attrs)) - - def handle_endtag(self, tag): - self.tag_stack.pop() - self.attrs_stack.pop() - - def handle_data(self, data): - # Check first if this is a creation attachment - if self.tag_stack[-6:-1] == self._ATTACH_TAGS: - attrs_classes = [ - x.get('class', '').lower() for x in self.attrs_stack[-6:-1] - ] - if self._match(attrs_classes, self._ATTACH_CLASSES): - if self.tag_stack[-1] == 'a': - url = self._PREFIX + self.attrs_stack[-1].get('href') - self.creation_info['url'] = url - if '.' in data: - self.creation_info['ext'] = data.rsplit('.')[-1] - # Next, check if this is within the div containing the creation info - if self.tag_stack[-5:-1] == self._INFO_TAGS: - attrs_classes = [ - x.get('class', '').lower() for x in self.attrs_stack[-5:-1] - ] - if self._match(attrs_classes, self._INFO_CLASSES): - if self.attrs_stack[-1].get('class') == 'utitle': - self.creation_info['title'] = data.strip() +from ..utils import js_to_json class PatreonIE(InfoExtractor): - IE_NAME = 'patreon' - _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(.+)' + _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(?P[^&#]+)' _TESTS = [ - # CSS names with "double" in the name, i.e. "boxwrapper double" { 'url': 'http://www.patreon.com/creation?hid=743933', 'md5': 'e25505eec1053a6e6813b8ed369875cc', @@ -84,6 +16,7 @@ class PatreonIE(InfoExtractor): 'ext': 'mp3', 'title': 'Episode 166: David Smalley of Dogma Debate', 'uploader': 'Cognitive Dissonance Podcast', + 'thumbnail': 're:^https?://.*$', }, }, { @@ -94,15 +27,33 @@ class PatreonIE(InfoExtractor): 'ext': 'mp3', 'title': 'CD 167 Extra', 'uploader': 'Cognitive Dissonance Podcast', + 'thumbnail': 're:^https?://.*$', }, }, + { + 'url': 'https://www.patreon.com/creation?hid=1682498', + 'info_dict': { + 'id': 'SU4fj_aEMVw', + 'ext': 'mp4', + 'title': 'I\'m on Patreon!', + 'uploader': 'TraciJHines', + 'thumbnail': 're:^https?://.*$', + 'upload_date': '20150211', + 'description': 'md5:c5a706b1f687817a3de09db1eb93acd4', + 'uploader_id': 'TraciJHines', + }, + 'params': { + 'noplaylist': True, + 'skip_download': True, + } + } ] # Currently Patreon exposes download URL via hidden CSS, so login is not # needed. Keeping this commented for when this inevitably changes. ''' def _login(self): - (username, password) = self._get_login_info() + username, password = self._get_login_info() if username is None: return @@ -112,11 +63,11 @@ class PatreonIE(InfoExtractor): 'password': password, } - request = compat_urllib_request.Request( + request = sanitized_Request( 'https://www.patreon.com/processLogin', - compat_urllib_parse.urlencode(login_form).encode('utf-8') + compat_urllib_parse_urlencode(login_form).encode('utf-8') ) - login_page = self._download_webpage(request, None, note='Logging in as %s' % username) + login_page = self._download_webpage(request, None, note='Logging in') if re.search(r'onLoginFailed', login_page): raise ExtractorError('Unable to login, incorrect username and/or password', expected=True) @@ -126,22 +77,39 @@ class PatreonIE(InfoExtractor): ''' def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group(1) - - info_page = self._download_webpage(url, video_id) - - ret = {'id': video_id} - try: - ret['uploader'] = re.search( - r'(.+) is creating', info_page - ).group(1) - except AttributeError: - pass - - parser = PatreonHTMLParser() - parser.get_creation_info(info_page) - if not parser.creation_info.get('url'): - raise ExtractorError('Unable to retrieve creation URL') - ret.update(parser.creation_info) - return ret + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + title = self._og_search_title(webpage).strip() + + attach_fn = self._html_search_regex( + r'
', + webpage, 'attachment URL', default=None) + embed = self._html_search_regex( + r']+id="watchCreation"[^>]*>\s*]+src="([^"]+)"', + webpage, 'embedded URL', default=None) + + if attach_fn is not None: + video_url = 'http://www.patreon.com' + attach_fn + thumbnail = self._og_search_thumbnail(webpage) + uploader = self._html_search_regex( + r'(.*?) is creating', webpage, 'uploader') + elif embed is not None: + return self.url_result(embed) + else: + playlist = self._parse_json(self._search_regex( + r'(?s)new\s+jPlayerPlaylist\(\s*\{\s*[^}]*},\s*(\[.*?,?\s*\])', + webpage, 'playlist JSON'), + video_id, transform_source=js_to_json) + data = playlist[0] + video_url = self._proto_relative_url(data['mp3']) + thumbnail = self._proto_relative_url(data.get('cover')) + uploader = data.get('artist') + + return { + 'id': video_id, + 'url': video_url, + 'ext': 'mp3', + 'title': title, + 'uploader': uploader, + 'thumbnail': thumbnail, + }