X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Fmixcloud.py;h=483f6925fda989fc5111694c8c82f1807a1f3d97;hb=e6da9240d44774495a7ae0f2780bd42e4d2628f5;hp=5cf42198d5c44fc2fb0a2c23c49503404451fdc4;hpb=9c250931f5e1e68a835065c0cc5fa58e3f1e4734;p=youtube-dl diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py index 5cf42198d..483f6925f 100644 --- a/youtube_dl/extractor/mixcloud.py +++ b/youtube_dl/extractor/mixcloud.py @@ -1,19 +1,21 @@ from __future__ import unicode_literals +import base64 import functools +import itertools import re from .common import InfoExtractor from ..compat import ( + compat_chr, + compat_ord, compat_urllib_parse_unquote, compat_urlparse, ) from ..utils import ( clean_html, ExtractorError, - HEADRequest, OnDemandPagedList, - NO_DEFAULT, parse_count, str_to_int, ) @@ -45,22 +47,22 @@ class MixcloudIE(InfoExtractor): 'description': 'md5:2b8aec6adce69f9d41724647c65875e8', 'uploader': 'Gilles Peterson Worldwide', 'uploader_id': 'gillespeterson', - 'thumbnail': 're:https?://.*/images/', + 'thumbnail': 're:https?://.*', 'view_count': int, 'like_count': int, }, }] - def _check_url(self, url, track_id, ext): - try: - # We only want to know if the request succeed - # don't download the whole file - self._request_webpage( - HEADRequest(url), track_id, - 'Trying %s URL' % ext) - return True - except ExtractorError: - return False + # See https://www.mixcloud.com/media/js2/www_js_2.9e23256562c080482435196ca3975ab5.js + @staticmethod + def _decrypt_play_info(play_info): + KEY = 'pleasedontdownloadourmusictheartistswontgetpaid' + + play_info = base64.b64decode(play_info.encode('ascii')) + + return ''.join([ + compat_chr(compat_ord(ch) ^ compat_ord(KEY[idx % len(KEY)])) + for idx, ch in enumerate(play_info)]) def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) @@ -74,19 +76,15 @@ class MixcloudIE(InfoExtractor): r'(?s)]+class="global-message cloudcast-disabled-notice-light"[^>]*>(.+?)<(?:a|/div)', webpage, 'error message', default=None) - preview_url = self._search_regex( - r'\s(?:data-preview-url|m-preview)="([^"]+)"', - webpage, 'preview url', default=None if message else NO_DEFAULT) + encrypted_play_info = self._search_regex( + r'm-play-info="([^"]+)"', webpage, 'play info') + play_info = self._parse_json( + self._decrypt_play_info(encrypted_play_info), track_id) - if message: + if message and 'stream_url' not in play_info: raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True) - song_url = re.sub(r'audiocdn(\d+)', r'stream\1', preview_url) - song_url = song_url.replace('/previews/', '/c/originals/') - if not self._check_url(song_url, track_id, 'mp3'): - song_url = song_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/') - if not self._check_url(song_url, track_id, 'm4a'): - raise ExtractorError('Unable to extract track url') + song_url = play_info['stream_url'] PREFIX = ( r'm-play-on-spacebar[^>]+' @@ -126,18 +124,26 @@ class MixcloudIE(InfoExtractor): class MixcloudPlaylistBaseIE(InfoExtractor): _PAGE_SIZE = 24 - def _fetch_tracks_page(self, path, video_id, page_name, current_page): - resp = self._download_webpage( + def _find_urls_in_page(self, page): + for url in re.findall(r'm-play-button m-url="(?P[^"]+)"', page): + yield self.url_result( + compat_urlparse.urljoin('https://www.mixcloud.com', clean_html(url)), + MixcloudIE.ie_key()) + + def _fetch_tracks_page(self, path, video_id, page_name, current_page, real_page_number=None): + real_page_number = real_page_number or current_page + 1 + return self._download_webpage( 'https://www.mixcloud.com/%s/' % path, video_id, note='Download %s (page %d)' % (page_name, current_page + 1), errnote='Unable to download %s' % page_name, - query={'page': (current_page + 1), 'list': 'main', '_ajax': '1'}, + query={'page': real_page_number, 'list': 'main', '_ajax': '1'}, headers={'X-Requested-With': 'XMLHttpRequest'}) - for url in re.findall(r'm-play-button m-url="(?P[^"]+)"', resp): - yield self.url_result( - compat_urlparse.urljoin('https://www.mixcloud.com', clean_html(url)), - MixcloudIE.ie_key()) + def _tracks_page_func(self, page, video_id, page_name, current_page): + resp = self._fetch_tracks_page(page, video_id, page_name, current_page) + + for item in self._find_urls_in_page(resp): + yield item def _get_user_description(self, page_content): return self._html_search_regex( @@ -210,7 +216,7 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE): entries = OnDemandPagedList( functools.partial( - self._fetch_tracks_page, + self._tracks_page_func, '%s/%s' % (user_id, list_type), video_id, 'list of %s' % list_type), self._PAGE_SIZE, use_cache=True) @@ -258,8 +264,56 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE): entries = OnDemandPagedList( functools.partial( - self._fetch_tracks_page, + self._tracks_page_func, '%s/playlists/%s' % (user_id, playlist_id), video_id, 'tracklist'), self._PAGE_SIZE) return self.playlist_result(entries, video_id, playlist_title, description) + + +class MixcloudStreamIE(MixcloudPlaylistBaseIE): + _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P[^/]+)/stream/?$' + IE_NAME = 'mixcloud:stream' + + _TEST = { + 'url': 'https://www.mixcloud.com/FirstEar/stream/', + 'info_dict': { + 'id': 'FirstEar', + 'title': 'First Ear', + 'description': 'Curators of good music\nfirstearmusic.com', + }, + 'playlist_mincount': 192, + } + + def _real_extract(self, url): + user_id = self._match_id(url) + + webpage = self._download_webpage(url, user_id) + + entries = [] + prev_page_url = None + + def _handle_page(page): + entries.extend(self._find_urls_in_page(page)) + return self._search_regex( + r'm-next-page-url="([^"]+)"', page, + 'next page URL', default=None) + + next_page_url = _handle_page(webpage) + + for idx in itertools.count(0): + if not next_page_url or prev_page_url == next_page_url: + break + + prev_page_url = next_page_url + current_page = int(self._search_regex( + r'\?page=(\d+)', next_page_url, 'next page number')) + + next_page_url = _handle_page(self._fetch_tracks_page( + '%s/stream' % user_id, user_id, 'stream', idx, + real_page_number=current_page)) + + username = self._og_search_title(webpage) + description = self._get_user_description(webpage) + + return self.playlist_result(entries, user_id, username, description)