X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Fmixcloud.py;h=9759560f1bee5642c7e4f11c8aa8891fd4b3ed7d;hb=HEAD;hp=30b33e7e98581fb0142d30837276dc476f6acd49;hpb=b3a9474ad13001bda6a6cc7fb9b00af1bafabcb5;p=youtube-dl diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py index 30b33e7e9..9759560f1 100644 --- a/youtube_dl/extractor/mixcloud.py +++ b/youtube_dl/extractor/mixcloud.py @@ -1,311 +1,351 @@ from __future__ import unicode_literals +import itertools import re from .common import InfoExtractor from ..compat import ( + compat_b64decode, + compat_chr, + compat_ord, + compat_str, compat_urllib_parse_unquote, - compat_urllib_request + compat_zip ) from ..utils import ( - ExtractorError, - HEADRequest, - NO_DEFAULT, - parse_count, - str_to_int, - clean_html + int_or_none, + parse_iso8601, + strip_or_none, + try_get, ) -class MixcloudIE(InfoExtractor): - _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)' +class MixcloudBaseIE(InfoExtractor): + def _call_api(self, object_type, object_fields, display_id, username, slug=None): + lookup_key = object_type + 'Lookup' + return self._download_json( + 'https://www.mixcloud.com/graphql', display_id, query={ + 'query': '''{ + %s(lookup: {username: "%s"%s}) { + %s + } +}''' % (lookup_key, username, ', slug: "%s"' % slug if slug else '', object_fields) + })['data'][lookup_key] + + +class MixcloudIE(MixcloudBaseIE): + _VALID_URL = r'https?://(?:(?:www|beta|m)\.)?mixcloud\.com/([^/]+)/(?!stream|uploads|favorites|listens|playlists)([^/]+)' IE_NAME = 'mixcloud' _TESTS = [{ 'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/', 'info_dict': { - 'id': 'dholbach-cryptkeeper', + 'id': 'dholbach_cryptkeeper', 'ext': 'm4a', 'title': 'Cryptkeeper', 'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.', 'uploader': 'Daniel Holbach', 'uploader_id': 'dholbach', - 'thumbnail': 're:https?://.*\.jpg', + 'thumbnail': r're:https?://.*\.jpg', 'view_count': int, - 'like_count': int, + 'timestamp': 1321359578, + 'upload_date': '20111115', }, }, { 'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/', 'info_dict': { - 'id': 'gillespeterson-caribou-7-inch-vinyl-mix-chat', + 'id': 'gillespeterson_caribou-7-inch-vinyl-mix-chat', 'ext': 'mp3', 'title': 'Caribou 7 inch Vinyl Mix & Chat', 'description': 'md5:2b8aec6adce69f9d41724647c65875e8', 'uploader': 'Gilles Peterson Worldwide', 'uploader_id': 'gillespeterson', - 'thumbnail': 're:https?://.*/images/', + 'thumbnail': 're:https?://.*', 'view_count': int, - 'like_count': int, + 'timestamp': 1422987057, + 'upload_date': '20150203', }, + }, { + 'url': 'https://beta.mixcloud.com/RedLightRadio/nosedrip-15-red-light-radio-01-18-2016/', + 'only_matching': True, }] + _DECRYPTION_KEY = 'IFYOUWANTTHEARTISTSTOGETPAIDDONOTDOWNLOADFROMMIXCLOUD' - def _check_url(self, url, track_id, ext): - try: - # We only want to know if the request succeed - # don't download the whole file - self._request_webpage( - HEADRequest(url), track_id, - 'Trying %s URL' % ext) - return True - except ExtractorError: - return False + @staticmethod + def _decrypt_xor_cipher(key, ciphertext): + """Encrypt/Decrypt XOR cipher. Both ways are possible because it's XOR.""" + return ''.join([ + compat_chr(compat_ord(ch) ^ compat_ord(k)) + for ch, k in compat_zip(ciphertext, itertools.cycle(key))]) def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - uploader = mobj.group(1) - cloudcast_name = mobj.group(2) - track_id = compat_urllib_parse_unquote('-'.join((uploader, cloudcast_name))) - - webpage = self._download_webpage(url, track_id) - - message = self._html_search_regex( - r'(?s)]+class="global-message cloudcast-disabled-notice-light"[^>]*>(.+?)<(?:a|/div)', - webpage, 'error message', default=None) - - preview_url = self._search_regex( - r'\s(?:data-preview-url|m-preview)="([^"]+)"', - webpage, 'preview url', default=None if message else NO_DEFAULT) - - if message: - raise ExtractorError('%s said: %s' % (self.IE_NAME, message), expected=True) - - song_url = re.sub(r'audiocdn(\d+)', r'stream\1', preview_url) - song_url = song_url.replace('/previews/', '/c/originals/') - if not self._check_url(song_url, track_id, 'mp3'): - song_url = song_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/') - if not self._check_url(song_url, track_id, 'm4a'): - raise ExtractorError('Unable to extract track url') - - PREFIX = ( - r'm-play-on-spacebar[^>]+' - r'(?:\s+[a-zA-Z0-9-]+(?:="[^"]+")?)*?\s+') - title = self._html_search_regex( - PREFIX + r'm-title="([^"]+)"', webpage, 'title') - thumbnail = self._proto_relative_url(self._html_search_regex( - PREFIX + r'm-thumbnail-url="([^"]+)"', webpage, 'thumbnail', - fatal=False)) - uploader = self._html_search_regex( - PREFIX + r'm-owner-name="([^"]+)"', - webpage, 'uploader', fatal=False) - uploader_id = self._search_regex( - r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False) - description = self._og_search_description(webpage) - like_count = parse_count(self._search_regex( - r'\bbutton-favorite[^>]+>.*?]+class=["\']toggle-number[^>]+>\s*([^<]+)', - webpage, 'like count', fatal=False)) - view_count = str_to_int(self._search_regex( - [r'([0-9,.]+)'], - webpage, 'play count', fatal=False)) + username, slug = re.match(self._VALID_URL, url).groups() + username, slug = compat_urllib_parse_unquote(username), compat_urllib_parse_unquote(slug) + track_id = '%s_%s' % (username, slug) + + cloudcast = self._call_api('cloudcast', '''audioLength + comments(first: 100) { + edges { + node { + comment + created + user { + displayName + username + } + } + } + totalCount + } + description + favorites { + totalCount + } + featuringArtistList + isExclusive + name + owner { + displayName + url + username + } + picture(width: 1024, height: 1024) { + url + } + plays + publishDate + reposts { + totalCount + } + streamInfo { + dashUrl + hlsUrl + url + } + tags { + tag { + name + } + }''', track_id, username, slug) + + title = cloudcast['name'] + + stream_info = cloudcast['streamInfo'] + formats = [] + + for url_key in ('url', 'hlsUrl', 'dashUrl'): + format_url = stream_info.get(url_key) + if not format_url: + continue + decrypted = self._decrypt_xor_cipher( + self._DECRYPTION_KEY, compat_b64decode(format_url)) + if url_key == 'hlsUrl': + formats.extend(self._extract_m3u8_formats( + decrypted, track_id, 'mp4', entry_protocol='m3u8_native', + m3u8_id='hls', fatal=False)) + elif url_key == 'dashUrl': + formats.extend(self._extract_mpd_formats( + decrypted, track_id, mpd_id='dash', fatal=False)) + else: + formats.append({ + 'format_id': 'http', + 'url': decrypted, + 'downloader_options': { + # Mixcloud starts throttling at >~5M + 'http_chunk_size': 5242880, + }, + }) + + if not formats and cloudcast.get('isExclusive'): + self.raise_login_required() + + self._sort_formats(formats) + + comments = [] + for edge in (try_get(cloudcast, lambda x: x['comments']['edges']) or []): + node = edge.get('node') or {} + text = strip_or_none(node.get('comment')) + if not text: + continue + user = node.get('user') or {} + comments.append({ + 'author': user.get('displayName'), + 'author_id': user.get('username'), + 'text': text, + 'timestamp': parse_iso8601(node.get('created')), + }) + + tags = [] + for t in cloudcast.get('tags'): + tag = try_get(t, lambda x: x['tag']['name'], compat_str) + if not tag: + tags.append(tag) + + get_count = lambda x: int_or_none(try_get(cloudcast, lambda y: y[x]['totalCount'])) + + owner = cloudcast.get('owner') or {} return { 'id': track_id, 'title': title, - 'url': song_url, - 'description': description, - 'thumbnail': thumbnail, - 'uploader': uploader, - 'uploader_id': uploader_id, - 'view_count': view_count, - 'like_count': like_count, + 'formats': formats, + 'description': cloudcast.get('description'), + 'thumbnail': try_get(cloudcast, lambda x: x['picture']['url'], compat_str), + 'uploader': owner.get('displayName'), + 'timestamp': parse_iso8601(cloudcast.get('publishDate')), + 'uploader_id': owner.get('username'), + 'uploader_url': owner.get('url'), + 'duration': int_or_none(cloudcast.get('audioLength')), + 'view_count': int_or_none(cloudcast.get('plays')), + 'like_count': get_count('favorites'), + 'repost_count': get_count('reposts'), + 'comment_count': get_count('comments'), + 'comments': comments, + 'tags': tags, + 'artist': ', '.join(cloudcast.get('featuringArtistList') or []) or None, } -class MixcloudUserIE(InfoExtractor): - """ - Information extractor for Mixcloud users. - It can retrieve a list of a user's uploads, favorites or listens. - """ +class MixcloudPlaylistBaseIE(MixcloudBaseIE): + def _get_cloudcast(self, node): + return node - _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P[^/]+)/(?Puploads|favorites|listens)?/?$' + def _get_playlist_title(self, title, slug): + return title + + def _real_extract(self, url): + username, slug = re.match(self._VALID_URL, url).groups() + username = compat_urllib_parse_unquote(username) + if not slug: + slug = 'uploads' + else: + slug = compat_urllib_parse_unquote(slug) + playlist_id = '%s_%s' % (username, slug) + + is_playlist_type = self._ROOT_TYPE == 'playlist' + playlist_type = 'items' if is_playlist_type else slug + list_filter = '' + + has_next_page = True + entries = [] + while has_next_page: + playlist = self._call_api( + self._ROOT_TYPE, '''%s + %s + %s(first: 100%s) { + edges { + node { + %s + } + } + pageInfo { + endCursor + hasNextPage + } + }''' % (self._TITLE_KEY, self._DESCRIPTION_KEY, playlist_type, list_filter, self._NODE_TEMPLATE), + playlist_id, username, slug if is_playlist_type else None) + + items = playlist.get(playlist_type) or {} + for edge in items.get('edges', []): + cloudcast = self._get_cloudcast(edge.get('node') or {}) + cloudcast_url = cloudcast.get('url') + if not cloudcast_url: + continue + entries.append(self.url_result( + cloudcast_url, MixcloudIE.ie_key(), cloudcast.get('slug'))) + + page_info = items['pageInfo'] + has_next_page = page_info['hasNextPage'] + list_filter = ', after: "%s"' % page_info['endCursor'] + + return self.playlist_result( + entries, playlist_id, + self._get_playlist_title(playlist[self._TITLE_KEY], slug), + playlist.get(self._DESCRIPTION_KEY)) + + +class MixcloudUserIE(MixcloudPlaylistBaseIE): + _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P[^/]+)/(?Puploads|favorites|listens|stream)?/?$' IE_NAME = 'mixcloud:user' _TESTS = [{ 'url': 'http://www.mixcloud.com/dholbach/', 'info_dict': { - 'id': 'dholbach/uploads', + 'id': 'dholbach_uploads', 'title': 'Daniel Holbach (uploads)', - 'description': 'md5:327af72d1efeb404a8216c27240d1370', + 'description': 'md5:b60d776f0bab534c5dabe0a34e47a789', }, - 'playlist_mincount': 11 + 'playlist_mincount': 36, }, { 'url': 'http://www.mixcloud.com/dholbach/uploads/', 'info_dict': { - 'id': 'dholbach/uploads', + 'id': 'dholbach_uploads', 'title': 'Daniel Holbach (uploads)', - 'description': 'md5:327af72d1efeb404a8216c27240d1370', + 'description': 'md5:b60d776f0bab534c5dabe0a34e47a789', }, - 'playlist_mincount': 11 + 'playlist_mincount': 36, }, { 'url': 'http://www.mixcloud.com/dholbach/favorites/', 'info_dict': { - 'id': 'dholbach/favorites', + 'id': 'dholbach_favorites', 'title': 'Daniel Holbach (favorites)', - 'description': 'md5:327af72d1efeb404a8216c27240d1370', + 'description': 'md5:b60d776f0bab534c5dabe0a34e47a789', }, - 'playlist_mincount': 244 + # 'params': { + # 'playlist_items': '1-100', + # }, + 'playlist_mincount': 396, }, { 'url': 'http://www.mixcloud.com/dholbach/listens/', 'info_dict': { - 'id': 'dholbach/listens', + 'id': 'dholbach_listens', 'title': 'Daniel Holbach (listens)', - 'description': 'md5:327af72d1efeb404a8216c27240d1370', + 'description': 'md5:b60d776f0bab534c5dabe0a34e47a789', + }, + # 'params': { + # 'playlist_items': '1-100', + # }, + 'playlist_mincount': 1623, + 'skip': 'Large list', + }, { + 'url': 'https://www.mixcloud.com/FirstEar/stream/', + 'info_dict': { + 'id': 'FirstEar_stream', + 'title': 'First Ear (stream)', + 'description': 'Curators of good music\r\n\r\nfirstearmusic.com', }, - 'playlist_mincount': 846 + 'playlist_mincount': 271, }] - def _fetch_tracks(self, base_url, video_id, dl_note=None, dl_errnote=None): - # retrieve all fragments of a list of tracks with fake AJAX calls - track_urls = [] - current_page = 1 - while True: - # fake a AJAX request to retrieve a list fragment - page_url = base_url + "?page=%d&list=main&_ajax=1" % current_page - req = compat_urllib_request.Request(page_url, headers={"X-Requested-With": "XMLHttpRequest"}) - resp = self._download_webpage(req, video_id, note=dl_note + " (page %d)" % current_page, errnote=dl_errnote) - - # extract all track URLs from fragment - urls = re.findall(r'm-play-button m-url="(?P[^"]+)"', resp) - # clean up URLs - urls = map(clean_html, urls) - # create absolute URLs - urls = map(lambda u: "https://www.mixcloud.com" + u, urls) - track_urls.extend(urls) - - # advance to next fragment, if any - if " m-next-page-url=" in resp: - current_page += 1 - else: - break - - return track_urls - - def _handle_track_urls(self, urls): - return map(lambda u: self.url_result(u, "Mixcloud"), urls) - - def _get_user_description(self, page_content): - return self._html_search_regex( - r'
.*?

(.*?)

', - page_content, - "user description", - fatal=False) - - def _get_username(self, page_content): - return self._og_search_title(page_content) - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - user_id = mobj.group("user") - list_type = mobj.group("type") - - # if only a profile URL was supplied, default to download all uploads - if list_type is None: - list_type = "uploads" - - video_id = "%s/%s" % (user_id, list_type) - - # download the user's profile to retrieve some metadata - profile = self._download_webpage("https://www.mixcloud.com/%s/" % user_id, - video_id, - note="Downloading user profile", - errnote="Unable to download user profile") - - username = self._get_username(profile) - description = self._get_user_description(profile) - - # retrieve all page fragments of uploads, favorites or listens - track_urls = self._fetch_tracks( - "https://www.mixcloud.com/%s/%s/" % (user_id, list_type), - video_id, - dl_note="Downloading list of %s" % list_type, - dl_errnote="Unable to download list of %s" % list_type) - - # let MixcloudIE handle each track URL - entries = self._handle_track_urls(track_urls) - - return { - '_type': 'playlist', - 'entries': entries, - 'title': "%s (%s)" % (username, list_type), - 'id': video_id, - "description": description - } + _TITLE_KEY = 'displayName' + _DESCRIPTION_KEY = 'biog' + _ROOT_TYPE = 'user' + _NODE_TEMPLATE = '''slug + url''' + def _get_playlist_title(self, title, slug): + return '%s (%s)' % (title, slug) -class MixcloudPlaylistIE(MixcloudUserIE): - """ - Information extractor for Mixcloud playlists. - """ - _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/(?P[^/]+)/playlists/(?P[^/]+)/?$' +class MixcloudPlaylistIE(MixcloudPlaylistBaseIE): + _VALID_URL = r'https?://(?:www\.)?mixcloud\.com/(?P[^/]+)/playlists/(?P[^/]+)/?$' IE_NAME = 'mixcloud:playlist' _TESTS = [{ - 'url': 'https://www.mixcloud.com/RedBullThre3style/playlists/tokyo-finalists-2015/', - 'info_dict': { - 'id': 'RedBullThre3style/playlists/tokyo-finalists-2015', - 'title': 'National Champions 2015', - 'description': 'md5:6ff5fb01ac76a31abc9b3939c16243a3', - }, - 'playlist_mincount': 16 - }, { 'url': 'https://www.mixcloud.com/maxvibes/playlists/jazzcat-on-ness-radio/', 'info_dict': { - 'id': 'maxvibes/playlists/jazzcat-on-ness-radio', - 'title': 'Jazzcat on Ness Radio', - 'description': 'md5:7bbbf0d6359a0b8cda85224be0f8f263', + 'id': 'maxvibes_jazzcat-on-ness-radio', + 'title': 'Ness Radio sessions', }, - 'playlist_mincount': 23 + 'playlist_mincount': 59, }] - - def _get_playlist_title(self, page_content): - return self._html_search_regex( - r'(?P.*?)</span>', - page_content, - "playlist title", - group="title", - fatal=True - ) - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - user_id = mobj.group("user") - playlist_id = mobj.group("playlist") - video_id = "%s/playlists/%s" % (user_id, playlist_id) - - # download the playlist page to retrieve some metadata - profile = self._download_webpage(url, - user_id, - note="Downloading playlist page", - errnote="Unable to download playlist page") - - description = self._get_user_description(profile) - playlist_title = self._get_playlist_title(profile) - - # retrieve all page fragments of playlist - track_urls = self._fetch_tracks( - "https://www.mixcloud.com/%s/playlists/%s/" % (user_id, playlist_id), - video_id, - dl_note="Downloading tracklist of %s" % playlist_title, - dl_errnote="Unable to tracklist of %s" % playlist_title) - - # let MixcloudIE handle each track - entries = self._handle_track_urls(track_urls) - - return { - '_type': 'playlist', - 'entries': entries, - 'title': playlist_title, - 'id': video_id, - "description": description - } + _TITLE_KEY = 'name' + _DESCRIPTION_KEY = 'description' + _ROOT_TYPE = 'playlist' + _NODE_TEMPLATE = '''cloudcast { + slug + url + }''' + + def _get_cloudcast(self, node): + return node.get('cloudcast') or {}