_ Git - youtube-dl/blob - youtube_dl/extractor/soundcloud.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import itertools
   5 import re
   6
   7 from .common import (
   8     InfoExtractor,
   9     SearchInfoExtractor
  10 )
  11 from ..compat import (
  12     compat_str,
  13     compat_urlparse,
  14 )
  15 from ..utils import (
  16     ExtractorError,
  17     float_or_none,
  18     HEADRequest,
  19     int_or_none,
  20     KNOWN_EXTENSIONS,
  21     mimetype2ext,
  22     str_or_none,
  23     try_get,
  24     unified_timestamp,
  25     update_url_query,
  26     url_or_none,
  27 )
  28
  29
  30 class SoundcloudEmbedIE(InfoExtractor):
  31     _VALID_URL = r'https?://(?:w|player|p)\.soundcloud\.com/player/?.*?url=(?P<id>.*)'
  32
  33     @staticmethod
  34     def _extract_urls(webpage):
  35         return [m.group('url') for m in re.finditer(
  36             r'<iframe[^>]+src=(["\'])(?P<url>(?:https?://)?(?:w\.)?soundcloud\.com/player.+?)\1',
  37             webpage)]
  38
  39     def _real_extract(self, url):
  40         return self.url_result(compat_urlparse.parse_qs(
  41             compat_urlparse.urlparse(url).query)['url'][0])
  42
  43
  44 class SoundcloudIE(InfoExtractor):
  45     """Information extractor for soundcloud.com
  46        To access the media, the uid of the song and a stream token
  47        must be extracted from the page source and the script must make
  48        a request to media.soundcloud.com/crossdomain.xml. Then
  49        the media can be grabbed by requesting from an url composed
  50        of the stream token and uid
  51      """
  52
  53     _VALID_URL = r'''(?x)^(?:https?://)?
  54                     (?:(?:(?:www\.|m\.)?soundcloud\.com/
  55                             (?!stations/track)
  56                             (?P<uploader>[\w\d-]+)/
  57                             (?!(?:tracks|albums|sets(?:/.+?)?|reposts|likes|spotlight)/?(?:$|[?#]))
  58                             (?P<title>[\w\d-]+)/?
  59                             (?P<token>[^?]+?)?(?:[?].*)?$)
  60                        |(?:api(?:-v2)?\.soundcloud\.com/tracks/(?P<track_id>\d+)
  61                           (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
  62                     )
  63                     '''
  64     IE_NAME = 'soundcloud'
  65     _TESTS = [
  66         {
  67             'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
  68             'md5': 'ebef0a451b909710ed1d7787dddbf0d7',
  69             'info_dict': {
  70                 'id': '62986583',
  71                 'ext': 'mp3',
  72                 'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
  73                 'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d',
  74                 'uploader': 'E.T. ExTerrestrial Music',
  75                 'uploader_id': '1571244',
  76                 'timestamp': 1349920598,
  77                 'upload_date': '20121011',
  78                 'duration': 143.216,
  79                 'license': 'all-rights-reserved',
  80                 'view_count': int,
  81                 'like_count': int,
  82                 'comment_count': int,
  83                 'repost_count': int,
  84             }
  85         },
  86         # not streamable song
  87         {
  88             'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
  89             'info_dict': {
  90                 'id': '47127627',
  91                 'ext': 'mp3',
  92                 'title': 'Goldrushed',
  93                 'description': 'From Stockholm Sweden\r\nPovel / Magnus / Filip / David\r\nwww.theroyalconcept.com',
  94                 'uploader': 'The Royal Concept',
  95                 'uploader_id': '9615865',
  96                 'timestamp': 1337635207,
  97                 'upload_date': '20120521',
  98                 'duration': 30,
  99                 'license': 'all-rights-reserved',
 100                 'view_count': int,
 101                 'like_count': int,
 102                 'comment_count': int,
 103                 'repost_count': int,
 104             },
 105             'params': {
 106                 # rtmp
 107                 'skip_download': True,
 108             },
 109             'skip': 'Preview',
 110         },
 111         # private link
 112         {
 113             'url': 'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp',
 114             'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
 115             'info_dict': {
 116                 'id': '123998367',
 117                 'ext': 'mp3',
 118                 'title': 'Youtube - Dl Test Video \'\' Ä↭',
 119                 'description': 'test chars:  \"\'/\\ä↭',
 120                 'uploader': 'jaimeMF',
 121                 'uploader_id': '69767071',
 122                 'timestamp': 1386604920,
 123                 'upload_date': '20131209',
 124                 'duration': 9.927,
 125                 'license': 'all-rights-reserved',
 126                 'view_count': int,
 127                 'like_count': int,
 128                 'comment_count': int,
 129                 'repost_count': int,
 130             },
 131         },
 132         # private link (alt format)
 133         {
 134             'url': 'https://api.soundcloud.com/tracks/123998367?secret_token=s-8Pjrp',
 135             'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
 136             'info_dict': {
 137                 'id': '123998367',
 138                 'ext': 'mp3',
 139                 'title': 'Youtube - Dl Test Video \'\' Ä↭',
 140                 'description': 'test chars:  \"\'/\\ä↭',
 141                 'uploader': 'jaimeMF',
 142                 'uploader_id': '69767071',
 143                 'timestamp': 1386604920,
 144                 'upload_date': '20131209',
 145                 'duration': 9.927,
 146                 'license': 'all-rights-reserved',
 147                 'view_count': int,
 148                 'like_count': int,
 149                 'comment_count': int,
 150                 'repost_count': int,
 151             },
 152         },
 153         # downloadable song
 154         {
 155             'url': 'https://soundcloud.com/oddsamples/bus-brakes',
 156             'md5': '7624f2351f8a3b2e7cd51522496e7631',
 157             'info_dict': {
 158                 'id': '128590877',
 159                 'ext': 'mp3',
 160                 'title': 'Bus Brakes',
 161                 'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66',
 162                 'uploader': 'oddsamples',
 163                 'uploader_id': '73680509',
 164                 'timestamp': 1389232924,
 165                 'upload_date': '20140109',
 166                 'duration': 17.346,
 167                 'license': 'cc-by-sa',
 168                 'view_count': int,
 169                 'like_count': int,
 170                 'comment_count': int,
 171                 'repost_count': int,
 172             },
 173         },
 174         # private link, downloadable format
 175         {
 176             'url': 'https://soundcloud.com/oriuplift/uponly-238-no-talking-wav/s-AyZUd',
 177             'md5': '64a60b16e617d41d0bef032b7f55441e',
 178             'info_dict': {
 179                 'id': '340344461',
 180                 'ext': 'wav',
 181                 'title': 'Uplifting Only 238 [No Talking] (incl. Alex Feed Guestmix) (Aug 31, 2017) [wav]',
 182                 'description': 'md5:fa20ee0fca76a3d6df8c7e57f3715366',
 183                 'uploader': 'Ori Uplift Music',
 184                 'uploader_id': '12563093',
 185                 'timestamp': 1504206263,
 186                 'upload_date': '20170831',
 187                 'duration': 7449.096,
 188                 'license': 'all-rights-reserved',
 189                 'view_count': int,
 190                 'like_count': int,
 191                 'comment_count': int,
 192                 'repost_count': int,
 193             },
 194         },
 195         # no album art, use avatar pic for thumbnail
 196         {
 197             'url': 'https://soundcloud.com/garyvee/sideways-prod-mad-real',
 198             'md5': '59c7872bc44e5d99b7211891664760c2',
 199             'info_dict': {
 200                 'id': '309699954',
 201                 'ext': 'mp3',
 202                 'title': 'Sideways (Prod. Mad Real)',
 203                 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
 204                 'uploader': 'garyvee',
 205                 'uploader_id': '2366352',
 206                 'timestamp': 1488152409,
 207                 'upload_date': '20170226',
 208                 'duration': 207.012,
 209                 'thumbnail': r're:https?://.*\.jpg',
 210                 'license': 'all-rights-reserved',
 211                 'view_count': int,
 212                 'like_count': int,
 213                 'comment_count': int,
 214                 'repost_count': int,
 215             },
 216             'params': {
 217                 'skip_download': True,
 218             },
 219         },
 220         # not available via api.soundcloud.com/i1/tracks/id/streams
 221         {
 222             'url': 'https://soundcloud.com/giovannisarani/mezzo-valzer',
 223             'md5': 'e22aecd2bc88e0e4e432d7dcc0a1abf7',
 224             'info_dict': {
 225                 'id': '583011102',
 226                 'ext': 'mp3',
 227                 'title': 'Mezzo Valzer',
 228                 'description': 'md5:4138d582f81866a530317bae316e8b61',
 229                 'uploader': 'Giovanni Sarani',
 230                 'uploader_id': '3352531',
 231                 'timestamp': 1551394171,
 232                 'upload_date': '20190228',
 233                 'duration': 180.157,
 234                 'thumbnail': r're:https?://.*\.jpg',
 235                 'license': 'all-rights-reserved',
 236                 'view_count': int,
 237                 'like_count': int,
 238                 'comment_count': int,
 239                 'repost_count': int,
 240             },
 241             'expected_warnings': ['Unable to download JSON metadata'],
 242         }
 243     ]
 244
 245     _API_BASE = 'https://api.soundcloud.com/'
 246     _API_V2_BASE = 'https://api-v2.soundcloud.com/'
 247     _BASE_URL = 'https://soundcloud.com/'
 248     _CLIENT_ID = 'BeGVhOrGmfboy1LtiHTQF6Ejpt9ULJCI'
 249     _IMAGE_REPL_RE = r'-([0-9a-z]+)\.jpg'
 250
 251     _ARTWORK_MAP = {
 252         'mini': 16,
 253         'tiny': 20,
 254         'small': 32,
 255         'badge': 47,
 256         't67x67': 67,
 257         'large': 100,
 258         't300x300': 300,
 259         'crop': 400,
 260         't500x500': 500,
 261         'original': 0,
 262     }
 263
 264     @classmethod
 265     def _resolv_url(cls, url):
 266         return SoundcloudIE._API_V2_BASE + 'resolve?url=' + url + '&client_id=' + cls._CLIENT_ID
 267
 268     def _extract_info_dict(self, info, full_title=None, secret_token=None, version=2):
 269         track_id = compat_str(info['id'])
 270         title = info['title']
 271         track_base_url = self._API_BASE + 'tracks/%s' % track_id
 272
 273         format_urls = set()
 274         formats = []
 275         query = {'client_id': self._CLIENT_ID}
 276         if secret_token:
 277             query['secret_token'] = secret_token
 278
 279         if info.get('downloadable'):
 280             format_url = update_url_query(
 281                 info.get('download_url') or track_base_url + '/download', query)
 282             format_urls.add(format_url)
 283             if version == 2:
 284                 v1_info = self._download_json(
 285                     track_base_url, track_id, query=query, fatal=False) or {}
 286             else:
 287                 v1_info = info
 288             formats.append({
 289                 'format_id': 'download',
 290                 'ext': v1_info.get('original_format') or 'mp3',
 291                 'filesize': int_or_none(v1_info.get('original_content_size')),
 292                 'url': format_url,
 293                 'preference': 10,
 294             })
 295
 296         def invalid_url(url):
 297             return not url or url in format_urls or re.search(r'/(?:preview|playlist)/0/30/', url)
 298
 299         def add_format(f, protocol):
 300             mobj = re.search(r'\.(?P<abr>\d+)\.(?P<ext>[0-9a-z]{3,4})(?=[/?])', stream_url)
 301             if mobj:
 302                 for k, v in mobj.groupdict().items():
 303                     if not f.get(k):
 304                         f[k] = v
 305             format_id_list = []
 306             if protocol:
 307                 format_id_list.append(protocol)
 308             for k in ('ext', 'abr'):
 309                 v = f.get(k)
 310                 if v:
 311                     format_id_list.append(v)
 312             abr = f.get('abr')
 313             if abr:
 314                 f['abr'] = int(abr)
 315             f.update({
 316                 'format_id': '_'.join(format_id_list),
 317                 'protocol': 'm3u8_native' if protocol == 'hls' else 'http',
 318             })
 319             formats.append(f)
 320
 321         # New API
 322         transcodings = try_get(
 323             info, lambda x: x['media']['transcodings'], list) or []
 324         for t in transcodings:
 325             if not isinstance(t, dict):
 326                 continue
 327             format_url = url_or_none(t.get('url'))
 328             if not format_url or t.get('snipped') or '/preview/' in format_url:
 329                 continue
 330             stream = self._download_json(
 331                 format_url, track_id, query=query, fatal=False)
 332             if not isinstance(stream, dict):
 333                 continue
 334             stream_url = url_or_none(stream.get('url'))
 335             if invalid_url(stream_url):
 336                 continue
 337             format_urls.add(stream_url)
 338             stream_format = t.get('format') or {}
 339             protocol = stream_format.get('protocol')
 340             if protocol != 'hls' and '/hls' in format_url:
 341                 protocol = 'hls'
 342             ext = None
 343             preset = str_or_none(t.get('preset'))
 344             if preset:
 345                 ext = preset.split('_')[0]
 346             if ext not in KNOWN_EXTENSIONS:
 347                 ext = mimetype2ext(stream_format.get('mime_type'))
 348             add_format({
 349                 'url': stream_url,
 350                 'ext': ext,
 351             }, 'http' if protocol == 'progressive' else protocol)
 352
 353         if not formats:
 354             # Old API, does not work for some tracks (e.g.
 355             # https://soundcloud.com/giovannisarani/mezzo-valzer)
 356             # and might serve preview URLs (e.g.
 357             # http://www.soundcloud.com/snbrn/ele)
 358             format_dict = self._download_json(
 359                 track_base_url + '/streams', track_id,
 360                 'Downloading track url', query=query, fatal=False) or {}
 361
 362             for key, stream_url in format_dict.items():
 363                 if invalid_url(stream_url):
 364                     continue
 365                 format_urls.add(stream_url)
 366                 mobj = re.search(r'(http|hls)_([^_]+)_(\d+)_url', key)
 367                 if mobj:
 368                     protocol, ext, abr = mobj.groups()
 369                     add_format({
 370                         'abr': abr,
 371                         'ext': ext,
 372                         'url': stream_url,
 373                     }, protocol)
 374
 375         if not formats:
 376             # We fallback to the stream_url in the original info, this
 377             # cannot be always used, sometimes it can give an HTTP 404 error
 378             urlh = self._request_webpage(
 379                 HEADRequest(info.get('stream_url') or track_base_url + '/stream'),
 380                 track_id, query=query, fatal=False)
 381             if urlh:
 382                 stream_url = urlh.geturl()
 383                 if not invalid_url(stream_url):
 384                     add_format({'url': stream_url}, 'http')
 385
 386         for f in formats:
 387             f['vcodec'] = 'none'
 388
 389         self._sort_formats(formats)
 390
 391         user = info.get('user') or {}
 392
 393         thumbnails = []
 394         artwork_url = info.get('artwork_url')
 395         thumbnail = artwork_url or user.get('avatar_url')
 396         if isinstance(thumbnail, compat_str):
 397             if re.search(self._IMAGE_REPL_RE, thumbnail):
 398                 for image_id, size in self._ARTWORK_MAP.items():
 399                     i = {
 400                         'id': image_id,
 401                         'url': re.sub(self._IMAGE_REPL_RE, '-%s.jpg' % image_id, thumbnail),
 402                     }
 403                     if image_id == 'tiny' and not artwork_url:
 404                         size = 18
 405                     elif image_id == 'original':
 406                         i['preference'] = 10
 407                     if size:
 408                         i.update({
 409                             'width': size,
 410                             'height': size,
 411                         })
 412                     thumbnails.append(i)
 413             else:
 414                 thumbnails = [{'url': thumbnail}]
 415
 416         def extract_count(key):
 417             return int_or_none(info.get('%s_count' % key))
 418
 419         return {
 420             'id': track_id,
 421             'uploader': user.get('username'),
 422             'uploader_id': str_or_none(user.get('id')) or user.get('permalink'),
 423             'uploader_url': user.get('permalink_url'),
 424             'timestamp': unified_timestamp(info.get('created_at')),
 425             'title': title,
 426             'description': info.get('description'),
 427             'thumbnails': thumbnails,
 428             'duration': float_or_none(info.get('duration'), 1000),
 429             'webpage_url': info.get('permalink_url'),
 430             'license': info.get('license'),
 431             'view_count': extract_count('playback'),
 432             'like_count': extract_count('favoritings') or extract_count('likes'),
 433             'comment_count': extract_count('comment'),
 434             'repost_count': extract_count('reposts'),
 435             'genre': info.get('genre'),
 436             'formats': formats
 437         }
 438
 439     def _real_extract(self, url):
 440         mobj = re.match(self._VALID_URL, url)
 441
 442         track_id = mobj.group('track_id')
 443
 444         query = {
 445             'client_id': self._CLIENT_ID,
 446         }
 447         if track_id:
 448             info_json_url = self._API_V2_BASE + 'tracks/' + track_id
 449             full_title = track_id
 450             token = mobj.group('secret_token')
 451             if token:
 452                 query['secret_token'] = token
 453         else:
 454             full_title = resolve_title = '%s/%s' % mobj.group('uploader', 'title')
 455             token = mobj.group('token')
 456             if token:
 457                 resolve_title += '/%s' % token
 458             info_json_url = self._resolv_url(self._BASE_URL + resolve_title)
 459
 460         version = 2
 461         info = self._download_json(
 462             info_json_url, full_title, 'Downloading info JSON', query=query, fatal=False)
 463         if not info:
 464             info = self._download_json(
 465                 info_json_url.replace(self._API_V2_BASE, self._API_BASE),
 466                 full_title, 'Downloading info JSON', query=query)
 467             version = 1
 468
 469         return self._extract_info_dict(info, full_title, token, version)
 470
 471
 472 class SoundcloudPlaylistBaseIE(SoundcloudIE):
 473     def _extract_track_entries(self, tracks, token=None):
 474         entries = []
 475         for track in tracks:
 476             track_id = str_or_none(track.get('id'))
 477             url = track.get('permalink_url')
 478             if not url:
 479                 if not track_id:
 480                     continue
 481                 url = self._API_V2_BASE + 'tracks/' + track_id
 482                 if token:
 483                     url += '?secret_token=' + token
 484             entries.append(self.url_result(
 485                 url, SoundcloudIE.ie_key(), track_id))
 486         return entries
 487
 488
 489 class SoundcloudSetIE(SoundcloudPlaylistBaseIE):
 490     _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
 491     IE_NAME = 'soundcloud:set'
 492     _TESTS = [{
 493         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
 494         'info_dict': {
 495             'id': '2284613',
 496             'title': 'The Royal Concept EP',
 497         },
 498         'playlist_mincount': 5,
 499     }, {
 500         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep/token',
 501         'only_matching': True,
 502     }]
 503
 504     def _real_extract(self, url):
 505         mobj = re.match(self._VALID_URL, url)
 506
 507         full_title = '%s/sets/%s' % mobj.group('uploader', 'slug_title')
 508         token = mobj.group('token')
 509         if token:
 510             full_title += '/' + token
 511
 512         info = self._download_json(self._resolv_url(
 513             self._BASE_URL + full_title), full_title)
 514
 515         if 'errors' in info:
 516             msgs = (compat_str(err['error_message']) for err in info['errors'])
 517             raise ExtractorError('unable to download video webpage: %s' % ','.join(msgs))
 518
 519         entries = self._extract_track_entries(info['tracks'], token)
 520
 521         return self.playlist_result(
 522             entries, str_or_none(info.get('id')), info.get('title'))
 523
 524
 525 class SoundcloudPagedPlaylistBaseIE(SoundcloudPlaylistBaseIE):
 526     def _extract_playlist(self, base_url, playlist_id, playlist_title):
 527         COMMON_QUERY = {
 528             'limit': 2000000000,
 529             'client_id': self._CLIENT_ID,
 530             'linked_partitioning': '1',
 531         }
 532
 533         query = COMMON_QUERY.copy()
 534         query['offset'] = 0
 535
 536         next_href = base_url
 537
 538         entries = []
 539         for i in itertools.count():
 540             response = self._download_json(
 541                 next_href, playlist_id,
 542                 'Downloading track page %s' % (i + 1), query=query)
 543
 544             collection = response['collection']
 545
 546             if not isinstance(collection, list):
 547                 collection = []
 548
 549             # Empty collection may be returned, in this case we proceed
 550             # straight to next_href
 551
 552             def resolve_entry(candidates):
 553                 for cand in candidates:
 554                     if not isinstance(cand, dict):
 555                         continue
 556                     permalink_url = url_or_none(cand.get('permalink_url'))
 557                     if not permalink_url:
 558                         continue
 559                     return self.url_result(
 560                         permalink_url,
 561                         SoundcloudIE.ie_key() if SoundcloudIE.suitable(permalink_url) else None,
 562                         str_or_none(cand.get('id')), cand.get('title'))
 563
 564             for e in collection:
 565                 entry = resolve_entry((e, e.get('track'), e.get('playlist')))
 566                 if entry:
 567                     entries.append(entry)
 568
 569             next_href = response.get('next_href')
 570             if not next_href:
 571                 break
 572
 573             next_href = response['next_href']
 574             parsed_next_href = compat_urlparse.urlparse(next_href)
 575             query = compat_urlparse.parse_qs(parsed_next_href.query)
 576             query.update(COMMON_QUERY)
 577
 578         return {
 579             '_type': 'playlist',
 580             'id': playlist_id,
 581             'title': playlist_title,
 582             'entries': entries,
 583         }
 584
 585
 586 class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE):
 587     _VALID_URL = r'''(?x)
 588                         https?://
 589                             (?:(?:www|m)\.)?soundcloud\.com/
 590                             (?P<user>[^/]+)
 591                             (?:/
 592                                 (?P<rsrc>tracks|albums|sets|reposts|likes|spotlight)
 593                             )?
 594                             /?(?:[?#].*)?$
 595                     '''
 596     IE_NAME = 'soundcloud:user'
 597     _TESTS = [{
 598         'url': 'https://soundcloud.com/soft-cell-official',
 599         'info_dict': {
 600             'id': '207965082',
 601             'title': 'Soft Cell (All)',
 602         },
 603         'playlist_mincount': 28,
 604     }, {
 605         'url': 'https://soundcloud.com/soft-cell-official/tracks',
 606         'info_dict': {
 607             'id': '207965082',
 608             'title': 'Soft Cell (Tracks)',
 609         },
 610         'playlist_mincount': 27,
 611     }, {
 612         'url': 'https://soundcloud.com/soft-cell-official/albums',
 613         'info_dict': {
 614             'id': '207965082',
 615             'title': 'Soft Cell (Albums)',
 616         },
 617         'playlist_mincount': 1,
 618     }, {
 619         'url': 'https://soundcloud.com/jcv246/sets',
 620         'info_dict': {
 621             'id': '12982173',
 622             'title': 'Jordi / cv (Sets)',
 623         },
 624         'playlist_mincount': 2,
 625     }, {
 626         'url': 'https://soundcloud.com/jcv246/reposts',
 627         'info_dict': {
 628             'id': '12982173',
 629             'title': 'Jordi / cv (Reposts)',
 630         },
 631         'playlist_mincount': 6,
 632     }, {
 633         'url': 'https://soundcloud.com/clalberg/likes',
 634         'info_dict': {
 635             'id': '11817582',
 636             'title': 'clalberg (Likes)',
 637         },
 638         'playlist_mincount': 5,
 639     }, {
 640         'url': 'https://soundcloud.com/grynpyret/spotlight',
 641         'info_dict': {
 642             'id': '7098329',
 643             'title': 'Grynpyret (Spotlight)',
 644         },
 645         'playlist_mincount': 1,
 646     }]
 647
 648     _BASE_URL_MAP = {
 649         'all': 'stream/users/%s',
 650         'tracks': 'users/%s/tracks',
 651         'albums': 'users/%s/albums',
 652         'sets': 'users/%s/playlists',
 653         'reposts': 'stream/users/%s/reposts',
 654         'likes': 'users/%s/likes',
 655         'spotlight': 'users/%s/spotlight',
 656     }
 657
 658     def _real_extract(self, url):
 659         mobj = re.match(self._VALID_URL, url)
 660         uploader = mobj.group('user')
 661
 662         user = self._download_json(
 663             self._resolv_url(self._BASE_URL + uploader),
 664             uploader, 'Downloading user info')
 665
 666         resource = mobj.group('rsrc') or 'all'
 667
 668         return self._extract_playlist(
 669             self._API_V2_BASE + self._BASE_URL_MAP[resource] % user['id'],
 670             str_or_none(user.get('id')),
 671             '%s (%s)' % (user['username'], resource.capitalize()))
 672
 673
 674 class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE):
 675     _VALID_URL = r'https?://(?:(?:www|m)\.)?soundcloud\.com/stations/track/[^/]+/(?P<id>[^/?#&]+)'
 676     IE_NAME = 'soundcloud:trackstation'
 677     _TESTS = [{
 678         'url': 'https://soundcloud.com/stations/track/officialsundial/your-text',
 679         'info_dict': {
 680             'id': '286017854',
 681             'title': 'Track station: your text',
 682         },
 683         'playlist_mincount': 47,
 684     }]
 685
 686     def _real_extract(self, url):
 687         track_name = self._match_id(url)
 688
 689         track = self._download_json(self._resolv_url(url), track_name)
 690         track_id = self._search_regex(
 691             r'soundcloud:track-stations:(\d+)', track['id'], 'track id')
 692
 693         return self._extract_playlist(
 694             self._API_V2_BASE + 'stations/%s/tracks' % track['id'],
 695             track_id, 'Track station: %s' % track['title'])
 696
 697
 698 class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE):
 699     _VALID_URL = r'https?://api(?:-v2)?\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
 700     IE_NAME = 'soundcloud:playlist'
 701     _TESTS = [{
 702         'url': 'https://api.soundcloud.com/playlists/4110309',
 703         'info_dict': {
 704             'id': '4110309',
 705             'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
 706             'description': 're:.*?TILT Brass - Bowery Poetry Club',
 707         },
 708         'playlist_count': 6,
 709     }]
 710
 711     def _real_extract(self, url):
 712         mobj = re.match(self._VALID_URL, url)
 713         playlist_id = mobj.group('id')
 714
 715         query = {
 716             'client_id': self._CLIENT_ID,
 717         }
 718         token = mobj.group('token')
 719         if token:
 720             query['secret_token'] = token
 721
 722         data = self._download_json(
 723             self._API_V2_BASE + 'playlists/' + playlist_id,
 724             playlist_id, 'Downloading playlist', query=query)
 725
 726         entries = self._extract_track_entries(data['tracks'], token)
 727
 728         return self.playlist_result(
 729             entries, playlist_id, data.get('title'), data.get('description'))
 730
 731
 732 class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
 733     IE_NAME = 'soundcloud:search'
 734     IE_DESC = 'Soundcloud search'
 735     _MAX_RESULTS = float('inf')
 736     _TESTS = [{
 737         'url': 'scsearch15:post-avant jazzcore',
 738         'info_dict': {
 739             'title': 'post-avant jazzcore',
 740         },
 741         'playlist_count': 15,
 742     }]
 743
 744     _SEARCH_KEY = 'scsearch'
 745     _MAX_RESULTS_PER_PAGE = 200
 746     _DEFAULT_RESULTS_PER_PAGE = 50
 747
 748     def _get_collection(self, endpoint, collection_id, **query):
 749         limit = min(
 750             query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
 751             self._MAX_RESULTS_PER_PAGE)
 752         query.update({
 753             'limit': limit,
 754             'client_id': self._CLIENT_ID,
 755             'linked_partitioning': 1,
 756             'offset': 0,
 757         })
 758         next_url = update_url_query(self._API_V2_BASE + endpoint, query)
 759
 760         collected_results = 0
 761
 762         for i in itertools.count(1):
 763             response = self._download_json(
 764                 next_url, collection_id, 'Downloading page {0}'.format(i),
 765                 'Unable to download API page')
 766
 767             collection = response.get('collection', [])
 768             if not collection:
 769                 break
 770
 771             collection = list(filter(bool, collection))
 772             collected_results += len(collection)
 773
 774             for item in collection:
 775                 yield self.url_result(item['uri'], SoundcloudIE.ie_key())
 776
 777             if not collection or collected_results >= limit:
 778                 break
 779
 780             next_url = response.get('next_href')
 781             if not next_url:
 782                 break
 783
 784     def _get_n_results(self, query, n):
 785         tracks = self._get_collection('search/tracks', query, limit=n, q=query)
 786         return self.playlist_result(tracks, playlist_title=query)