[stretchinternet] fix extraction(closes #4319)

[youtube-dl] / youtube_dl / extractor / soundcloud.py

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index b3ffef8df9450eb2f66ef5f4fba8197e2f9f72af..a0b09f5b1747e3b1b9b3a17b10d482b47512143a 100644 (file)
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -15,6 +15,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    error_to_compat_str,
     ExtractorError,
     float_or_none,
     HEADRequest,
@@ -95,7 +96,7 @@ class SoundcloudIE(InfoExtractor):
                 'repost_count': int,
             }
         },
-        # not streamable song
+        # not streamable song, preview
         {
             'url': 'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
             'info_dict': {
@@ -118,7 +119,6 @@ class SoundcloudIE(InfoExtractor):
                 # rtmp
                 'skip_download': True,
             },
-            'skip': 'Preview',
         },
         # private link
         {
@@ -272,6 +272,9 @@ class SoundcloudIE(InfoExtractor):
         'original': 0,
     }
 
+    def _store_client_id(self, client_id):
+        self._downloader.cache.store('soundcloud', 'client_id', client_id)
+
     def _update_client_id(self):
         webpage = self._download_webpage('https://soundcloud.com/', None)
         for src in reversed(re.findall(r'<script[^>]+src="([^"]+)"', webpage)):
@@ -282,11 +285,14 @@ class SoundcloudIE(InfoExtractor):
                     script, 'client id', default=None)
                 if client_id:
                     self._CLIENT_ID = client_id
-                    self._downloader.cache.store('soundcloud', 'client_id', client_id)
+                    self._store_client_id(client_id)
                     return
         raise ExtractorError('Unable to extract client id')
 
     def _download_json(self, *args, **kwargs):
+        non_fatal = kwargs.get('fatal') is False
+        if non_fatal:
+            del kwargs['fatal']
         query = kwargs.get('query', {}).copy()
         for _ in range(2):
             query['client_id'] = self._CLIENT_ID
@@ -295,8 +301,12 @@ class SoundcloudIE(InfoExtractor):
                 return super(SoundcloudIE, self)._download_json(*args, **compat_kwargs(kwargs))
             except ExtractorError as e:
                 if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                    self._store_client_id(None)
                     self._update_client_id()
                     continue
+                elif non_fatal:
+                    self._downloader.report_warning(error_to_compat_str(e))
+                    return False
                 raise
 
     def _real_initialize(self):
@@ -335,9 +345,9 @@ class SoundcloudIE(InfoExtractor):
             })
 
         def invalid_url(url):
-            return not url or url in format_urls or re.search(r'/(?:preview|playlist)/0/30/', url)
+            return not url or url in format_urls
 
-        def add_format(f, protocol):
+        def add_format(f, protocol, is_preview=False):
             mobj = re.search(r'\.(?P<abr>\d+)\.(?P<ext>[0-9a-z]{3,4})(?=[/?])', stream_url)
             if mobj:
                 for k, v in mobj.groupdict().items():
@@ -350,12 +360,16 @@ class SoundcloudIE(InfoExtractor):
                 v = f.get(k)
                 if v:
                     format_id_list.append(v)
+            preview = is_preview or re.search(r'/(?:preview|playlist)/0/30/', f['url'])
+            if preview:
+                format_id_list.append('preview')
             abr = f.get('abr')
             if abr:
                 f['abr'] = int(abr)
             f.update({
                 'format_id': '_'.join(format_id_list),
                 'protocol': 'm3u8_native' if protocol == 'hls' else 'http',
+                'preference': -10 if preview else None,
             })
             formats.append(f)
 
@@ -366,7 +380,7 @@ class SoundcloudIE(InfoExtractor):
             if not isinstance(t, dict):
                 continue
             format_url = url_or_none(t.get('url'))
-            if not format_url or t.get('snipped') or '/preview/' in format_url:
+            if not format_url:
                 continue
             stream = self._download_json(
                 format_url, track_id, query=query, fatal=False)
@@ -389,7 +403,8 @@ class SoundcloudIE(InfoExtractor):
             add_format({
                 'url': stream_url,
                 'ext': ext,
-            }, 'http' if protocol == 'progressive' else protocol)
+            }, 'http' if protocol == 'progressive' else protocol,
+                t.get('snipped') or '/preview/' in format_url)
 
         if not formats:
             # Old API, does not work for some tracks (e.g.