[youtube] Fix extraction.

[youtube-dl] / youtube_dl / extractor / neteasemusic.py

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index e73c77f89b47f5ac3e9b5b460cb173be3247c86a..978a05841ce68161330f9db24169dd330e51efc1 100644 (file)
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -4,13 +4,17 @@ from __future__ import unicode_literals
 from hashlib import md5
 from base64 import b64encode
 from datetime import datetime
-import itertools
 import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_request,
-    compat_urllib_parse,
+    compat_urllib_parse_urlencode,
+    compat_str,
+    compat_itertools_count,
+)
+from ..utils import (
+    sanitized_Request,
+    float_or_none,
 )
 
 
@@ -21,33 +25,42 @@ class NetEaseMusicBaseIE(InfoExtractor):
 
     @classmethod
     def _encrypt(cls, dfsid):
-        salt_bytes = bytearray(cls._NETEASE_SALT, 'utf-8')
-        string_bytes = bytearray(str(dfsid))
+        salt_bytes = bytearray(cls._NETEASE_SALT.encode('utf-8'))
+        string_bytes = bytearray(compat_str(dfsid).encode('ascii'))
         salt_len = len(salt_bytes)
         for i in range(len(string_bytes)):
             string_bytes[i] = string_bytes[i] ^ salt_bytes[i % salt_len]
         m = md5()
-        m.update(string_bytes)
-        result = b64encode(m.digest())
+        m.update(bytes(string_bytes))
+        result = b64encode(m.digest()).decode('ascii')
         return result.replace('/', '_').replace('+', '-')
 
-    @classmethod
-    def extract_formats(cls, info):
+    def extract_formats(self, info):
         formats = []
-        for song_format in cls._FORMATS:
+        for song_format in self._FORMATS:
             details = info.get(song_format)
             if not details:
                 continue
-            formats.append({
-                'url': 'http://m1.music.126.net/%s/%s.%s' %
-                       (cls._encrypt(details['dfsId']), details['dfsId'],
-                        details['extension']),
-                'ext': details.get('extension'),
-                'abr': details.get('bitrate', 0) / 1000,
-                'format_id': song_format,
-                'filesize': details.get('size'),
-                'asr': details.get('sr')
-            })
+            song_file_path = '/%s/%s.%s' % (
+                self._encrypt(details['dfsId']), details['dfsId'], details['extension'])
+
+            # 203.130.59.9, 124.40.233.182, 115.231.74.139, etc is a reverse proxy-like feature
+            # from NetEase's CDN provider that can be used if m5.music.126.net does not
+            # work, especially for users outside of Mainland China
+            # via: https://github.com/JixunMoe/unblock-163/issues/3#issuecomment-163115880
+            for host in ('http://m5.music.126.net', 'http://115.231.74.139/m1.music.126.net',
+                         'http://124.40.233.182/m1.music.126.net', 'http://203.130.59.9/m1.music.126.net'):
+                song_url = host + song_file_path
+                if self._is_valid_url(song_url, info['id'], 'song'):
+                    formats.append({
+                        'url': song_url,
+                        'ext': details.get('extension'),
+                        'abr': float_or_none(details.get('bitrate'), scale=1000),
+                        'format_id': song_format,
+                        'filesize': details.get('size'),
+                        'asr': details.get('sr')
+                    })
+                    break
         return formats
 
     @classmethod
@@ -55,13 +68,14 @@ class NetEaseMusicBaseIE(InfoExtractor):
         return int(round(ms / 1000.0))
 
     def query_api(self, endpoint, video_id, note):
-        req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
+        req = sanitized_Request('%s%s' % (self._API_BASE, endpoint))
         req.add_header('Referer', self._API_BASE)
         return self._download_json(req, video_id, note)
 
 
 class NetEaseMusicIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:song'
+    IE_DESC = '网易云音乐'
     _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/song?id=32102397',
@@ -75,6 +89,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'timestamp': 1431878400,
             'description': 'md5:a10a54589c2860300d02e1de821eb2ef',
         },
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'No lyrics translation.',
         'url': 'http://music.163.com/#/song?id=29822014',
@@ -87,6 +102,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'timestamp': 1419523200,
             'description': 'md5:a4d8d89f44656af206b7b2555c0bce6c',
         },
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'No lyrics.',
         'url': 'http://music.163.com/song?id=17241424',
@@ -98,6 +114,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'upload_date': '20080211',
             'timestamp': 1202745600,
         },
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'Has translated name.',
         'url': 'http://music.163.com/#/song?id=22735043',
@@ -110,7 +127,8 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'upload_date': '20100127',
             'timestamp': 1264608000,
             'alt_title': '说出愿望吧(Genie)',
-        }
+        },
+        'skip': 'Blocked outside Mainland China',
     }]
 
     def _process_lyrics(self, lyrics_info):
@@ -139,7 +157,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'ids': '[%s]' % song_id
         }
         info = self.query_api(
-            'song/detail?' + compat_urllib_parse.urlencode(params),
+            'song/detail?' + compat_urllib_parse_urlencode(params),
             song_id, 'Downloading song info')['songs'][0]
 
         formats = self.extract_formats(info)
@@ -169,6 +187,7 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:album'
+    IE_DESC = '网易云音乐 - 专辑'
     _VALID_URL = r'https?://music\.163\.com/(#/)?album\?id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://music.163.com/#/album?id=220780',
@@ -177,6 +196,7 @@ class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
             'title': 'B\'day',
         },
         'playlist_count': 23,
+        'skip': 'Blocked outside Mainland China',
     }
 
     def _real_extract(self, url):
@@ -198,6 +218,7 @@ class NetEaseMusicAlbumIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:singer'
+    IE_DESC = '网易云音乐 - 歌手'
     _VALID_URL = r'https?://music\.163\.com/(#/)?artist\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'note': 'Singer has aliases.',
@@ -207,6 +228,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
             'title': '张惠妹 - aMEI;阿密特',
         },
         'playlist_count': 50,
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'Singer has translated name.',
         'url': 'http://music.163.com/#/artist?id=124098',
@@ -215,6 +237,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
             'title': '李昇基 - 이승기',
         },
         'playlist_count': 50,
+        'skip': 'Blocked outside Mainland China',
     }]
 
     def _real_extract(self, url):
@@ -228,7 +251,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
         if info['artist']['trans']:
             name = '%s - %s' % (name, info['artist']['trans'])
         if info['artist']['alias']:
-            name = '%s - %s' % (name, ";".join(info['artist']['alias']))
+            name = '%s - %s' % (name, ';'.join(info['artist']['alias']))
 
         entries = [
             self.url_result('http://music.163.com/#/song?id=%s' % song['id'],
@@ -240,6 +263,7 @@ class NetEaseMusicSingerIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicListIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:playlist'
+    IE_DESC = '网易云音乐 - 歌单'
     _VALID_URL = r'https?://music\.163\.com/(#/)?(playlist|discover/toplist)\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/playlist?id=79177352',
@@ -249,6 +273,7 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
             'description': 'md5:12fd0819cab2965b9583ace0f8b7b022'
         },
         'playlist_count': 99,
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'Toplist/Charts sample',
         'url': 'http://music.163.com/#/discover/toplist?id=3733003',
@@ -258,6 +283,7 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
             'description': 'md5:73ec782a612711cadc7872d9c1e134fc',
         },
         'playlist_count': 50,
+        'skip': 'Blocked outside Mainland China',
     }]
 
     def _real_extract(self, url):
@@ -285,6 +311,7 @@ class NetEaseMusicListIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicMvIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:mv'
+    IE_DESC = '网易云音乐 - MV'
     _VALID_URL = r'https?://music\.163\.com/(#/)?mv\?id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://music.163.com/#/mv?id=415350',
@@ -296,6 +323,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
             'creator': '白雅言',
             'upload_date': '20150520',
         },
+        'skip': 'Blocked outside Mainland China',
     }
 
     def _real_extract(self, url):
@@ -325,6 +353,7 @@ class NetEaseMusicMvIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:program'
+    IE_DESC = '网易云音乐 - 电台节目'
     _VALID_URL = r'https?://music\.163\.com/(#/?)program\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/program?id=10109055',
@@ -338,6 +367,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
             'upload_date': '20150613',
             'duration': 900,
         },
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'This program has accompanying songs.',
         'url': 'http://music.163.com/#/program?id=10141022',
@@ -347,6 +377,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
             'description': 'md5:8d594db46cc3e6509107ede70a4aaa3b',
         },
         'playlist_count': 4,
+        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'This program has accompanying songs.',
         'url': 'http://music.163.com/#/program?id=10141022',
@@ -360,7 +391,8 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
         },
         'params': {
             'noplaylist': True
-        }
+        },
+        'skip': 'Blocked outside Mainland China',
     }]
 
     def _real_extract(self, url):
@@ -409,6 +441,7 @@ class NetEaseMusicProgramIE(NetEaseMusicBaseIE):
 
 class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:djradio'
+    IE_DESC = '网易云音乐 - 电台'
     _VALID_URL = r'https?://music\.163\.com/(#/)?djradio\?id=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://music.163.com/#/djradio?id=42',
@@ -418,6 +451,7 @@ class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
             'description': 'md5:766220985cbd16fdd552f64c578a6b15'
         },
         'playlist_mincount': 40,
+        'skip': 'Blocked outside Mainland China',
     }
     _PAGE_SIZE = 1000
 
@@ -427,7 +461,7 @@ class NetEaseMusicDjRadioIE(NetEaseMusicBaseIE):
         name = None
         desc = None
         entries = []
-        for offset in itertools.count(start=0, step=self._PAGE_SIZE):
+        for offset in compat_itertools_count(start=0, step=self._PAGE_SIZE):
             info = self.query_api(
                 'dj/program/byradio?asc=false&limit=%d&radioId=%s&offset=%d'
                 % (self._PAGE_SIZE, dj_id, offset),