[youtube] Unescape HTML for series (closes #18641)

[youtube-dl] / youtube_dl / extractor / vrv.py

diff --git a/youtube_dl/extractor/vrv.py b/youtube_dl/extractor/vrv.py
index 487047fd78c65a758ccfa04fd50c16bfddda2623..483a3be3a8c1f1b0c557c308a86bd7d55de82bdd 100644 (file)
--- a/youtube_dl/extractor/vrv.py
+++ b/youtube_dl/extractor/vrv.py
@@ -12,7 +12,7 @@ import time
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_urlencode,
-    compat_urlparse,
+    compat_urllib_parse,
 )
 from ..utils import (
     float_or_none,
@@ -39,11 +39,11 @@ class VRVBaseIE(InfoExtractor):
             data = json.dumps(data).encode()
             headers['Content-Type'] = 'application/json'
         method = 'POST' if data else 'GET'
-        base_string = '&'.join([method, compat_urlparse.quote(base_url, ''), compat_urlparse.quote(encoded_query, '')])
+        base_string = '&'.join([method, compat_urllib_parse.quote(base_url, ''), compat_urllib_parse.quote(encoded_query, '')])
         oauth_signature = base64.b64encode(hmac.new(
             (self._API_PARAMS['oAuthSecret'] + '&').encode('ascii'),
             base_string.encode(), hashlib.sha1).digest()).decode()
-        encoded_query += '&oauth_signature=' + compat_urlparse.quote(oauth_signature, '')
+        encoded_query += '&oauth_signature=' + compat_urllib_parse.quote(oauth_signature, '')
         return self._download_json(
             '?'.join([base_url, encoded_query]), video_id,
             note='Downloading %s JSON metadata' % note, headers=headers, data=data)
@@ -72,7 +72,7 @@ class VRVBaseIE(InfoExtractor):
 class VRVIE(VRVBaseIE):
     IE_NAME = 'vrv'
     _VALID_URL = r'https?://(?:www\.)?vrv\.co/watch/(?P<id>[A-Z0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://vrv.co/watch/GR9PNZ396/Hidden-America-with-Jonah-Ray:BOSTON-WHERE-THE-PAST-IS-THE-PRESENT',
         'info_dict': {
             'id': 'GR9PNZ396',
@@ -85,7 +85,34 @@ class VRVIE(VRVBaseIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }]
+
+    def _extract_vrv_formats(self, url, video_id, stream_format, audio_lang, hardsub_lang):
+        if not url or stream_format not in ('hls', 'dash'):
+            return []
+        assert audio_lang or hardsub_lang
+        stream_id_list = []
+        if audio_lang:
+            stream_id_list.append('audio-%s' % audio_lang)
+        if hardsub_lang:
+            stream_id_list.append('hardsub-%s' % hardsub_lang)
+        stream_id = '-'.join(stream_id_list)
+        format_id = '%s-%s' % (stream_format, stream_id)
+        if stream_format == 'hls':
+            adaptive_formats = self._extract_m3u8_formats(
+                url, video_id, 'mp4', m3u8_id=format_id,
+                note='Downloading %s m3u8 information' % stream_id,
+                fatal=False)
+        elif stream_format == 'dash':
+            adaptive_formats = self._extract_mpd_formats(
+                url, video_id, mpd_id=format_id,
+                note='Downloading %s MPD information' % stream_id,
+                fatal=False)
+        if audio_lang:
+            for f in adaptive_formats:
+                if f.get('acodec') != 'none':
+                    f['language'] = audio_lang
+        return adaptive_formats
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -93,8 +120,10 @@ class VRVIE(VRVBaseIE):
             url, video_id,
             headers=self.geo_verification_headers())
         media_resource = self._parse_json(self._search_regex(
-            r'window\.__INITIAL_STATE__\s*=\s*({.+?})</script>',
-            webpage, 'inital state'), video_id).get('watch', {}).get('mediaResource') or {}
+            [
+                r'window\.__INITIAL_STATE__\s*=\s*({.+?})(?:</script>|;)',
+                r'window\.__INITIAL_STATE__\s*=\s*({.+})'
+            ], webpage, 'inital state'), video_id).get('watch', {}).get('mediaResource') or {}
 
         video_data = media_resource.get('json')
         if not video_data:
@@ -112,21 +141,24 @@ class VRVIE(VRVBaseIE):
 
         audio_locale = streams_json.get('audio_locale')
         formats = []
-        for stream_id, stream in streams_json.get('streams', {}).get('adaptive_hls', {}).items():
-            stream_url = stream.get('url')
-            if not stream_url:
-                continue
-            stream_id = stream_id or audio_locale
-            m3u8_formats = self._extract_m3u8_formats(
-                stream_url, video_id, 'mp4', m3u8_id=stream_id,
-                note='Downloading %s m3u8 information' % stream_id,
-                fatal=False)
-            if audio_locale:
-                for f in m3u8_formats:
-                    f['language'] = audio_locale
-            formats.extend(m3u8_formats)
+        for stream_type, streams in streams_json.get('streams', {}).items():
+            if stream_type in ('adaptive_hls', 'adaptive_dash'):
+                for stream in streams.values():
+                    formats.extend(self._extract_vrv_formats(
+                        stream.get('url'), video_id, stream_type.split('_')[1],
+                        audio_locale, stream.get('hardsub_locale')))
         self._sort_formats(formats)
 
+        subtitles = {}
+        for subtitle in streams_json.get('subtitles', {}).values():
+            subtitle_url = subtitle.get('url')
+            if not subtitle_url:
+                continue
+            subtitles.setdefault(subtitle.get('locale', 'en-US'), []).append({
+                'url': subtitle_url,
+                'ext': subtitle.get('format', 'ass'),
+            })
+
         thumbnails = []
         for thumbnail in video_data.get('images', {}).get('thumbnails', []):
             thumbnail_url = thumbnail.get('source')
@@ -142,6 +174,7 @@ class VRVIE(VRVBaseIE):
             'id': video_id,
             'title': title,
             'formats': formats,
+            'subtitles': subtitles,
             'thumbnails': thumbnails,
             'description': video_data.get('description'),
             'duration': float_or_none(video_data.get('duration_ms'), 1000),