[youtube] Fix extraction.

[youtube-dl] / youtube_dl / extractor / vk.py

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index c289fcad34daac3ea2b2863654551bfa1aba7094..00ec006c465e4f350bdc3f1099e6d38e5e2a8b8b 100644 (file)
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import collections
+import functools
 import re
 
 from .common import InfoExtractor
@@ -11,6 +12,7 @@ from ..utils import (
     ExtractorError,
     get_element_by_class,
     int_or_none,
+    OnDemandPagedList,
     orderedSet,
     str_or_none,
     str_to_int,
@@ -216,8 +218,7 @@ class VKIE(VKBaseIE):
                 'id': 'k3lz2cmXyRuJQSjGHUv',
                 'ext': 'mp4',
                 'title': 'md5:d52606645c20b0ddbb21655adaa4f56f',
-                # TODO: fix test by fixing dailymotion description extraction
-                'description': 'md5:c651358f03c56f1150b555c26d90a0fd',
+                'description': 'md5:424b8e88cc873217f520e582ba28bb36',
                 'uploader': 'AniLibria.Tv',
                 'upload_date': '20160914',
                 'uploader_id': 'x1p5vl5',
@@ -478,14 +479,23 @@ class VKIE(VKBaseIE):
 class VKUserVideosIE(VKBaseIE):
     IE_NAME = 'vk:uservideos'
     IE_DESC = "VK - User's Videos"
-    _VALID_URL = r'https?://(?:(?:m|new)\.)?vk\.com/videos(?P<id>-?[0-9]+)(?!\?.*\bz=video)(?:[/?#&]|$)'
+    _VALID_URL = r'https?://(?:(?:m|new)\.)?vk\.com/videos(?P<id>-?[0-9]+)(?!\?.*\bz=video)(?:[/?#&](?:.*?\bsection=(?P<section>\w+))?|$)'
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TESTS = [{
-        'url': 'http://vk.com/videos205387401',
+        'url': 'https://vk.com/videos-767561',
+        'info_dict': {
+            'id': '-767561_all',
+        },
+        'playlist_mincount': 1150,
+    }, {
+        'url': 'https://vk.com/videos-767561?section=uploaded',
         'info_dict': {
-            'id': '205387401',
+            'id': '-767561_uploaded',
         },
-        'playlist_mincount': 4,
+        'playlist_mincount': 425,
+    }, {
+        'url': 'http://vk.com/videos205387401',
+        'only_matching': True,
     }, {
         'url': 'http://vk.com/videos-77521',
         'only_matching': True,
@@ -499,25 +509,33 @@ class VKUserVideosIE(VKBaseIE):
         'url': 'http://new.vk.com/videos205387401',
         'only_matching': True,
     }]
-    _VIDEO = collections.namedtuple(
-        'Video', ['owner_id', 'id', 'thumb', 'title', 'flags', 'duration', 'hash', 'moder_acts', 'owner', 'date', 'views', 'platform', 'blocked', 'music_video_meta'])
-
-    def _real_extract(self, url):
-        page_id = self._match_id(url)
+    _PAGE_SIZE = 1000
+    _VIDEO = collections.namedtuple('Video', ['owner_id', 'id'])
 
+    def _fetch_page(self, page_id, section, page):
         l = self._download_payload('al_video', page_id, {
             'act': 'load_videos_silent',
+            'offset': page * self._PAGE_SIZE,
             'oid': page_id,
-        })[0]['']['list']
+            'section': section,
+        })[0][section]['list']
 
-        entries = []
         for video in l:
-            v = self._VIDEO._make(video)
+            v = self._VIDEO._make(video[:2])
             video_id = '%d_%d' % (v.owner_id, v.id)
-            entries.append(self.url_result(
-                'http://vk.com/video' + video_id, 'VK', video_id=video_id))
+            yield self.url_result(
+                'http://vk.com/video' + video_id, VKIE.ie_key(), video_id)
+
+    def _real_extract(self, url):
+        page_id, section = re.match(self._VALID_URL, url).groups()
+        if not section:
+            section = 'all'
+
+        entries = OnDemandPagedList(
+            functools.partial(self._fetch_page, page_id, section),
+            self._PAGE_SIZE)
 
-        return self.playlist_result(entries, page_id)
+        return self.playlist_result(entries, '%s_%s' % (page_id, section))
 
 
 class VKWallPostIE(VKBaseIE):
@@ -581,8 +599,7 @@ class VKWallPostIE(VKBaseIE):
         'only_matching': True,
     }]
     _BASE64_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN0PQRSTUVWXYZO123456789+/='
-    _AUDIO = collections.namedtuple(
-        'Audio', ['id', 'owner_id', 'url', 'title', 'performer', 'duration', 'album_id', 'unk', 'author_link', 'lyrics', 'flags', 'context', 'extra', 'hashes', 'cover_url', 'ads', 'subtitle', 'main_artists', 'feat_artists', 'album', 'track_code', 'restriction', 'album_part', 'new_stats', 'access_key'])
+    _AUDIO = collections.namedtuple('Audio', ['id', 'owner_id', 'url', 'title', 'performer', 'duration', 'album_id', 'unk', 'author_link', 'lyrics', 'flags', 'context', 'extra', 'hashes', 'cover_url', 'ads'])
 
     def _decode(self, enc):
         dec = ''
@@ -600,7 +617,6 @@ class VKWallPostIE(VKBaseIE):
         if 'audio_api_unavailable' in mask_url:
             extra = mask_url.split('?extra=')[1].split('#')
             func, base = self._decode(extra[1]).split(chr(11))
-            assert (func == 'i')
             mask_url = list(self._decode(extra[0]))
             url_len = len(mask_url)
             indexes = [None] * url_len
@@ -631,18 +647,19 @@ class VKWallPostIE(VKBaseIE):
 
         for audio in re.findall(r'data-audio="([^"]+)', webpage):
             audio = self._parse_json(unescapeHTML(audio), post_id)
-            a = self._AUDIO._make(audio)
+            a = self._AUDIO._make(audio[:16])
             if not a.url:
                 continue
             title = unescapeHTML(a.title)
+            performer = unescapeHTML(a.performer)
             entries.append({
                 'id': '%s_%s' % (a.owner_id, a.id),
                 'url': self._unmask_url(a.url, a.ads['vk_id']),
-                'title': '%s - %s' % (a.performer, title) if a.performer else title,
-                'thumbnail': a.cover_url.split(',') if a.cover_url else None,
-                'duration': a.duration,
+                'title': '%s - %s' % (performer, title) if performer else title,
+                'thumbnails': [{'url': c_url} for c_url in a.cover_url.split(',')] if a.cover_url else None,
+                'duration': int_or_none(a.duration),
                 'uploader': uploader,
-                'artist': a.performer,
+                'artist': performer,
                 'track': title,
                 'ext': 'mp4',
                 'protocol': 'm3u8',