X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2Fextractor%2Fyouku.py;h=61d1ab20908bf1a233cf90624170d9ee824ff337;hb=HEAD;hp=37be440faab9b68a5c656883a212f2a0d996de6e;hpb=59ed87cbd9ea08c889514a05b646141004f432a1;p=youtube-dl diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py index 37be440fa..61d1ab209 100644 --- a/youtube_dl/extractor/youku.py +++ b/youtube_dl/extractor/youku.py @@ -1,7 +1,6 @@ # coding: utf-8 from __future__ import unicode_literals -import itertools import random import re import string @@ -10,7 +9,10 @@ import time from .common import InfoExtractor from ..utils import ( ExtractorError, - get_element_by_attribute, + get_element_by_class, + js_to_json, + str_or_none, + strip_jsonp, ) @@ -19,7 +21,9 @@ class YoukuIE(InfoExtractor): IE_DESC = '优酷' _VALID_URL = r'''(?x) (?: - http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)| + https?://( + (?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)| + video\.tudou\.com/v/)| youku:) (?P[A-Za-z0-9]+)(?:\.html|/v\.swf|) ''' @@ -31,6 +35,12 @@ class YoukuIE(InfoExtractor): 'id': 'XMTc1ODE5Njcy', 'title': '★Smile﹗♡ Git Fresh -Booty Music舞蹈.', 'ext': 'mp4', + 'duration': 74.73, + 'thumbnail': r're:^https?://.*', + 'uploader': '。躲猫猫、', + 'uploader_id': '36017967', + 'uploader_url': 'http://i.youku.com/u/UMTQ0MDcxODY4', + 'tags': list, } }, { 'url': 'http://player.youku.com/player.php/sid/XNDgyMDQ2NTQw/v.swf', @@ -41,6 +51,12 @@ class YoukuIE(InfoExtractor): 'id': 'XODgxNjg1Mzk2', 'ext': 'mp4', 'title': '武媚娘传奇 85', + 'duration': 1999.61, + 'thumbnail': r're:^https?://.*', + 'uploader': '疯狂豆花', + 'uploader_id': '62583473', + 'uploader_url': 'http://i.youku.com/u/UMjUwMzMzODky', + 'tags': list, }, }, { 'url': 'http://v.youku.com/v_show/id_XMTI1OTczNDM5Mg==.html', @@ -48,6 +64,12 @@ class YoukuIE(InfoExtractor): 'id': 'XMTI1OTczNDM5Mg', 'ext': 'mp4', 'title': '花千骨 04', + 'duration': 2363, + 'thumbnail': r're:^https?://.*', + 'uploader': '放剧场-花千骨', + 'uploader_id': '772849359', + 'uploader_url': 'http://i.youku.com/u/UMzA5MTM5NzQzNg==', + 'tags': list, }, }, { 'url': 'http://v.youku.com/v_show/id_XNjA1NzA2Njgw.html', @@ -56,6 +78,12 @@ class YoukuIE(InfoExtractor): 'id': 'XNjA1NzA2Njgw', 'ext': 'mp4', 'title': '邢義田复旦讲座之想象中的胡人—从“左衽孔子”说起', + 'duration': 7264.5, + 'thumbnail': r're:^https?://.*', + 'uploader': 'FoxJin1006', + 'uploader_id': '322014285', + 'uploader_url': 'http://i.youku.com/u/UMTI4ODA1NzE0MA==', + 'tags': list, }, 'params': { 'videopassword': '100600', @@ -67,7 +95,29 @@ class YoukuIE(InfoExtractor): 'id': 'XOTUxMzg4NDMy', 'ext': 'mp4', 'title': '我的世界☆明月庄主☆车震猎杀☆杀人艺术Minecraft', + 'duration': 702.08, + 'thumbnail': r're:^https?://.*', + 'uploader': '明月庄主moon', + 'uploader_id': '38465621', + 'uploader_url': 'http://i.youku.com/u/UMTUzODYyNDg0', + 'tags': list, }, + }, { + 'url': 'http://video.tudou.com/v/XMjIyNzAzMTQ4NA==.html?f=46177805', + 'info_dict': { + 'id': 'XMjIyNzAzMTQ4NA', + 'ext': 'mp4', + 'title': '卡马乔国足开大脚长传冲吊集锦', + 'duration': 289, + 'thumbnail': r're:^https?://.*', + 'uploader': '阿卜杜拉之星', + 'uploader_id': '2382249', + 'uploader_url': 'http://i.youku.com/u/UOTUyODk5Ng==', + 'tags': list, + }, + }, { + 'url': 'http://video.tudou.com/v/XMjE4ODI3OTg2MA==.html', + 'only_matching': True, }] @staticmethod @@ -104,7 +154,7 @@ class YoukuIE(InfoExtractor): # request basic data basic_data_params = { 'vid': video_id, - 'ccode': '0401', + 'ccode': '0590', 'client_ip': '192.168.1.1', 'utid': cna, 'client_ts': time.time() / 1000, @@ -139,7 +189,8 @@ class YoukuIE(InfoExtractor): raise ExtractorError(msg) # get video title - title = data['video']['title'] + video_data = data['video'] + title = video_data['title'] formats = [{ 'url': stream['m3u8_url'], @@ -156,53 +207,103 @@ class YoukuIE(InfoExtractor): 'id': video_id, 'title': title, 'formats': formats, + 'duration': video_data.get('seconds'), + 'thumbnail': video_data.get('logo'), + 'uploader': video_data.get('username'), + 'uploader_id': str_or_none(video_data.get('userid')), + 'uploader_url': data.get('uploader', {}).get('homepage'), + 'tags': video_data.get('tags'), } class YoukuShowIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?youku\.com/show_page/id_(?P[0-9a-z]+)\.html' + _VALID_URL = r'https?://list\.youku\.com/show/id_(?P[0-9a-z]+)\.html' IE_NAME = 'youku:show' - _TEST = { - 'url': 'http://www.youku.com/show_page/id_zc7c670be07ff11e48b3f.html', + _TESTS = [{ + 'url': 'http://list.youku.com/show/id_zc7c670be07ff11e48b3f.html', 'info_dict': { 'id': 'zc7c670be07ff11e48b3f', - 'title': '花千骨 未删减版', - 'description': 'md5:578d4f2145ae3f9128d9d4d863312910', + 'title': '花千骨 DVD版', + 'description': 'md5:a1ae6f5618571bbeb5c9821f9c81b558', }, 'playlist_count': 50, - } - - _PAGE_SIZE = 40 + }, { + # Episode number not starting from 1 + 'url': 'http://list.youku.com/show/id_zefbfbd70efbfbd780bef.html', + 'info_dict': { + 'id': 'zefbfbd70efbfbd780bef', + 'title': '超级飞侠3', + 'description': 'md5:275715156abebe5ccc2a1992e9d56b98', + }, + 'playlist_count': 24, + }, { + # Ongoing playlist. The initial page is the last one + 'url': 'http://list.youku.com/show/id_za7c275ecd7b411e1a19e.html', + 'only_matching': True, + }, { + # No data-id value. + 'url': 'http://list.youku.com/show/id_zefbfbd61237fefbfbdef.html', + 'only_matching': True, + }, { + # Wrong number of reload_id. + 'url': 'http://list.youku.com/show/id_z20eb4acaf5c211e3b2ad.html', + 'only_matching': True, + }] - def _find_videos_in_page(self, webpage): - videos = re.findall( - r'
  • ]+href="(?Phttps?://v\.youku\.com/[^"]+)"[^>]+title="(?P[^"]+)"', webpage) - return [ - self.url_result(video_url, YoukuIE.ie_key(), title) - for video_url, title in videos] + def _extract_entries(self, playlist_data_url, show_id, note, query): + query['callback'] = 'cb' + playlist_data = self._download_json( + playlist_data_url, show_id, query=query, note=note, + transform_source=lambda s: js_to_json(strip_jsonp(s))).get('html') + if playlist_data is None: + return [None, None] + drama_list = (get_element_by_class('p-drama-grid', playlist_data) + or get_element_by_class('p-drama-half-row', playlist_data)) + if drama_list is None: + raise ExtractorError('No episodes found') + video_urls = re.findall(r'<a[^>]+href="([^"]+)"', drama_list) + return playlist_data, [ + self.url_result(self._proto_relative_url(video_url, 'http:'), YoukuIE.ie_key()) + for video_url in video_urls] def _real_extract(self, url): show_id = self._match_id(url) webpage = self._download_webpage(url, show_id) - entries = self._find_videos_in_page(webpage) - - playlist_title = self._html_search_regex( - r'<span[^>]+class="name">([^<]+)</span>', webpage, 'playlist title', fatal=False) - detail_div = get_element_by_attribute('class', 'detail', webpage) or '' - playlist_description = self._html_search_regex( - r'<span[^>]+style="display:none"[^>]*>([^<]+)</span>', - detail_div, 'playlist description', fatal=False) - - for idx in itertools.count(1): - episodes_page = self._download_webpage( - 'http://www.youku.com/show_episode/id_%s.html' % show_id, - show_id, query={'divid': 'reload_%d' % (idx * self._PAGE_SIZE + 1)}, - note='Downloading episodes page %d' % idx) - new_entries = self._find_videos_in_page(episodes_page) - entries.extend(new_entries) - if len(new_entries) < self._PAGE_SIZE: - break - - return self.playlist_result(entries, show_id, playlist_title, playlist_description) + entries = [] + page_config = self._parse_json(self._search_regex( + r'var\s+PageConfig\s*=\s*({.+});', webpage, 'page config'), + show_id, transform_source=js_to_json) + first_page, initial_entries = self._extract_entries( + 'http://list.youku.com/show/module', show_id, + note='Downloading initial playlist data page', + query={ + 'id': page_config['showid'], + 'tab': 'showInfo', + }) + first_page_reload_id = self._html_search_regex( + r'<div[^>]+id="(reload_\d+)', first_page, 'first page reload id') + # The first reload_id has the same items as first_page + reload_ids = re.findall('<li[^>]+data-id="([^"]+)">', first_page) + entries.extend(initial_entries) + for idx, reload_id in enumerate(reload_ids): + if reload_id == first_page_reload_id: + continue + _, new_entries = self._extract_entries( + 'http://list.youku.com/show/episode', show_id, + note='Downloading playlist data page %d' % (idx + 1), + query={ + 'id': page_config['showid'], + 'stage': reload_id, + }) + if new_entries is not None: + entries.extend(new_entries) + desc = self._html_search_meta('description', webpage, fatal=False) + playlist_title = desc.split(',')[0] if desc else None + detail_li = get_element_by_class('p-intro', webpage) + playlist_description = get_element_by_class( + 'intro-more', detail_li) if detail_li else None + + return self.playlist_result( + entries, show_id, playlist_title, playlist_description)