old_report_warning(message)
self.report_warning = types.MethodType(report_warning, self)
-def gettestcases():
+
+def gettestcases(include_onlymatching=False):
for ie in youtube_dl.extractor.gen_extractors():
t = getattr(ie, '_TEST', None)
if t:
- t['name'] = type(ie).__name__[:-len('IE')]
- yield t
- for t in getattr(ie, '_TESTS', []):
+ assert not hasattr(ie, '_TESTS'), \
+ '%s has _TEST and _TESTS' % type(ie).__name__
+ tests = [t]
+ else:
+ tests = getattr(ie, '_TESTS', [])
+ for t in tests:
+ if not include_onlymatching and getattr(t, 'only_matching', False):
+ continue
t['name'] = type(ie).__name__[:-len('IE')]
yield t
self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
def test_justin_tv_channelid_matching(self):
- self.assertTrue(JustinTVIE.suitable(u"justin.tv/vanillatv"))
- self.assertTrue(JustinTVIE.suitable(u"twitch.tv/vanillatv"))
- self.assertTrue(JustinTVIE.suitable(u"www.justin.tv/vanillatv"))
- self.assertTrue(JustinTVIE.suitable(u"www.twitch.tv/vanillatv"))
- self.assertTrue(JustinTVIE.suitable(u"http://www.justin.tv/vanillatv"))
- self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/vanillatv"))
- self.assertTrue(JustinTVIE.suitable(u"http://www.justin.tv/vanillatv/"))
- self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/vanillatv/"))
+ self.assertTrue(JustinTVIE.suitable('justin.tv/vanillatv'))
+ self.assertTrue(JustinTVIE.suitable('twitch.tv/vanillatv'))
+ self.assertTrue(JustinTVIE.suitable('www.justin.tv/vanillatv'))
+ self.assertTrue(JustinTVIE.suitable('www.twitch.tv/vanillatv'))
+ self.assertTrue(JustinTVIE.suitable('http://www.justin.tv/vanillatv'))
+ self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv'))
+ self.assertTrue(JustinTVIE.suitable('http://www.justin.tv/vanillatv/'))
+ self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv/'))
def test_justintv_videoid_matching(self):
- self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/vanillatv/b/328087483"))
+ self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv/b/328087483'))
def test_justin_tv_chapterid_matching(self):
- self.assertTrue(JustinTVIE.suitable(u"http://www.twitch.tv/tsm_theoddone/c/2349361"))
+ self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/tsm_theoddone/c/2349361'))
def test_youtube_extract(self):
assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)
def test_no_duplicates(self):
ies = gen_extractors()
- for tc in gettestcases():
+ for tc in gettestcases(include_onlymatching=True):
url = tc['url']
for ie in ies:
if type(ie).__name__ in ('GenericIE', tc['name'] + 'IE'):
'https://screen.yahoo.com/smartwatches-latest-wearable-gadgets-163745379-cbs.html',
['Yahoo'])
+
if __name__ == '__main__':
unittest.main()
self.assertIsPlaylist(result)
self.assertEqual(result['id'], 'dezhurnyi_angel')
self.assertEqual(result['title'], 'Дежурный ангел (2010 - 2012)')
- self.assertTrue(len(result['entries']) >= 36)
-
+ self.assertTrue(len(result['entries']) >= 23)
+
def test_ivi_compilation_season(self):
dl = FakeYDL()
ie = IviCompilationIE(dl)
self.assertIsPlaylist(result)
self.assertEqual(result['id'], 'dezhurnyi_angel/season2')
self.assertEqual(result['title'], 'Дежурный ангел (2010 - 2012) 2 сезон')
- self.assertTrue(len(result['entries']) >= 20)
+ self.assertTrue(len(result['entries']) >= 7)
def test_imdb_list(self):
dl = FakeYDL()
from .nhl import NHLIE, NHLVideocenterIE
from .niconico import NiconicoIE
from .ninegag import NineGagIE
+from .noco import NocoIE
from .normalboots import NormalbootsIE
from .novamov import NovaMovIE
from .nowness import NownessIE
from ..utils import (
compat_urllib_request,
compat_str,
- get_element_by_id,
orderedSet,
str_to_int,
int_or_none,
-import os
+from __future__ import unicode_literals
+
import re
from .common import InfoExtractor
compat_urllib_parse,
)
+
class ExtremeTubeIE(InfoExtractor):
- _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>extremetube\.com/video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)'
- _TEST = {
- u'url': u'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
- u'file': u'652431.mp4',
- u'md5': u'1fb9228f5e3332ec8c057d6ac36f33e0',
- u'info_dict': {
- u"title": u"Music Video 14 british euro brit european cumshots swallow",
- u"uploader": u"unknown",
- u"age_limit": 18,
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>extremetube\.com/.*?video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)'
+ _TESTS = [{
+ 'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
+ 'md5': '1fb9228f5e3332ec8c057d6ac36f33e0',
+ 'info_dict': {
+ 'id': '652431',
+ 'ext': 'mp4',
+ 'title': 'Music Video 14 british euro brit european cumshots swallow',
+ 'uploader': 'unknown',
+ 'age_limit': 18,
}
- }
+ }, {
+ 'url': 'http://www.extremetube.com/gay/video/abcde-1234',
+ 'only_matching': True,
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id)
- video_title = self._html_search_regex(r'<h1 [^>]*?title="([^"]+)"[^>]*>\1<', webpage, u'title')
- uploader = self._html_search_regex(r'>Posted by:(?=<)(?:\s|<[^>]*>)*(.+?)\|', webpage, u'uploader', fatal=False)
- video_url = compat_urllib_parse.unquote(self._html_search_regex(r'video_url=(.+?)&', webpage, u'video_url'))
+ video_title = self._html_search_regex(
+ r'<h1 [^>]*?title="([^"]+)"[^>]*>\1<', webpage, 'title')
+ uploader = self._html_search_regex(
+ r'>Posted by:(?=<)(?:\s|<[^>]*>)*(.+?)\|', webpage, 'uploader',
+ fatal=False)
+ video_url = compat_urllib_parse.unquote(self._html_search_regex(
+ r'video_url=(.+?)&', webpage, 'video_url'))
path = compat_urllib_parse_urlparse(video_url).path
- extension = os.path.splitext(path)[1][1:]
format = path.split('/')[5].split('_')[:2]
format = "-".join(format)
'title': video_title,
'uploader': uploader,
'url': video_url,
- 'ext': extension,
'format': format,
'format_id': format,
'age_limit': 18,
_DESCRIPTION_RE = r'<div id="vid_summary">(.+?)</div>'
- _TEST = {
+ _TESTS = [{
'url': 'http://gamevideos.1up.com/video/id/34976',
'md5': '68a54ce4ebc772e4b71e3123d413163d',
'info_dict': {
'title': 'Sniper Elite V2 - Trailer',
'description': 'md5:5d289b722f5a6d940ca3136e9dae89cf',
}
- }
-
- # Override IGN tests
- _TESTS = []
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
--- /dev/null
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ unified_strdate,
+ compat_str,
+)
+
+
+class NocoIE(InfoExtractor):
+ _VALID_URL = r'http://(?:(?:www\.)?noco\.tv/emission/|player\.noco\.tv/\?idvideo=)(?P<id>\d+)'
+
+ _TEST = {
+ 'url': 'http://noco.tv/emission/11538/nolife/ami-ami-idol-hello-france/',
+ 'md5': '0a993f0058ddbcd902630b2047ef710e',
+ 'info_dict': {
+ 'id': '11538',
+ 'ext': 'mp4',
+ 'title': 'Ami Ami Idol - Hello! France',
+ 'description': 'md5:4eaab46ab68fa4197a317a88a53d3b86',
+ 'upload_date': '20140412',
+ 'uploader': 'Nolife',
+ 'uploader_id': 'NOL',
+ 'duration': 2851.2,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ medias = self._download_json(
+ 'http://api.noco.tv/1.0/video/medias/%s' % video_id, video_id, 'Downloading video JSON')
+
+ formats = []
+
+ for fmt in medias['fr']['video_list']['default']['quality_list']:
+ format_id = fmt['quality_key']
+
+ file = self._download_json(
+ 'http://api.noco.tv/1.0/video/file/%s/fr/%s' % (format_id.lower(), video_id),
+ video_id, 'Downloading %s video JSON' % format_id)
+
+ file_url = file['file']
+ if not file_url:
+ continue
+
+ if file_url == 'forbidden':
+ raise ExtractorError(
+ '%s returned error: %s - %s' % (
+ self.IE_NAME, file['popmessage']['title'], file['popmessage']['message']),
+ expected=True)
+
+ formats.append({
+ 'url': file_url,
+ 'format_id': format_id,
+ 'width': fmt['res_width'],
+ 'height': fmt['res_lines'],
+ 'abr': fmt['audiobitrate'],
+ 'vbr': fmt['videobitrate'],
+ 'filesize': fmt['filesize'],
+ 'format_note': fmt['quality_name'],
+ 'preference': fmt['priority'],
+ })
+
+ self._sort_formats(formats)
+
+ show = self._download_json(
+ 'http://api.noco.tv/1.0/shows/show/%s' % video_id, video_id, 'Downloading show JSON')[0]
+
+ upload_date = unified_strdate(show['indexed'])
+ uploader = show['partner_name']
+ uploader_id = show['partner_key']
+ duration = show['duration_ms'] / 1000.0
+ thumbnail = show['screenshot']
+
+ episode = show.get('show_TT') or show.get('show_OT')
+ family = show.get('family_TT') or show.get('family_OT')
+ episode_number = show.get('episode_number')
+
+ title = ''
+ if family:
+ title += family
+ if episode_number:
+ title += ' #' + compat_str(episode_number)
+ if episode:
+ title += ' - ' + episode
+
+ description = show.get('show_resume') or show.get('family_resume')
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'upload_date': upload_date,
+ 'uploader': uploader,
+ 'uploader_id': uploader_id,
+ 'duration': duration,
+ 'formats': formats,
+ }
\ No newline at end of file
'http://rutube.ru/api/video/%s/?format=json' % video_id,
video_id, 'Downloading video JSON')
- trackinfo = self._download_json(
- 'http://rutube.ru/api/play/trackinfo/%s/?format=json' % video_id,
- video_id, 'Downloading trackinfo JSON')
-
# Some videos don't have the author field
- author = trackinfo.get('author') or {}
- m3u8_url = trackinfo['video_balancer'].get('m3u8')
+ author = video.get('author') or {}
+
+ options = self._download_json(
+ 'http://rutube.ru/api/play/options/%s/?format=json' %video_id,
+ video_id, 'Downloading options JSON')
+
+ m3u8_url = options['video_balancer'].get('m3u8')
if m3u8_url is None:
raise ExtractorError('Couldn\'t find m3u8 manifest url')
+from __future__ import unicode_literals
+
import re
from .common import InfoExtractor
class SteamIE(InfoExtractor):
- _VALID_URL = r"""http://store\.steampowered\.com/
- (agecheck/)?
- (?P<urltype>video|app)/ #If the page is only for videos or for a game
- (?P<gameID>\d+)/?
- (?P<videoID>\d*)(?P<extra>\??) #For urltype == video we sometimes get the videoID
- """
+ _VALID_URL = r"""(?x)
+ https?://store\.steampowered\.com/
+ (agecheck/)?
+ (?P<urltype>video|app)/ #If the page is only for videos or for a game
+ (?P<gameID>\d+)/?
+ (?P<videoID>\d*)(?P<extra>\??) # For urltype == video we sometimes get the videoID
+ |
+ https?://(?:www\.)?steamcommunity\.com/sharedfiles/filedetails/\?id=(?P<fileID>[0-9]+)
+ """
_VIDEO_PAGE_TEMPLATE = 'http://store.steampowered.com/video/%s/'
_AGECHECK_TEMPLATE = 'http://store.steampowered.com/agecheck/video/%s/?snr=1_agecheck_agecheck__age-gate&ageDay=1&ageMonth=January&ageYear=1970'
- _TEST = {
- u"url": u"http://store.steampowered.com/video/105600/",
- u"playlist": [
+ _TESTS = [{
+ "url": "http://store.steampowered.com/video/105600/",
+ "playlist": [
{
- u"file": u"81300.flv",
- u"md5": u"f870007cee7065d7c76b88f0a45ecc07",
- u"info_dict": {
- u"title": u"Terraria 1.1 Trailer",
- u'playlist_index': 1,
+ "md5": "f870007cee7065d7c76b88f0a45ecc07",
+ "info_dict": {
+ 'id': '81300',
+ 'ext': 'flv',
+ "title": "Terraria 1.1 Trailer",
+ 'playlist_index': 1,
}
},
{
- u"file": u"80859.flv",
- u"md5": u"61aaf31a5c5c3041afb58fb83cbb5751",
- u"info_dict": {
- u"title": u"Terraria Trailer",
- u'playlist_index': 2,
+ "md5": "61aaf31a5c5c3041afb58fb83cbb5751",
+ "info_dict": {
+ 'id': '80859',
+ 'ext': 'flv',
+ "title": "Terraria Trailer",
+ 'playlist_index': 2,
}
}
- ]
- }
-
-
- @classmethod
- def suitable(cls, url):
- """Receives a URL and returns True if suitable for this IE."""
- return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
+ ],
+ 'params': {
+ 'playlistend': 2,
+ }
+ }, {
+ 'url': 'http://steamcommunity.com/sharedfiles/filedetails/?id=242472205',
+ 'info_dict': {
+ 'id': 'WB5DvDOOvAY',
+ 'ext': 'mp4',
+ 'upload_date': '20140329',
+ 'title': 'FRONTIERS - Final Greenlight Trailer',
+ 'description': "The final trailer for the Steam Greenlight launch. Hooray, progress! Here's the official Greenlight page: http://steamcommunity.com/sharedfiles/filedetails/?id=242472205",
+ 'uploader': 'AAD Productions',
+ 'uploader_id': 'AtomicAgeDogGames',
+ }
+ }]
def _real_extract(self, url):
- m = re.match(self._VALID_URL, url, re.VERBOSE)
- gameID = m.group('gameID')
-
- videourl = self._VIDEO_PAGE_TEMPLATE % gameID
- webpage = self._download_webpage(videourl, gameID)
+ m = re.match(self._VALID_URL, url)
+ fileID = m.group('fileID')
+ if fileID:
+ videourl = url
+ playlist_id = fileID
+ else:
+ gameID = m.group('gameID')
+ playlist_id = gameID
+ videourl = self._VIDEO_PAGE_TEMPLATE % playlist_id
+ webpage = self._download_webpage(videourl, playlist_id)
if re.search('<h2>Please enter your birth date to continue:</h2>', webpage) is not None:
- videourl = self._AGECHECK_TEMPLATE % gameID
+ videourl = self._AGECHECK_TEMPLATE % playlist_id
self.report_age_confirmation()
- webpage = self._download_webpage(videourl, gameID)
+ webpage = self._download_webpage(videourl, playlist_id)
+
+ if fileID:
+ playlist_title = self._html_search_regex(
+ r'<div class="workshopItemTitle">(.+)</div>', webpage, 'title')
+ mweb = re.finditer(r'''(?x)
+ 'movie_(?P<videoID>[0-9]+)':\s*\{\s*
+ YOUTUBE_VIDEO_ID:\s*"(?P<youtube_id>[^"]+)",
+ ''', webpage)
+ videos = [{
+ '_type': 'url',
+ 'url': vid.group('youtube_id'),
+ 'ie_key': 'Youtube',
+ } for vid in mweb]
+ else:
+ playlist_title = self._html_search_regex(
+ r'<h2 class="pageheader">(.*?)</h2>', webpage, 'game title')
+
+ mweb = re.finditer(r'''(?x)
+ 'movie_(?P<videoID>[0-9]+)':\s*\{\s*
+ FILENAME:\s*"(?P<videoURL>[\w:/\.\?=]+)"
+ (,\s*MOVIE_NAME:\s*\"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\},
+ ''', webpage)
+ titles = re.finditer(
+ r'<span class="title">(?P<videoName>.+?)</span>', webpage)
+ thumbs = re.finditer(
+ r'<img class="movie_thumb" src="(?P<thumbnail>.+?)">', webpage)
+ videos = []
- self.report_extraction(gameID)
- game_title = self._html_search_regex(r'<h2 class="pageheader">(.*?)</h2>',
- webpage, 'game title')
+ for vid, vtitle, thumb in zip(mweb, titles, thumbs):
+ video_id = vid.group('videoID')
+ title = vtitle.group('videoName')
+ video_url = vid.group('videoURL')
+ video_thumb = thumb.group('thumbnail')
+ if not video_url:
+ raise ExtractorError('Cannot find video url for %s' % video_id)
+ videos.append({
+ 'id': video_id,
+ 'url': video_url,
+ 'ext': 'flv',
+ 'title': unescapeHTML(title),
+ 'thumbnail': video_thumb
+ })
+ if not videos:
+ raise ExtractorError('Could not find any videos')
- urlRE = r"'movie_(?P<videoID>\d+)': \{\s*FILENAME: \"(?P<videoURL>[\w:/\.\?=]+)\"(,\s*MOVIE_NAME: \"(?P<videoName>[\w:/\.\?=\+-]+)\")?\s*\},"
- mweb = re.finditer(urlRE, webpage)
- namesRE = r'<span class="title">(?P<videoName>.+?)</span>'
- titles = re.finditer(namesRE, webpage)
- thumbsRE = r'<img class="movie_thumb" src="(?P<thumbnail>.+?)">'
- thumbs = re.finditer(thumbsRE, webpage)
- videos = []
- for vid,vtitle,thumb in zip(mweb,titles,thumbs):
- video_id = vid.group('videoID')
- title = vtitle.group('videoName')
- video_url = vid.group('videoURL')
- video_thumb = thumb.group('thumbnail')
- if not video_url:
- raise ExtractorError(u'Cannot find video url for %s' % video_id)
- info = {
- 'id':video_id,
- 'url':video_url,
- 'ext': 'flv',
- 'title': unescapeHTML(title),
- 'thumbnail': video_thumb
- }
- videos.append(info)
- return [self.playlist_result(videos, gameID, game_title)]
+ return self.playlist_result(videos, playlist_id, playlist_title)
import re
from .common import InfoExtractor
-from ..utils import (
- ExtractorError,
-)
class TeamcocoIE(InfoExtractor):
IE_NAME = 'yahoo:news'
_VALID_URL = r'http://news\.yahoo\.com/video/.*?-(?P<id>\d*?)\.html'
- _TEST = {
+ _TESTS = [{
'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
'md5': '67010fdf3a08d290e060a4dd96baa07b',
'info_dict': {
'title': 'China Moses Is Crazy About the Blues',
'description': 'md5:9900ab8cd5808175c7b3fe55b979bed0',
},
- }
-
- # Overwrite YahooIE properties we don't want
- _TESTS = []
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
-__version__ = '2014.04.13'
+__version__ = '2014.04.21'
projects / youtube-dl / commitdiff