From: Sergey M. Date: Fri, 8 May 2015 17:01:50 +0000 (+0500) Subject: Merge pull request #5641 from dstftw/preserve-best-for-stdout-outtmpl X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=commitdiff_plain;h=d47e980d0de4aeeca757433365836b61c2ee9f9f;hp=3799834dcff27ffeea66e3ec96166f8da8fa73ab;p=youtube-dl Merge pull request #5641 from dstftw/preserve-best-for-stdout-outtmpl [YoutubeDL] Do not force bestvideo+bestaudio when outtmpl is stdout --- diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py index 6865b5e2f..7d19bb808 100644 --- a/youtube_dl/downloader/rtmp.py +++ b/youtube_dl/downloader/rtmp.py @@ -131,7 +131,7 @@ class RtmpFD(FileDownloader): if play_path is not None: basic_args += ['--playpath', play_path] if tc_url is not None: - basic_args += ['--tcUrl', url] + basic_args += ['--tcUrl', tc_url] if test: basic_args += ['--stop', '1'] if flash_version is not None: diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 5dfa781f8..96cf28efe 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -587,7 +587,11 @@ from .veoh import VeohIE from .vessel import VesselIE from .vesti import VestiIE from .vevo import VevoIE -from .vgtv import VGTVIE +from .vgtv import ( + BTArticleIE, + BTVestlendingenIE, + VGTVIE, +) from .vh1 import VH1IE from .vice import ViceIE from .viddler import ViddlerIE @@ -650,6 +654,7 @@ from .xboxclips import XboxClipsIE from .xhamster import XHamsterIE from .xminus import XMinusIE from .xnxx import XNXXIE +from .xstream import XstreamIE from .xvideos import XVideosIE from .xtube import XTubeUserIE, XTubeIE from .xuite import XuiteIE diff --git a/youtube_dl/extractor/aftenposten.py b/youtube_dl/extractor/aftenposten.py index e15c015fb..0c00acfb5 100644 --- a/youtube_dl/extractor/aftenposten.py +++ b/youtube_dl/extractor/aftenposten.py @@ -1,21 +1,11 @@ # coding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor -from ..utils import ( - int_or_none, - parse_iso8601, - xpath_with_ns, - xpath_text, - find_xpath_attr, -) class AftenpostenIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/(?:#!/)?video/(?P\d+)' - _TEST = { 'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more', 'md5': 'fd828cd29774a729bf4d4425fe192972', @@ -30,69 +20,4 @@ class AftenpostenIE(InfoExtractor): } def _real_extract(self, url): - video_id = self._match_id(url) - - data = self._download_xml( - 'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=%s' % video_id, video_id) - - NS_MAP = { - 'atom': 'http://www.w3.org/2005/Atom', - 'xt': 'http://xstream.dk/', - 'media': 'http://search.yahoo.com/mrss/', - } - - entry = data.find(xpath_with_ns('./atom:entry', NS_MAP)) - - title = xpath_text( - entry, xpath_with_ns('./atom:title', NS_MAP), 'title') - description = xpath_text( - entry, xpath_with_ns('./atom:summary', NS_MAP), 'description') - timestamp = parse_iso8601(xpath_text( - entry, xpath_with_ns('./atom:published', NS_MAP), 'upload date')) - - formats = [] - media_group = entry.find(xpath_with_ns('./media:group', NS_MAP)) - for media_content in media_group.findall(xpath_with_ns('./media:content', NS_MAP)): - media_url = media_content.get('url') - if not media_url: - continue - tbr = int_or_none(media_content.get('bitrate')) - mobj = re.search(r'^(?Prtmp://[^/]+/(?P[^/]+))/(?P.+)$', media_url) - if mobj: - formats.append({ - 'url': mobj.group('url'), - 'play_path': 'mp4:%s' % mobj.group('playpath'), - 'app': mobj.group('app'), - 'ext': 'flv', - 'tbr': tbr, - 'format_id': 'rtmp-%d' % tbr, - }) - else: - formats.append({ - 'url': media_url, - 'tbr': tbr, - }) - self._sort_formats(formats) - - link = find_xpath_attr( - entry, xpath_with_ns('./atom:link', NS_MAP), 'rel', 'original') - if link is not None: - formats.append({ - 'url': link.get('href'), - 'format_id': link.get('rel'), - }) - - thumbnails = [{ - 'url': splash.get('url'), - 'width': int_or_none(splash.get('width')), - 'height': int_or_none(splash.get('height')), - } for splash in media_group.findall(xpath_with_ns('./xt:splash', NS_MAP))] - - return { - 'id': video_id, - 'title': title, - 'description': description, - 'timestamp': timestamp, - 'formats': formats, - 'thumbnails': thumbnails, - } + return self.url_result('xstream:ap:%s' % self._match_id(url), 'Xstream') diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py index d2abd4d77..26b934543 100644 --- a/youtube_dl/extractor/bet.py +++ b/youtube_dl/extractor/bet.py @@ -16,11 +16,11 @@ class BetIE(InfoExtractor): { 'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html', 'info_dict': { - 'id': '740ab250-bb94-4a8a-8787-fe0de7c74471', + 'id': 'news/national/2014/a-conversation-with-president-obama', 'display_id': 'in-bet-exclusive-obama-talks-race-and-racism', 'ext': 'flv', - 'title': 'BET News Presents: A Conversation With President Obama', - 'description': 'md5:5a88d8ae912c1b33e090290af7ec33c6', + 'title': 'A Conversation With President Obama', + 'description': 'md5:699d0652a350cf3e491cd15cc745b5da', 'duration': 1534, 'timestamp': 1418075340, 'upload_date': '20141208', @@ -35,7 +35,7 @@ class BetIE(InfoExtractor): { 'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html', 'info_dict': { - 'id': 'bcd1b1df-673a-42cf-8d01-b282db608f2d', + 'id': 'news/national/2014/justice-for-ferguson-a-community-reacts', 'display_id': 'justice-for-ferguson-a-community-reacts', 'ext': 'flv', 'title': 'Justice for Ferguson: A Community Reacts', @@ -61,6 +61,9 @@ class BetIE(InfoExtractor): [r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"], webpage, 'media URL')) + video_id = self._search_regex( + r'/video/(.*)/_jcr_content/', media_url, 'video id') + mrss = self._download_xml(media_url, display_id) item = mrss.find('./channel/item') @@ -75,8 +78,6 @@ class BetIE(InfoExtractor): description = xpath_text( item, './description', 'description', fatal=False) - video_id = xpath_text(item, './guid', 'video id', fatal=False) - timestamp = parse_iso8601(xpath_text( item, xpath_with_ns('./dc:date', NS_MAP), 'upload date', fatal=False)) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index d09e85665..cd7c47d6d 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -1453,7 +1453,7 @@ class GenericIE(InfoExtractor): if refresh_header: found = re.search(REDIRECT_REGEX, refresh_header) if found: - new_url = found.group(1) + new_url = compat_urlparse.urljoin(url, found.group(1)) self.report_following_redirect(new_url) return { '_type': 'url', diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py index ee9ff73bf..4e054fb53 100644 --- a/youtube_dl/extractor/mlb.py +++ b/youtube_dl/extractor/mlb.py @@ -10,7 +10,21 @@ from ..utils import ( class MLBIE(InfoExtractor): - _VALID_URL = r'https?://m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/(?:embed|m-internal-embed)\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?Pn?\d+)' + _VALID_URL = r'''(?x) + https?:// + m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/ + (?: + (?: + (?:.*?/)?video/(?:topic/[\da-z_-]+/)?v| + (?: + shared/video/embed/(?:embed|m-internal-embed)\.html| + [^/]+/video/play\.jsp + )\?.*?\bcontent_id= + ) + (?Pn?\d+)| + (?:[^/]+/)*(?P[^/]+) + ) + ''' _TESTS = [ { 'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea', @@ -95,6 +109,12 @@ class MLBIE(InfoExtractor): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') + if not video_id: + video_path = mobj.group('path') + webpage = self._download_webpage(url, video_path) + video_id = self._search_regex( + r'data-videoid="(\d+)"', webpage, 'video id') + detail = self._download_xml( 'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml' % (video_id[-3], video_id[-2], video_id[-1], video_id), video_id) diff --git a/youtube_dl/extractor/netzkino.py b/youtube_dl/extractor/netzkino.py index bc17e20aa..0d165a82a 100644 --- a/youtube_dl/extractor/netzkino.py +++ b/youtube_dl/extractor/netzkino.py @@ -49,7 +49,7 @@ class NetzkinoIE(InfoExtractor): 'http://www.netzkino.de/beta/dist/production.min.js', video_id, note='Downloading player code') avo_js = self._search_regex( - r'window\.avoCore\s*=.*?urlTemplate:\s*(\{.*?"\})', + r'var urlTemplate=(\{.*?"\})', production_js, 'URL templates') templates = self._parse_json( avo_js, video_id, transform_source=js_to_json) diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py index 407465998..279b18386 100644 --- a/youtube_dl/extractor/nhl.py +++ b/youtube_dl/extractor/nhl.py @@ -21,6 +21,9 @@ class NHLBaseInfoExtractor(InfoExtractor): return json_string.replace('\\\'', '\'') def _real_extract_video(self, video_id): + vid_parts = video_id.split(',') + if len(vid_parts) == 3: + video_id = '%s0%s%s-X-h' % (vid_parts[0][:4], vid_parts[1], vid_parts[2].rjust(4, '0')) json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id data = self._download_json( json_url, video_id, transform_source=self._fix_json) @@ -47,7 +50,7 @@ class NHLBaseInfoExtractor(InfoExtractor): video_url = initial_video_url join = compat_urlparse.urljoin - return { + ret = { 'id': video_id, 'title': info['name'], 'url': video_url, @@ -56,11 +59,20 @@ class NHLBaseInfoExtractor(InfoExtractor): 'thumbnail': join(join(video_url, '/u/'), info['bigImage']), 'upload_date': unified_strdate(info['releaseDate'].split('.')[0]), } + if video_url.startswith('rtmp:'): + mobj = re.match(r'(?Prtmp://[^/]+/(?P[a-z0-9/]+))/(?Pmp4:.*)', video_url) + ret.update({ + 'tc_url': mobj.group('tc_url'), + 'play_path': mobj.group('play_path'), + 'app': mobj.group('app'), + 'no_resume': True, + }) + return ret class NHLIE(NHLBaseInfoExtractor): IE_NAME = 'nhl.com' - _VALID_URL = r'https?://video(?P\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)id=(?P[-0-9a-zA-Z]+)' + _VALID_URL = r'https?://video(?P\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)(?:id|hlg)=(?P[-0-9a-zA-Z,]+)' _TESTS = [{ 'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614', @@ -101,6 +113,29 @@ class NHLIE(NHLBaseInfoExtractor): }, { 'url': 'http://video.nhl.com/videocenter/?id=736722', 'only_matching': True, + }, { + 'url': 'http://video.nhl.com/videocenter/console?hlg=20142015,2,299&lang=en', + 'md5': '076fcb88c255154aacbf0a7accc3f340', + 'info_dict': { + 'id': '2014020299-X-h', + 'ext': 'mp4', + 'title': 'Penguins at Islanders / Game Highlights', + 'description': 'Home broadcast - Pittsburgh Penguins at New York Islanders - November 22, 2014', + 'duration': 268, + 'upload_date': '20141122', + } + }, { + 'url': 'http://video.oilers.nhl.com/videocenter/console?id=691469&catid=4', + 'info_dict': { + 'id': '691469', + 'ext': 'mp4', + 'title': 'RAW | Craig MacTavish Full Press Conference', + 'description': 'Oilers GM Craig MacTavish addresses the media at Rexall Place on Friday.', + 'upload_date': '20141205', + }, + 'params': { + 'skip_download': True, # Requires rtmpdump + } }] def _real_extract(self, url): diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py index 0b049274a..c0e6d643d 100644 --- a/youtube_dl/extractor/ooyala.py +++ b/youtube_dl/extractor/ooyala.py @@ -121,7 +121,7 @@ class OoyalaIE(InfoExtractor): 'abr': int_or_none(stream.get('audio_bitrate')), 'vbr': int_or_none(stream.get('video_bitrate')), }) - if len(formats): + if formats: return { 'id': embedCode, 'formats': formats, diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py index 6215f0642..3c8d2a943 100644 --- a/youtube_dl/extractor/vessel.py +++ b/youtube_dl/extractor/vessel.py @@ -38,9 +38,13 @@ class VesselIE(InfoExtractor): return req @staticmethod - def find_assets(data, asset_type): + def find_assets(data, asset_type, asset_id=None): for asset in data.get('assets', []): - if asset.get('type') == asset_type: + if not asset.get('type') == asset_type: + continue + elif asset_id is not None and not asset.get('id') == asset_id: + continue + else: yield asset def _check_access_rights(self, data): @@ -82,11 +86,13 @@ class VesselIE(InfoExtractor): req = VesselIE.make_json_request( self._API_URL_TEMPLATE % asset_id, {'client': 'web'}) data = self._download_json(req, video_id) + video_asset_id = data.get('main_video_asset') self._check_access_rights(data) try: - video_asset = next(VesselIE.find_assets(data, 'video')) + video_asset = next( + VesselIE.find_assets(data, 'video', asset_id=video_asset_id)) except StopIteration: raise ExtractorError('No video assets found') diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py index 69dc9a759..db7a4bdb1 100644 --- a/youtube_dl/extractor/vgtv.py +++ b/youtube_dl/extractor/vgtv.py @@ -8,7 +8,19 @@ from ..utils import float_or_none class VGTVIE(InfoExtractor): - _VALID_URL = r'http://(?:www\.)?vgtv\.no/#!/[^/]+/(?P[0-9]+)' + IE_DESC = 'VGTV and BTTV' + _VALID_URL = r'''(?x) + (?: + vgtv:| + http://(?:www\.)? + ) + (?Pvgtv|bt) + (?: + :| + \.no/(?:tv/)?#!/(?:video|live)/ + ) + (?P[0-9]+) + ''' _TESTS = [ { # streamType: vod @@ -64,12 +76,25 @@ class VGTVIE(InfoExtractor): 'skip_download': True, }, }, + { + 'url': 'http://www.bt.no/tv/#!/video/100250/norling-dette-er-forskjellen-paa-1-divisjon-og-eliteserien', + 'only_matching': True, + }, ] def _real_extract(self, url): - video_id = self._match_id(url) + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + host = mobj.group('host') + + HOST_WEBSITES = { + 'vgtv': 'vgtv', + 'bt': 'bttv', + } + data = self._download_json( - 'http://svp.vg.no/svp/api/v1/vgtv/assets/%s?appName=vgtv-website' % video_id, + 'http://svp.vg.no/svp/api/v1/%s/assets/%s?appName=%s-website' + % (host, video_id, HOST_WEBSITES[host]), video_id, 'Downloading media JSON') streams = data['streamUrls'] @@ -115,3 +140,53 @@ class VGTVIE(InfoExtractor): 'view_count': data['displays'], 'formats': formats, } + + +class BTArticleIE(InfoExtractor): + IE_NAME = 'bt:article' + IE_DESC = 'Bergens Tidende Articles' + _VALID_URL = 'http://(?:www\.)?bt\.no/(?:[^/]+/)+(?P[^/]+)-\d+\.html' + _TEST = { + 'url': 'http://www.bt.no/nyheter/lokalt/Kjemper-for-internatet-1788214.html', + 'md5': 'd055e8ee918ef2844745fcfd1a4175fb', + 'info_dict': { + 'id': '23199', + 'ext': 'mp4', + 'title': 'Alrekstad internat', + 'description': 'md5:dc81a9056c874fedb62fc48a300dac58', + 'thumbnail': 're:^https?://.*\.jpg', + 'duration': 191, + 'timestamp': 1289991323, + 'upload_date': '20101117', + 'view_count': int, + }, + } + + def _real_extract(self, url): + webpage = self._download_webpage(url, self._match_id(url)) + video_id = self._search_regex( + r'SVP\.Player\.load\(\s*(\d+)', webpage, 'video id') + return self.url_result('vgtv:bt:%s' % video_id, 'VGTV') + + +class BTVestlendingenIE(InfoExtractor): + IE_NAME = 'bt:vestlendingen' + IE_DESC = 'Bergens Tidende - Vestlendingen' + _VALID_URL = 'http://(?:www\.)?bt\.no/spesial/vestlendingen/#!/(?P\d+)' + _TEST = { + 'url': 'http://www.bt.no/spesial/vestlendingen/#!/86588', + 'md5': 'd7d17e3337dc80de6d3a540aefbe441b', + 'info_dict': { + 'id': '86588', + 'ext': 'mov', + 'title': 'Otto Wollertsen', + 'description': 'Vestlendingen Otto Fredrik Wollertsen', + 'timestamp': 1430473209, + 'upload_date': '20150501', + }, + } + + def _real_extract(self, url): + return self.url_result('xstream:btno:%s' % self._match_id(url), 'Xstream') + + diff --git a/youtube_dl/extractor/xstream.py b/youtube_dl/extractor/xstream.py new file mode 100644 index 000000000..71584c291 --- /dev/null +++ b/youtube_dl/extractor/xstream.py @@ -0,0 +1,115 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + int_or_none, + parse_iso8601, + xpath_with_ns, + xpath_text, + find_xpath_attr, +) + + +class XstreamIE(InfoExtractor): + _VALID_URL = r'''(?x) + (?: + xstream:| + https?://frontend\.xstream\.(?:dk|net)/ + ) + (?P[^/]+) + (?: + :| + /feed/video/\?.*?\bid= + ) + (?P\d+) + ''' + _TESTS = [{ + 'url': 'http://frontend.xstream.dk/btno/feed/video/?platform=web&id=86588', + 'md5': 'd7d17e3337dc80de6d3a540aefbe441b', + 'info_dict': { + 'id': '86588', + 'ext': 'mov', + 'title': 'Otto Wollertsen', + 'description': 'Vestlendingen Otto Fredrik Wollertsen', + 'timestamp': 1430473209, + 'upload_date': '20150501', + }, + }, { + 'url': 'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=21039', + 'only_matching': True, + }] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + partner_id = mobj.group('partner_id') + video_id = mobj.group('id') + + data = self._download_xml( + 'http://frontend.xstream.dk/%s/feed/video/?platform=web&id=%s' + % (partner_id, video_id), + video_id) + + NS_MAP = { + 'atom': 'http://www.w3.org/2005/Atom', + 'xt': 'http://xstream.dk/', + 'media': 'http://search.yahoo.com/mrss/', + } + + entry = data.find(xpath_with_ns('./atom:entry', NS_MAP)) + + title = xpath_text( + entry, xpath_with_ns('./atom:title', NS_MAP), 'title') + description = xpath_text( + entry, xpath_with_ns('./atom:summary', NS_MAP), 'description') + timestamp = parse_iso8601(xpath_text( + entry, xpath_with_ns('./atom:published', NS_MAP), 'upload date')) + + formats = [] + media_group = entry.find(xpath_with_ns('./media:group', NS_MAP)) + for media_content in media_group.findall(xpath_with_ns('./media:content', NS_MAP)): + media_url = media_content.get('url') + if not media_url: + continue + tbr = int_or_none(media_content.get('bitrate')) + mobj = re.search(r'^(?Prtmp://[^/]+/(?P[^/]+))/(?P.+)$', media_url) + if mobj: + formats.append({ + 'url': mobj.group('url'), + 'play_path': 'mp4:%s' % mobj.group('playpath'), + 'app': mobj.group('app'), + 'ext': 'flv', + 'tbr': tbr, + 'format_id': 'rtmp-%d' % tbr, + }) + else: + formats.append({ + 'url': media_url, + 'tbr': tbr, + }) + self._sort_formats(formats) + + link = find_xpath_attr( + entry, xpath_with_ns('./atom:link', NS_MAP), 'rel', 'original') + if link is not None: + formats.append({ + 'url': link.get('href'), + 'format_id': link.get('rel'), + }) + + thumbnails = [{ + 'url': splash.get('url'), + 'width': int_or_none(splash.get('width')), + 'height': int_or_none(splash.get('height')), + } for splash in media_group.findall(xpath_with_ns('./xt:splash', NS_MAP))] + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'timestamp': timestamp, + 'formats': formats, + 'thumbnails': thumbnails, + } diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 1013f7c18..de09b53b2 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1380,7 +1380,7 @@ def get_exe_version(exe, args=['--version'], or False if the executable is not present """ try: out, _ = subprocess.Popen( - [exe] + args, + [encodeArgument(exe)] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate() except OSError: return False