X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2FYoutubeDL.py;h=5794fdbe9f16897357892db883beef88e9f3a1e5;hb=1ff7c0f7d804dedb9186490f2d61c773e62650c5;hp=f202ba4f0233172d5184e71a348c9dcfe652b4fb;hpb=fd28827864f94aee1cb4103179b6c4965f0b6641;p=youtube-dl diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index f202ba4f0..5794fdbe9 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -4,9 +4,11 @@ from __future__ import absolute_import, unicode_literals import collections +import datetime import errno import io import json +import locale import os import platform import re @@ -93,6 +95,7 @@ class YoutubeDL(object): usenetrc: Use netrc for authentication instead. verbose: Print additional info to stdout. quiet: Do not print messages to stdout. + no_warnings: Do not print out anything for warnings. forceurl: Force printing final URL. forcetitle: Force printing title. forceid: Force printing ID. @@ -147,6 +150,8 @@ class YoutubeDL(object): again. cookiefile: File name where cookies should be read from and dumped to. nocheckcertificate:Do not verify SSL certificates + prefer_insecure: Use HTTP instead of HTTPS to retrieve information. + At the moment, this is only supported by YouTube. proxy: URL of the proxy server to use socket_timeout: Time to wait for unresponsive hosts, in seconds bidi_workaround: Work around buggy terminals without bidirectional text @@ -155,6 +160,7 @@ class YoutubeDL(object): include_ads: Download ads as well default_search: Prepend this string if an input url is not valid. 'auto' for elaborate guessing + encoding: Use this encoding instead of the system-specified. The following parameters are not used by YoutubeDL itself, they are used by the FileDownloader: @@ -331,7 +337,7 @@ class YoutubeDL(object): def __exit__(self, *args): self.restore_console_title() - + if self.params.get('cookiefile') is not None: self.cookiejar.save() @@ -370,12 +376,17 @@ class YoutubeDL(object): Print the message to stderr, it will be prefixed with 'WARNING:' If stderr is a tty file the 'WARNING:' will be colored ''' - if self._err_file.isatty() and os.name != 'nt': - _msg_header = '\033[0;33mWARNING:\033[0m' + if self.params.get('logger') is not None: + self.params['logger'].warning(message) else: - _msg_header = 'WARNING:' - warning_message = '%s %s' % (_msg_header, message) - self.to_stderr(warning_message) + if self.params.get('no_warnings'): + return + if self._err_file.isatty() and os.name != 'nt': + _msg_header = '\033[0;33mWARNING:\033[0m' + else: + _msg_header = 'WARNING:' + warning_message = '%s %s' % (_msg_header, message) + self.to_stderr(warning_message) def report_error(self, message, tb=None): ''' @@ -409,6 +420,13 @@ class YoutubeDL(object): template_dict['autonumber'] = autonumber_templ % self._num_downloads if template_dict.get('playlist_index') is not None: template_dict['playlist_index'] = '%05d' % template_dict['playlist_index'] + if template_dict.get('resolution') is None: + if template_dict.get('width') and template_dict.get('height'): + template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height']) + elif template_dict.get('height'): + template_dict['resolution'] = '%sp' % template_dict['height'] + elif template_dict.get('width'): + template_dict['resolution'] = '?x%d' % template_dict['width'] sanitize = lambda k, v: sanitize_filename( compat_str(v), @@ -499,13 +517,7 @@ class YoutubeDL(object): '_type': 'compat_list', 'entries': ie_result, } - self.add_extra_info(ie_result, - { - 'extractor': ie.IE_NAME, - 'webpage_url': url, - 'webpage_url_basename': url_basename(url), - 'extractor_key': ie.ie_key(), - }) + self.add_default_extra_info(ie_result, ie, url) if process: return self.process_ie_result(ie_result, download, extra_info) else: @@ -522,7 +534,15 @@ class YoutubeDL(object): else: raise else: - self.report_error('no suitable InfoExtractor: %s' % url) + self.report_error('no suitable InfoExtractor for URL %s' % url) + + def add_default_extra_info(self, ie_result, ie, url): + self.add_extra_info(ie_result, { + 'extractor': ie.IE_NAME, + 'webpage_url': url, + 'webpage_url_basename': url_basename(url), + 'extractor_key': ie.ie_key(), + }) def process_ie_result(self, ie_result, download=True, extra_info={}): """ @@ -656,6 +676,18 @@ class YoutubeDL(object): if f.get('vcodec') == 'none'] if audio_formats: return audio_formats[0] + elif format_spec == 'bestvideo': + video_formats = [ + f for f in available_formats + if f.get('acodec') == 'none'] + if video_formats: + return video_formats[-1] + elif format_spec == 'worstvideo': + video_formats = [ + f for f in available_formats + if f.get('acodec') == 'none'] + if video_formats: + return video_formats[0] else: extensions = ['mp4', 'flv', 'webm', '3gp'] if format_spec in extensions: @@ -670,11 +702,24 @@ class YoutubeDL(object): def process_video_result(self, info_dict, download=True): assert info_dict.get('_type', 'video') == 'video' + if 'id' not in info_dict: + raise ExtractorError('Missing "id" field in extractor result') + if 'title' not in info_dict: + raise ExtractorError('Missing "title" field in extractor result') + if 'playlist' not in info_dict: # It isn't part of a playlist info_dict['playlist'] = None info_dict['playlist_index'] = None + if 'display_id' not in info_dict and 'id' in info_dict: + info_dict['display_id'] = info_dict['id'] + + if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None: + upload_date = datetime.datetime.utcfromtimestamp( + info_dict['timestamp']) + info_dict['upload_date'] = upload_date.strftime('%Y%m%d') + # This extractors handle format selection themselves if info_dict['extractor'] in ['Youku']: if download: @@ -688,8 +733,14 @@ class YoutubeDL(object): else: formats = info_dict['formats'] + if not formats: + raise ExtractorError('No video formats found!') + # We check that all the formats have the format and format_id fields - for (i, format) in enumerate(formats): + for i, format in enumerate(formats): + if 'url' not in format: + raise ExtractorError('Missing "url" key in result (index %d)' % i) + if format.get('format_id') is None: format['format_id'] = compat_str(i) if format.get('format') is None: @@ -700,7 +751,7 @@ class YoutubeDL(object): ) # Automatically determine file extension if missing if 'ext' not in format: - format['ext'] = determine_ext(format['url']) + format['ext'] = determine_ext(format['url']).lower() format_limit = self.params.get('format_limit', None) if format_limit: @@ -710,10 +761,10 @@ class YoutubeDL(object): # TODO Central sorting goes here - if formats[0] is not info_dict: + if formats[0] is not info_dict: # only set the 'formats' fields if the original info_dict list them # otherwise we end up with a circular reference, the first (and unique) - # element in the 'formats' field in info_dict is info_dict itself, + # element in the 'formats' field in info_dict is info_dict itself, # wich can't be exported to json info_dict['formats'] = formats if self.params.get('listformats', None): @@ -825,7 +876,7 @@ class YoutubeDL(object): try: dn = os.path.dirname(encodeFilename(filename)) - if dn != '' and not os.path.exists(dn): + if dn and not os.path.exists(dn): os.makedirs(dn) except (OSError, IOError) as err: self.report_error('unable to create directory ' + compat_str(err)) @@ -908,7 +959,7 @@ class YoutubeDL(object): self.to_screen('[%s] %s: Downloading thumbnail ...' % (info_dict['extractor'], info_dict['id'])) try: - uf = compat_urllib_request.urlopen(info_dict['thumbnail']) + uf = self.urlopen(info_dict['thumbnail']) with open(thumb_filename, 'wb') as thumbf: shutil.copyfileobj(uf, thumbf) self.to_screen('[%s] %s: Writing thumbnail to: %s' % @@ -1094,9 +1145,15 @@ class YoutubeDL(object): res += fdict['format_note'] + ' ' if fdict.get('tbr') is not None: res += '%4dk ' % fdict['tbr'] + if fdict.get('container') is not None: + if res: + res += ', ' + res += '%s container' % fdict['container'] if (fdict.get('vcodec') is not None and fdict.get('vcodec') != 'none'): - res += '%-5s' % fdict['vcodec'] + if res: + res += ', ' + res += fdict['vcodec'] if fdict.get('vbr') is not None: res += '@' elif fdict.get('vbr') is not None and fdict.get('abr') is not None: @@ -1106,7 +1163,10 @@ class YoutubeDL(object): if fdict.get('acodec') is not None: if res: res += ', ' - res += '%-5s' % fdict['acodec'] + if fdict['acodec'] == 'none': + res += 'video only' + else: + res += '%-5s' % fdict['acodec'] elif fdict.get('abr') is not None: if res: res += ', ' @@ -1145,11 +1205,14 @@ class YoutubeDL(object): def urlopen(self, req): """ Start an HTTP download """ - return self._opener.open(req) + return self._opener.open(req, timeout=self._socket_timeout) def print_debug_header(self): if not self.params.get('verbose'): return + + write_string('[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % + (locale.getpreferredencoding(), sys.getfilesystemencoding(), sys.stdout.encoding, self.get_encoding())) write_string('[debug] youtube-dl version ' + __version__ + '\n') try: sp = subprocess.Popen( @@ -1176,7 +1239,7 @@ class YoutubeDL(object): def _setup_opener(self): timeout_val = self.params.get('socket_timeout') - timeout = 600 if timeout_val is None else float(timeout_val) + self._socket_timeout = 600 if timeout_val is None else float(timeout_val) opts_cookiefile = self.params.get('cookiefile') opts_proxy = self.params.get('proxy') @@ -1215,6 +1278,18 @@ class YoutubeDL(object): opener.addheaders = [] self._opener = opener - # TODO remove this global modification - compat_urllib_request.install_opener(opener) - socket.setdefaulttimeout(timeout) + def encode(self, s): + if isinstance(s, bytes): + return s # Already encoded + + try: + return s.encode(self.get_encoding()) + except UnicodeEncodeError as err: + err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.' + raise + + def get_encoding(self): + encoding = self.params.get('encoding') + if encoding is None: + encoding = preferredencoding() + return encoding