X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2FYoutubeDL.py;h=9519594c9ad2dab36e1d66fe2f016d7edc6e798d;hb=fc96eb4e2180d9a2371d84daa4305a5f34f12321;hp=5794fdbe9f16897357892db883beef88e9f3a1e5;hpb=b84d6e7fc42affddeb1baf989cf394fedc41a96d;p=youtube-dl diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py old mode 100644 new mode 100755 index 5794fdbe9..9519594c9 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -28,9 +28,11 @@ from .utils import ( compat_str, compat_urllib_error, compat_urllib_request, + escape_url, ContentTooShortError, date_from_str, DateRange, + DEFAULT_OUTTMPL, determine_ext, DownloadError, encodeFilename, @@ -56,6 +58,7 @@ from .utils import ( YoutubeDLHandler, prepend_extension, ) +from .cache import Cache from .extractor import get_info_extractor, gen_extractors from .downloader import get_suitable_downloader from .postprocessor import FFmpegMergerPP @@ -132,7 +135,7 @@ class YoutubeDL(object): daterange: A DateRange object, download only if the upload_date is in the range. skip_download: Skip the actual download of the video file cachedir: Location of the cache files in the filesystem. - None to disable filesystem cache. + False to disable filesystem cache. noplaylist: Download single video instead of a playlist if in doubt. age_limit: An integer representing the user's age in years. Unsuitable videos for the given age are skipped. @@ -161,6 +164,7 @@ class YoutubeDL(object): default_search: Prepend this string if an input url is not valid. 'auto' for elaborate guessing encoding: Use this encoding instead of the system-specified. + extract_flat: Do not resolve URLs, return the immediate result. The following parameters are not used by YoutubeDL itself, they are used by the FileDownloader: @@ -170,6 +174,7 @@ class YoutubeDL(object): The following options are used by the post processors: prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available, otherwise prefer avconv. + exec_cmd: Arbitrary command to run after downloading """ params = None @@ -192,6 +197,7 @@ class YoutubeDL(object): self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] self._err_file = sys.stderr self.params = params + self.cache = Cache(self) if params.get('bidi_workaround', False): try: @@ -274,7 +280,7 @@ class YoutubeDL(object): return message assert hasattr(self, '_output_process') - assert type(message) == type('') + assert isinstance(message, compat_str) line_count = message.count('\n') + 1 self._output_process.stdin.write((message + '\n').encode('utf-8')) self._output_process.stdin.flush() @@ -286,6 +292,9 @@ class YoutubeDL(object): """Print message to stdout if not in quiet mode.""" return self.to_stdout(message, skip_eol, check_quiet=True) + def _write_string(self, s, out=None): + write_string(s, out=out, encoding=self.params.get('encoding')) + def to_stdout(self, message, skip_eol=False, check_quiet=False): """Print message to stdout if not in quiet mode.""" if self.params.get('logger'): @@ -295,17 +304,17 @@ class YoutubeDL(object): terminator = ['\n', ''][skip_eol] output = message + terminator - write_string(output, self._screen_file) + self._write_string(output, self._screen_file) def to_stderr(self, message): """Print message to stderr.""" - assert type(message) == type('') + assert isinstance(message, compat_str) if self.params.get('logger'): self.params['logger'].error(message) else: message = self._bidi_workaround(message) output = message + '\n' - write_string(output, self._err_file) + self._write_string(output, self._err_file) def to_console_title(self, message): if not self.params.get('consoletitle', False): @@ -315,21 +324,21 @@ class YoutubeDL(object): # already of type unicode() ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) elif 'TERM' in os.environ: - write_string('\033]0;%s\007' % message, self._screen_file) + self._write_string('\033]0;%s\007' % message, self._screen_file) def save_console_title(self): if not self.params.get('consoletitle', False): return if 'TERM' in os.environ: # Save the title on stack - write_string('\033[22;0t', self._screen_file) + self._write_string('\033[22;0t', self._screen_file) def restore_console_title(self): if not self.params.get('consoletitle', False): return if 'TERM' in os.environ: # Restore the title from stack - write_string('\033[23;0t', self._screen_file) + self._write_string('\033[23;0t', self._screen_file) def __enter__(self): self.save_console_title() @@ -419,7 +428,7 @@ class YoutubeDL(object): autonumber_templ = '%0' + str(autonumber_size) + 'd' template_dict['autonumber'] = autonumber_templ % self._num_downloads if template_dict.get('playlist_index') is not None: - template_dict['playlist_index'] = '%05d' % template_dict['playlist_index'] + template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index']) if template_dict.get('resolution') is None: if template_dict.get('width') and template_dict.get('height'): template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height']) @@ -437,7 +446,8 @@ class YoutubeDL(object): if v is not None) template_dict = collections.defaultdict(lambda: 'NA', template_dict) - tmpl = os.path.expanduser(self.params['outtmpl']) + outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL) + tmpl = os.path.expanduser(outtmpl) filename = tmpl % template_dict return filename except ValueError as err: @@ -474,7 +484,10 @@ class YoutubeDL(object): return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views) age_limit = self.params.get('age_limit') if age_limit is not None: - if age_limit < info_dict.get('age_limit', 0): + actual_age_limit = info_dict.get('age_limit') + if actual_age_limit is None: + actual_age_limit = 0 + if age_limit < actual_age_limit: return 'Skipping "' + title + '" because it is age restricted' if self.in_download_archive(info_dict): return '%s has already been recorded in archive' % video_title @@ -553,7 +566,12 @@ class YoutubeDL(object): Returns the resolved ie_result. """ - result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system + result_type = ie_result.get('_type', 'video') + + if self.params.get('extract_flat', False): + if result_type in ('url', 'url_transparent'): + return ie_result + if result_type == 'video': self.add_extra_info(ie_result, extra_info) return self.process_video_result(ie_result, download=download) @@ -622,6 +640,7 @@ class YoutubeDL(object): for i, entry in enumerate(entries, 1): self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries)) extra = { + 'n_entries': n_entries, 'playlist': playlist, 'playlist_index': i + playliststart, 'extractor': ie_result['extractor'], @@ -712,6 +731,17 @@ class YoutubeDL(object): info_dict['playlist'] = None info_dict['playlist_index'] = None + thumbnails = info_dict.get('thumbnails') + if thumbnails: + thumbnails.sort(key=lambda t: ( + t.get('width'), t.get('height'), t.get('url'))) + for t in thumbnails: + if 'width' in t and 'height' in t: + t['resolution'] = '%dx%d' % (t['width'], t['height']) + + if thumbnails and 'thumbnail' not in info_dict: + info_dict['thumbnail'] = thumbnails[-1]['url'] + if 'display_id' not in info_dict and 'id' in info_dict: info_dict['display_id'] = info_dict['id'] @@ -833,7 +863,7 @@ class YoutubeDL(object): # Keep for backwards compatibility info_dict['stitle'] = info_dict['title'] - if not 'format' in info_dict: + if 'format' not in info_dict: info_dict['format'] = info_dict['ext'] reason = self._match_entry(info_dict) @@ -933,7 +963,7 @@ class YoutubeDL(object): with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: subfile.write(sub) except (OSError, IOError): - self.report_error('Cannot write subtitles file ' + descfn) + self.report_error('Cannot write subtitles file ' + sub_filename) return if self.params.get('writeinfojson', False): @@ -977,11 +1007,13 @@ class YoutubeDL(object): fd = get_suitable_downloader(info)(self, self.params) for ph in self._progress_hooks: fd.add_progress_hook(ph) + if self.params.get('verbose'): + self.to_stdout('[debug] Invoking downloader on %r' % info.get('url')) return fd.download(name, info) if info_dict.get('requested_formats') is not None: downloaded = [] success = True - merger = FFmpegMergerPP(self) + merger = FFmpegMergerPP(self, not self.params.get('keepvideo')) if not merger._get_executable(): postprocessors = [] self.report_warning('You have requested multiple ' @@ -1022,10 +1054,11 @@ class YoutubeDL(object): def download(self, url_list): """Download a given list of URLs.""" + outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL) if (len(url_list) > 1 and - '%' not in self.params['outtmpl'] + '%' not in outtmpl and self.params.get('max_downloads') != 1): - raise SameFileError(self.params['outtmpl']) + raise SameFileError(outtmpl) for url in url_list: try: @@ -1136,57 +1169,61 @@ class YoutubeDL(object): res = default return res - def list_formats(self, info_dict): - def format_note(fdict): - res = '' - if fdict.get('ext') in ['f4f', 'f4m']: - res += '(unsupported) ' - if fdict.get('format_note') is not None: - res += fdict['format_note'] + ' ' - if fdict.get('tbr') is not None: - res += '%4dk ' % fdict['tbr'] - if fdict.get('container') is not None: - if res: - res += ', ' - res += '%s container' % fdict['container'] - if (fdict.get('vcodec') is not None and - fdict.get('vcodec') != 'none'): - if res: - res += ', ' - res += fdict['vcodec'] - if fdict.get('vbr') is not None: - res += '@' - elif fdict.get('vbr') is not None and fdict.get('abr') is not None: - res += 'video@' + def _format_note(self, fdict): + res = '' + if fdict.get('ext') in ['f4f', 'f4m']: + res += '(unsupported) ' + if fdict.get('format_note') is not None: + res += fdict['format_note'] + ' ' + if fdict.get('tbr') is not None: + res += '%4dk ' % fdict['tbr'] + if fdict.get('container') is not None: + if res: + res += ', ' + res += '%s container' % fdict['container'] + if (fdict.get('vcodec') is not None and + fdict.get('vcodec') != 'none'): + if res: + res += ', ' + res += fdict['vcodec'] if fdict.get('vbr') is not None: - res += '%4dk' % fdict['vbr'] - if fdict.get('acodec') is not None: - if res: - res += ', ' - if fdict['acodec'] == 'none': - res += 'video only' - else: - res += '%-5s' % fdict['acodec'] - elif fdict.get('abr') is not None: - if res: - res += ', ' - res += 'audio' - if fdict.get('abr') is not None: - res += '@%3dk' % fdict['abr'] - if fdict.get('asr') is not None: - res += ' (%5dHz)' % fdict['asr'] - if fdict.get('filesize') is not None: - if res: - res += ', ' - res += format_bytes(fdict['filesize']) - return res + res += '@' + elif fdict.get('vbr') is not None and fdict.get('abr') is not None: + res += 'video@' + if fdict.get('vbr') is not None: + res += '%4dk' % fdict['vbr'] + if fdict.get('acodec') is not None: + if res: + res += ', ' + if fdict['acodec'] == 'none': + res += 'video only' + else: + res += '%-5s' % fdict['acodec'] + elif fdict.get('abr') is not None: + if res: + res += ', ' + res += 'audio' + if fdict.get('abr') is not None: + res += '@%3dk' % fdict['abr'] + if fdict.get('asr') is not None: + res += ' (%5dHz)' % fdict['asr'] + if fdict.get('filesize') is not None: + if res: + res += ', ' + res += format_bytes(fdict['filesize']) + elif fdict.get('filesize_approx') is not None: + if res: + res += ', ' + res += '~' + format_bytes(fdict['filesize_approx']) + return res + def list_formats(self, info_dict): def line(format, idlen=20): return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % ( format['format_id'], format['ext'], self.format_resolution(format), - format_note(format), + self._format_note(format), )) formats = info_dict.get('formats', [info_dict]) @@ -1194,8 +1231,8 @@ class YoutubeDL(object): max(len(f['format_id']) for f in formats)) formats_s = [line(f, idlen) for f in formats] if len(formats) > 1: - formats_s[0] += (' ' if format_note(formats[0]) else '') + '(worst)' - formats_s[-1] += (' ' if format_note(formats[-1]) else '') + '(best)' + formats_s[0] += (' ' if self._format_note(formats[0]) else '') + '(worst)' + formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)' header_line = line({ 'format_id': 'format code', 'ext': 'extension', @@ -1205,15 +1242,45 @@ class YoutubeDL(object): def urlopen(self, req): """ Start an HTTP download """ + + # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not + # always respected by websites, some tend to give out URLs with non percent-encoded + # non-ASCII characters (see telemb.py, ard.py [#3412]) + # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991) + # To work around aforementioned issue we will replace request's original URL with + # percent-encoded one + url = req if isinstance(req, compat_str) else req.get_full_url() + url_escaped = escape_url(url) + + # Substitute URL if any change after escaping + if url != url_escaped: + if isinstance(req, compat_str): + req = url_escaped + else: + req = compat_urllib_request.Request( + url_escaped, data=req.data, headers=req.headers, + origin_req_host=req.origin_req_host, unverifiable=req.unverifiable) + return self._opener.open(req, timeout=self._socket_timeout) def print_debug_header(self): if not self.params.get('verbose'): return - write_string('[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % - (locale.getpreferredencoding(), sys.getfilesystemencoding(), sys.stdout.encoding, self.get_encoding())) - write_string('[debug] youtube-dl version ' + __version__ + '\n') + if type('') is not compat_str: + # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326) + self.report_warning( + 'Your Python is broken! Update to a newer and supported version') + + encoding_str = ( + '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % ( + locale.getpreferredencoding(), + sys.getfilesystemencoding(), + sys.stdout.encoding, + self.get_encoding())) + write_string(encoding_str, encoding=None) + + self._write_string('[debug] youtube-dl version ' + __version__ + '\n') try: sp = subprocess.Popen( ['git', 'rev-parse', '--short', 'HEAD'], @@ -1222,20 +1289,20 @@ class YoutubeDL(object): out, err = sp.communicate() out = out.decode().strip() if re.match('[0-9a-f]+', out): - write_string('[debug] Git HEAD: ' + out + '\n') + self._write_string('[debug] Git HEAD: ' + out + '\n') except: try: sys.exc_clear() except: pass - write_string('[debug] Python version %s - %s' % + self._write_string('[debug] Python version %s - %s' % (platform.python_version(), platform_name()) + '\n') proxy_map = {} for handler in self._opener.handlers: if hasattr(handler, 'proxies'): proxy_map.update(handler.proxies) - write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n') + self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n') def _setup_opener(self): timeout_val = self.params.get('socket_timeout')