X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube_dl%2FFileDownloader.py;h=8a3bdf21b55ac39942cd1c421079750e9c44f9be;hb=1bd96c3a60e40d2490be834f95b9acca6c74eb07;hp=7139adf6bb6490b31dc3661b0fc220df30b0dad2;hpb=44696667805343a2f60bdec25d8ab9ed90b5963c;p=youtube-dl diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 7139adf6b..8a3bdf21b 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -7,6 +7,7 @@ import math import io import os import re +import shutil import socket import subprocess import sys @@ -53,6 +54,7 @@ class FileDownloader(object): quiet: Do not print messages to stdout. forceurl: Force printing final URL. forcetitle: Force printing title. + forceid: Force printing ID. forcethumbnail: Force printing thumbnail URL. forcedescription: Force printing description. forcefilename: Force printing final filename. @@ -79,8 +81,8 @@ class FileDownloader(object): updatetime: Use the Last-modified header to set output file timestamps. writedescription: Write the video description to a .description file writeinfojson: Write the video description to a .info.json file + writethumbnail: Write the thumbnail image to a file writesubtitles: Write the video subtitles to a file - onlysubtitles: Downloads only the subtitles of the video allsubtitles: Downloads all the subtitles of the video listsubtitles: Lists all available subtitles for the video subtitlesformat: Subtitle format [sbv/srt] (default=srt) @@ -89,6 +91,8 @@ class FileDownloader(object): keepvideo: Keep the video file after post-processing min_filesize: Skip files smaller than this size max_filesize: Skip files larger than this size + daterange: A DateRange object, download only if the upload_date is in the range. + skip_download: Skip the actual download of the video file """ params = None @@ -121,7 +125,7 @@ class FileDownloader(object): exponent = 0 else: exponent = int(math.log(bytes, 1024.0)) - suffix = 'bkMGTPEZY'[exponent] + suffix = ['B','KiB','MiB','GiB','TiB','PiB','EiB','ZiB','YiB'][exponent] converted = float(bytes) / float(1024 ** exponent) return '%.2f%s' % (converted, suffix) @@ -254,7 +258,7 @@ class FileDownloader(object): Print the message to stderr, it will be prefixed with 'WARNING:' If stderr is a tty file the 'WARNING:' will be colored ''' - if sys.stderr.isatty(): + if sys.stderr.isatty() and os.name != 'nt': _msg_header=u'\033[0;33mWARNING:\033[0m' else: _msg_header=u'WARNING:' @@ -266,7 +270,7 @@ class FileDownloader(object): Do the same as trouble, but prefixes the message with 'ERROR:', colored in red if stderr is a tty file. ''' - if sys.stderr.isatty(): + if sys.stderr.isatty() and os.name != 'nt': _msg_header = u'\033[0;31mERROR:\033[0m' else: _msg_header = u'ERROR:' @@ -344,12 +348,13 @@ class FileDownloader(object): """Report download progress.""" if self.params.get('noprogress', False): return + clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'') if self.params.get('progress_with_newline', False): self.to_screen(u'[download] %s of %s at %s ETA %s' % (percent_str, data_len_str, speed_str, eta_str)) else: - self.to_screen(u'\r[download] %s of %s at %s ETA %s' % - (percent_str, data_len_str, speed_str, eta_str), skip_eol=True) + self.to_screen(u'\r%s[download] %s of %s at %s ETA %s' % + (clear_line, percent_str, data_len_str, speed_str, eta_str), skip_eol=True) self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' % (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip())) @@ -406,10 +411,10 @@ class FileDownloader(object): filename = self.params['outtmpl'] % template_dict return filename except KeyError as err: - self.trouble(u'ERROR: Erroneous output template') + self.report_error(u'Erroneous output template') return None except ValueError as err: - self.trouble(u'ERROR: Insufficient system charset ' + repr(preferredencoding())) + self.report_error(u'Insufficient system charset ' + repr(preferredencoding())) return None def _match_entry(self, info_dict): @@ -424,85 +429,81 @@ class FileDownloader(object): if rejecttitle: if re.search(rejecttitle, title, re.IGNORECASE): return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' + date = info_dict.get('upload_date', None) + if date is not None: + dateRange = self.params.get('daterange', DateRange()) + if date not in dateRange: + return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange) return None - def extract_info(self, url, download = True, ie_name = None): + def extract_info(self, url, download=True, ie_key=None): ''' Returns a list with a dictionary for each video we find. If 'download', also downloads the videos. ''' - suitable_found = False - #We copy the original list - ies = list(self._ies) - - if ie_name is not None: - #We put in the first place the given info extractor - first_ie = get_info_extractor(ie_name)() - first_ie.set_downloader(self) - ies.insert(0, first_ie) + if ie_key: + ie = get_info_extractor(ie_key)() + ie.set_downloader(self) + ies = [ie] + else: + ies = self._ies for ie in ies: - # Go to next InfoExtractor if not suitable if not ie.suitable(url): continue - # Warn if the _WORKING attribute is False if not ie.working(): - self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, ' - u'and will probably not work. If you want to go on, use the -i option.') + self.report_warning(u'The program functionality for this site has been marked as broken, ' + u'and will probably not work.') - # Suitable InfoExtractor found - suitable_found = True - - # Extract information from URL and process it try: - ie_results = ie.extract(url) - if ie_results is None: # Finished already (backwards compatibility; listformats and friends should be moved here) + ie_result = ie.extract(url) + if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here) break - results = [] - for ie_result in ie_results: - if not 'extractor' in ie_result: - #The extractor has already been set somewhere else - ie_result['extractor'] = ie.IE_NAME - results.append(self.process_ie_result(ie_result, download)) - return results + if isinstance(ie_result, list): + # Backwards compatibility: old IE result format + ie_result = { + '_type': 'compat_list', + 'entries': ie_result, + } + if 'extractor' not in ie_result: + ie_result['extractor'] = ie.IE_NAME + return self.process_ie_result(ie_result, download=download) except ExtractorError as de: # An error we somewhat expected - self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback()) + self.report_error(compat_str(de), de.format_traceback()) break except Exception as e: if self.params.get('ignoreerrors', False): - self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc())) + self.report_error(compat_str(e), tb=compat_str(traceback.format_exc())) break else: raise - if not suitable_found: - self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) + else: + self.report_error(u'no suitable InfoExtractor: %s' % url) - def process_ie_result(self, ie_result, download = True): + def process_ie_result(self, ie_result, download=True): """ - Take the result of the ie and return a list of videos. - For url elements it will search the suitable ie and get the videos - For playlist elements it will process each of the elements of the 'entries' key - + Take the result of the ie(may be modified) and resolve all unresolved + references (URLs, playlist items). + It will also download the videos if 'download'. + Returns the resolved ie_result. """ - result_type = ie_result.get('_type', 'video') #If not given we suppose it's a video, support the dafault old system + + result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system if result_type == 'video': if 'playlist' not in ie_result: - #It isn't part of a playlist + # It isn't part of a playlist ie_result['playlist'] = None ie_result['playlist_index'] = None if download: - #Do the download: self.process_info(ie_result) return ie_result elif result_type == 'url': - #We get the video pointed by the url - result = self.extract_info(ie_result['url'], download, ie_name = ie_result['ie_key'])[0] - return result + return self.extract_info(ie_result['url'], download, ie_key=ie_result.get('ie_key')) elif result_type == 'playlist': - #We process each entry in the playlist + # We process each entry in the playlist playlist = ie_result.get('title', None) or ie_result.get('id', None) self.to_screen(u'[download] Downloading playlist: %s' % playlist) @@ -524,23 +525,31 @@ class FileDownloader(object): for i,entry in enumerate(entries,1): self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries)) - entry_result = self.process_ie_result(entry, False) - entry_result['playlist'] = playlist - entry_result['playlist_index'] = i + playliststart - #We must do the download here to correctly set the 'playlist' key - if download: - self.process_info(entry_result) + entry['playlist'] = playlist + entry['playlist_index'] = i + playliststart + entry_result = self.process_ie_result(entry, download=download) playlist_results.append(entry_result) - result = ie_result.copy() - result['entries'] = playlist_results - return result + ie_result['entries'] = playlist_results + return ie_result + elif result_type == 'compat_list': + def _fixup(r): + r.setdefault('extractor', ie_result['extractor']) + return r + ie_result['entries'] = [ + self.process_ie_result(_fixup(r), download=download) + for r in ie_result['entries'] + ] + return ie_result + else: + raise Exception('Invalid result type: %s' % result_type) def process_info(self, info_dict): - """Process a single dictionary returned by an InfoExtractor.""" + """Process a single resolved IE result.""" + assert info_dict.get('_type', 'video') == 'video' #We increment the download the download count here to match the previous behaviour. self.increment_downloads() - + info_dict['fulltitle'] = info_dict['title'] if len(info_dict['title']) > 200: info_dict['title'] = info_dict['title'][:197] + u'...' @@ -566,6 +575,8 @@ class FileDownloader(object): # Forced printings if self.params.get('forcetitle', False): compat_print(info_dict['title']) + if self.params.get('forceid', False): + compat_print(info_dict['id']) if self.params.get('forceurl', False): compat_print(info_dict['url']) if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict: @@ -586,7 +597,7 @@ class FileDownloader(object): try: dn = os.path.dirname(encodeFilename(filename)) - if dn != '' and not os.path.exists(dn): # dn is already encoded + if dn != '' and not os.path.exists(dn): os.makedirs(dn) except (OSError, IOError) as err: self.report_error(u'unable to create directory ' + compat_str(err)) @@ -619,8 +630,6 @@ class FileDownloader(object): except (OSError, IOError): self.report_error(u'Cannot write subtitles file ' + descfn) return - if self.params.get('onlysubtitles', False): - return if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']: subtitles = info_dict['subtitles'] @@ -636,10 +645,8 @@ class FileDownloader(object): with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: subfile.write(sub) except (OSError, IOError): - self.trouble(u'ERROR: Cannot write subtitles file ' + descfn) + self.report_error(u'Cannot write subtitles file ' + descfn) return - if self.params.get('onlysubtitles', False): - return if self.params.get('writeinfojson', False): infofn = filename + u'.info.json' @@ -651,6 +658,20 @@ class FileDownloader(object): self.report_error(u'Cannot write metadata to JSON file ' + infofn) return + if self.params.get('writethumbnail', False): + if 'thumbnail' in info_dict: + thumb_format = info_dict['thumbnail'].rpartition(u'/')[2].rpartition(u'.')[2] + if not thumb_format: + thumb_format = 'jpg' + thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format + self.to_screen(u'[%s] %s: Downloading thumbnail ...' % + (info_dict['extractor'], info_dict['id'])) + uf = compat_urllib_request.urlopen(info_dict['thumbnail']) + with open(thumb_filename, 'wb') as thumbf: + shutil.copyfileobj(uf, thumbf) + self.to_screen(u'[%s] %s: Writing thumbnail to: %s' % + (info_dict['extractor'], info_dict['id'], thumb_filename)) + if not self.params.get('skip_download', False): if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)): success = True @@ -683,7 +704,7 @@ class FileDownloader(object): #It also downloads the videos videos = self.extract_info(url) except UnavailableVideoError: - self.trouble(u'\nERROR: unable to download video') + self.report_error(u'unable to download video') except MaxDownloadsReached: self.to_screen(u'[info] Maximum number of downloaded files reached.') raise