2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import
22 class FileDownloader(object):
23 """File Downloader class.
25 File downloader objects are the ones responsible of downloading the
26 actual video file and writing it to disk if the user has requested
27 it, among some other tasks. In most cases there should be one per
28 program. As, given a video URL, the downloader doesn't know how to
29 extract all the needed information, task that InfoExtractors do, it
30 has to pass the URL to one of them.
32 For this, file downloader objects have a method that allows
33 InfoExtractors to be registered in a given order. When it is passed
34 a URL, the file downloader handles it to the first InfoExtractor it
35 finds that reports being able to handle it. The InfoExtractor extracts
36 all the information about the video or videos the URL refers to, and
37 asks the FileDownloader to process the video information, possibly
38 downloading the video.
40 File downloaders accept a lot of parameters. In order not to saturate
41 the object constructor with arguments, it receives a dictionary of
42 options instead. These options are available through the params
43 attribute for the InfoExtractors to use. The FileDownloader also
44 registers itself as the downloader in charge for the InfoExtractors
45 that are added to it, so this is a "mutual registration".
49 username: Username for authentication purposes.
50 password: Password for authentication purposes.
51 usenetrc: Use netrc for authentication instead.
52 quiet: Do not print messages to stdout.
53 forceurl: Force printing final URL.
54 forcetitle: Force printing title.
55 forcethumbnail: Force printing thumbnail URL.
56 forcedescription: Force printing description.
57 forcefilename: Force printing final filename.
58 simulate: Do not download the video files.
59 format: Video format code.
60 format_limit: Highest quality format to try.
61 outtmpl: Template for output names.
62 restrictfilenames: Do not allow "&" and spaces in file names
63 ignoreerrors: Do not stop on download errors.
64 ratelimit: Download speed limit, in bytes/sec.
65 nooverwrites: Prevent overwriting files.
66 retries: Number of times to retry for HTTP error 5xx
67 buffersize: Size of download buffer in bytes.
68 noresizebuffer: Do not automatically resize the download buffer.
69 continuedl: Try to continue downloads if possible.
70 noprogress: Do not print the progress bar.
71 playliststart: Playlist item to start at.
72 playlistend: Playlist item to end at.
73 matchtitle: Download only matching titles.
74 rejecttitle: Reject downloads for matching titles.
75 logtostderr: Log messages to stderr instead of stdout.
76 consoletitle: Display progress in console window's titlebar.
77 nopart: Do not use temporary .part files.
78 updatetime: Use the Last-modified header to set output file timestamps.
79 writedescription: Write the video description to a .description file
80 writeinfojson: Write the video description to a .info.json file
81 writesubtitles: Write the video subtitles to a file
82 onlysubtitles: Downloads only the subtitles of the video
83 allsubtitles: Downloads all the subtitles of the video
84 subtitlesformat: Subtitle format [sbv/srt] (default=srt)
85 subtitleslang: Language of the subtitles to download
86 test: Download only first bytes to test the downloader.
87 keepvideo: Keep the video file after post-processing
88 min_filesize: Skip files smaller than this size
89 max_filesize: Skip files larger than this size
95 _download_retcode = None
99 def __init__(self, params):
100 """Create a FileDownloader object with the given options."""
103 self._progress_hooks = []
104 self._download_retcode = 0
105 self._num_downloads = 0
106 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
109 if '%(stitle)s' in self.params['outtmpl']:
110 self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
113 def format_bytes(bytes):
116 if type(bytes) is str:
121 exponent = int(math.log(bytes, 1024.0))
122 suffix = 'bkMGTPEZY'[exponent]
123 converted = float(bytes) / float(1024 ** exponent)
124 return '%.2f%s' % (converted, suffix)
127 def calc_percent(byte_counter, data_len):
130 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
133 def calc_eta(start, now, total, current):
137 if current == 0 or dif < 0.001: # One millisecond
139 rate = float(current) / dif
140 eta = int((float(total) - float(current)) / rate)
141 (eta_mins, eta_secs) = divmod(eta, 60)
144 return '%02d:%02d' % (eta_mins, eta_secs)
147 def calc_speed(start, now, bytes):
149 if bytes == 0 or dif < 0.001: # One millisecond
150 return '%10s' % '---b/s'
151 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
154 def best_block_size(elapsed_time, bytes):
155 new_min = max(bytes / 2.0, 1.0)
156 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
157 if elapsed_time < 0.001:
159 rate = bytes / elapsed_time
167 def parse_bytes(bytestr):
168 """Parse a string indicating a byte quantity into an integer."""
169 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
172 number = float(matchobj.group(1))
173 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
174 return int(round(number * multiplier))
176 def add_info_extractor(self, ie):
177 """Add an InfoExtractor object to the end of the list."""
179 ie.set_downloader(self)
181 def add_post_processor(self, pp):
182 """Add a PostProcessor object to the end of the chain."""
184 pp.set_downloader(self)
186 def to_screen(self, message, skip_eol=False):
187 """Print message to stdout if not in quiet mode."""
188 assert type(message) == type(u'')
189 if not self.params.get('quiet', False):
190 terminator = [u'\n', u''][skip_eol]
191 output = message + terminator
192 if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
193 output = output.encode(preferredencoding(), 'ignore')
194 self._screen_file.write(output)
195 self._screen_file.flush()
197 def to_stderr(self, message):
198 """Print message to stderr."""
199 assert type(message) == type(u'')
200 output = message + u'\n'
201 if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
202 output = output.encode(preferredencoding())
203 sys.stderr.write(output)
205 def to_cons_title(self, message):
206 """Set console/terminal window title to message."""
207 if not self.params.get('consoletitle', False):
209 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
210 # c_wchar_p() might not be necessary if `message` is
211 # already of type unicode()
212 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
213 elif 'TERM' in os.environ:
214 self.to_screen('\033]0;%s\007' % message, skip_eol=True)
216 def fixed_template(self):
217 """Checks if the output template is fixed."""
218 return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
220 def trouble(self, message=None, tb=None):
221 """Determine action to take when a download problem appears.
223 Depending on if the downloader has been configured to ignore
224 download errors or not, this method may throw an exception or
225 not when errors are found, after printing the message.
227 tb, if given, is additional traceback information.
229 if message is not None:
230 self.to_stderr(message)
231 if self.params.get('verbose'):
233 tb_data = traceback.format_list(traceback.extract_stack())
234 tb = u''.join(tb_data)
236 if not self.params.get('ignoreerrors', False):
237 raise DownloadError(message)
238 self._download_retcode = 1
240 def slow_down(self, start_time, byte_counter):
241 """Sleep if the download speed is over the rate limit."""
242 rate_limit = self.params.get('ratelimit', None)
243 if rate_limit is None or byte_counter == 0:
246 elapsed = now - start_time
249 speed = float(byte_counter) / elapsed
250 if speed > rate_limit:
251 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
253 def temp_name(self, filename):
254 """Returns a temporary filename for the given filename."""
255 if self.params.get('nopart', False) or filename == u'-' or \
256 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
258 return filename + u'.part'
260 def undo_temp_name(self, filename):
261 if filename.endswith(u'.part'):
262 return filename[:-len(u'.part')]
265 def try_rename(self, old_filename, new_filename):
267 if old_filename == new_filename:
269 os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
270 except (IOError, OSError) as err:
271 self.trouble(u'ERROR: unable to rename file')
273 def try_utime(self, filename, last_modified_hdr):
274 """Try to set the last-modified time of the given file."""
275 if last_modified_hdr is None:
277 if not os.path.isfile(encodeFilename(filename)):
279 timestr = last_modified_hdr
282 filetime = timeconvert(timestr)
286 os.utime(filename, (time.time(), filetime))
291 def report_writedescription(self, descfn):
292 """ Report that the description file is being written """
293 self.to_screen(u'[info] Writing video description to: ' + descfn)
295 def report_writesubtitles(self, sub_filename):
296 """ Report that the subtitles file is being written """
297 self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename)
299 def report_writeinfojson(self, infofn):
300 """ Report that the metadata file has been written """
301 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
303 def report_destination(self, filename):
304 """Report destination filename."""
305 self.to_screen(u'[download] Destination: ' + filename)
307 def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
308 """Report download progress."""
309 if self.params.get('noprogress', False):
311 if self.params.get('progress_with_newline', False):
312 self.to_screen(u'[download] %s of %s at %s ETA %s' %
313 (percent_str, data_len_str, speed_str, eta_str))
315 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
316 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
317 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
318 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
320 def report_resuming_byte(self, resume_len):
321 """Report attempt to resume at given byte."""
322 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
324 def report_retry(self, count, retries):
325 """Report retry in case of HTTP error 5xx"""
326 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
328 def report_file_already_downloaded(self, file_name):
329 """Report file has already been fully downloaded."""
331 self.to_screen(u'[download] %s has already been downloaded' % file_name)
332 except (UnicodeEncodeError) as err:
333 self.to_screen(u'[download] The file has already been downloaded')
335 def report_unable_to_resume(self):
336 """Report it was impossible to resume download."""
337 self.to_screen(u'[download] Unable to resume')
339 def report_finish(self):
340 """Report download finished."""
341 if self.params.get('noprogress', False):
342 self.to_screen(u'[download] Download completed')
346 def increment_downloads(self):
347 """Increment the ordinal that assigns a number to each file."""
348 self._num_downloads += 1
350 def prepare_filename(self, info_dict):
351 """Generate the output filename."""
353 template_dict = dict(info_dict)
355 template_dict['epoch'] = int(time.time())
356 template_dict['autonumber'] = u'%05d' % self._num_downloads
358 sanitize = lambda k,v: sanitize_filename(
359 u'NA' if v is None else compat_str(v),
360 restricted=self.params.get('restrictfilenames'),
362 template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
364 filename = self.params['outtmpl'] % template_dict
366 except (ValueError, KeyError) as err:
367 self.trouble(u'ERROR: invalid system charset or erroneous output template')
370 def _match_entry(self, info_dict):
371 """ Returns None iff the file should be downloaded """
373 title = info_dict['title']
374 matchtitle = self.params.get('matchtitle', False)
376 if not re.search(matchtitle, title, re.IGNORECASE):
377 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
378 rejecttitle = self.params.get('rejecttitle', False)
380 if re.search(rejecttitle, title, re.IGNORECASE):
381 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
384 def process_info(self, info_dict):
385 """Process a single dictionary returned by an InfoExtractor."""
387 # Keep for backwards compatibility
388 info_dict['stitle'] = info_dict['title']
390 if not 'format' in info_dict:
391 info_dict['format'] = info_dict['ext']
393 reason = self._match_entry(info_dict)
394 if reason is not None:
395 self.to_screen(u'[download] ' + reason)
398 max_downloads = self.params.get('max_downloads')
399 if max_downloads is not None:
400 if self._num_downloads > int(max_downloads):
401 raise MaxDownloadsReached()
403 filename = self.prepare_filename(info_dict)
406 if self.params.get('forcetitle', False):
407 compat_print(info_dict['title'])
408 if self.params.get('forceurl', False):
409 compat_print(info_dict['url'])
410 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
411 compat_print(info_dict['thumbnail'])
412 if self.params.get('forcedescription', False) and 'description' in info_dict:
413 compat_print(info_dict['description'])
414 if self.params.get('forcefilename', False) and filename is not None:
415 compat_print(filename)
416 if self.params.get('forceformat', False):
417 compat_print(info_dict['format'])
419 # Do nothing else if in simulate mode
420 if self.params.get('simulate', False):
427 dn = os.path.dirname(encodeFilename(filename))
428 if dn != '' and not os.path.exists(dn): # dn is already encoded
430 except (OSError, IOError) as err:
431 self.trouble(u'ERROR: unable to create directory ' + compat_str(err))
434 if self.params.get('writedescription', False):
436 descfn = filename + u'.description'
437 self.report_writedescription(descfn)
438 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
439 descfile.write(info_dict['description'])
440 except (OSError, IOError):
441 self.trouble(u'ERROR: Cannot write description file ' + descfn)
444 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
445 # subtitles download errors are already managed as troubles in relevant IE
446 # that way it will silently go on when used with unsupporting IE
447 subtitle = info_dict['subtitles'][0]
448 (sub_error, sub_lang, sub) = subtitle
449 sub_format = self.params.get('subtitlesformat')
451 sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
452 self.report_writesubtitles(sub_filename)
453 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
455 except (OSError, IOError):
456 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
458 if self.params.get('onlysubtitles', False):
461 if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
462 subtitles = info_dict['subtitles']
463 sub_format = self.params.get('subtitlesformat')
464 for subtitle in subtitles:
465 (sub_error, sub_lang, sub) = subtitle
467 sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
468 self.report_writesubtitles(sub_filename)
469 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
471 except (OSError, IOError):
472 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
474 if self.params.get('onlysubtitles', False):
477 if self.params.get('writeinfojson', False):
478 infofn = filename + u'.info.json'
479 self.report_writeinfojson(infofn)
481 json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
482 write_json_file(json_info_dict, encodeFilename(infofn))
483 except (OSError, IOError):
484 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
487 if not self.params.get('skip_download', False):
488 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
492 success = self._do_download(filename, info_dict)
493 except (OSError, IOError) as err:
494 raise UnavailableVideoError()
495 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
496 self.trouble(u'ERROR: unable to download video data: %s' % str(err))
498 except (ContentTooShortError, ) as err:
499 self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
504 self.post_process(filename, info_dict)
505 except (PostProcessingError) as err:
506 self.trouble(u'ERROR: postprocessing: %s' % str(err))
509 def download(self, url_list):
510 """Download a given list of URLs."""
511 if len(url_list) > 1 and self.fixed_template():
512 raise SameFileError(self.params['outtmpl'])
515 suitable_found = False
517 # Go to next InfoExtractor if not suitable
518 if not ie.suitable(url):
521 # Warn if the _WORKING attribute is False
523 self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, '
524 u'and will probably not work. If you want to go on, use the -i option.')
526 # Suitable InfoExtractor found
527 suitable_found = True
529 # Extract information from URL and process it
531 videos = ie.extract(url)
532 except ExtractorError as de: # An error we somewhat expected
533 self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback())
535 except Exception as e:
536 if self.params.get('ignoreerrors', False):
537 self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc()))
542 if len(videos or []) > 1 and self.fixed_template():
543 raise SameFileError(self.params['outtmpl'])
545 for video in videos or []:
546 video['extractor'] = ie.IE_NAME
548 self.increment_downloads()
549 self.process_info(video)
550 except UnavailableVideoError:
551 self.trouble(u'\nERROR: unable to download video')
553 # Suitable InfoExtractor had been found; go to next URL
556 if not suitable_found:
557 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
559 return self._download_retcode
561 def post_process(self, filename, ie_info):
562 """Run all the postprocessors on the given file."""
564 info['filepath'] = filename
568 keep_video_wish,new_info = pp.run(info)
569 if keep_video_wish is not None:
571 keep_video = keep_video_wish
572 elif keep_video is None:
573 # No clear decision yet, let IE decide
574 keep_video = keep_video_wish
575 except PostProcessingError as e:
576 self.to_stderr(u'ERROR: ' + e.msg)
577 if keep_video is False and not self.params.get('keepvideo', False):
579 self.to_stderr(u'Deleting original file %s (pass -k to keep)' % filename)
580 os.remove(encodeFilename(filename))
581 except (IOError, OSError):
582 self.to_stderr(u'WARNING: Unable to remove downloaded video file')
584 def _download_with_rtmpdump(self, filename, url, player_url, page_url):
585 self.report_destination(filename)
586 tmpfilename = self.temp_name(filename)
588 # Check for rtmpdump first
590 subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
591 except (OSError, IOError):
592 self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
595 # Download using rtmpdump. rtmpdump returns exit code 2 when
596 # the connection was interrumpted and resuming appears to be
597 # possible. This is part of rtmpdump's normal usage, AFAIK.
598 basic_args = ['rtmpdump', '-q', '-r', url, '-o', tmpfilename]
599 if player_url is not None:
600 basic_args += ['-W', player_url]
601 if page_url is not None:
602 basic_args += ['--pageUrl', page_url]
603 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
604 if self.params.get('verbose', False):
607 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
610 self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
611 retval = subprocess.call(args)
612 while retval == 2 or retval == 1:
613 prevsize = os.path.getsize(encodeFilename(tmpfilename))
614 self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
615 time.sleep(5.0) # This seems to be needed
616 retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
617 cursize = os.path.getsize(encodeFilename(tmpfilename))
618 if prevsize == cursize and retval == 1:
620 # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
621 if prevsize == cursize and retval == 2 and cursize > 1024:
622 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
626 fsize = os.path.getsize(encodeFilename(tmpfilename))
627 self.to_screen(u'\r[rtmpdump] %s bytes' % fsize)
628 self.try_rename(tmpfilename, filename)
629 self._hook_progress({
630 'downloaded_bytes': fsize,
631 'total_bytes': fsize,
632 'filename': filename,
633 'status': 'finished',
637 self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
640 def _do_download(self, filename, info_dict):
641 url = info_dict['url']
643 # Check file already present
644 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
645 self.report_file_already_downloaded(filename)
646 self._hook_progress({
647 'filename': filename,
648 'status': 'finished',
652 # Attempt to download using rtmpdump
653 if url.startswith('rtmp'):
654 return self._download_with_rtmpdump(filename, url,
655 info_dict.get('player_url', None),
656 info_dict.get('page_url', None))
658 tmpfilename = self.temp_name(filename)
661 # Do not include the Accept-Encoding header
662 headers = {'Youtubedl-no-compression': 'True'}
663 if 'user_agent' in info_dict:
664 headers['Youtubedl-user-agent'] = info_dict['user_agent']
665 basic_request = compat_urllib_request.Request(url, None, headers)
666 request = compat_urllib_request.Request(url, None, headers)
668 if self.params.get('test', False):
669 request.add_header('Range','bytes=0-10240')
671 # Establish possible resume length
672 if os.path.isfile(encodeFilename(tmpfilename)):
673 resume_len = os.path.getsize(encodeFilename(tmpfilename))
679 if self.params.get('continuedl', False):
680 self.report_resuming_byte(resume_len)
681 request.add_header('Range','bytes=%d-' % resume_len)
687 retries = self.params.get('retries', 0)
688 while count <= retries:
689 # Establish connection
691 if count == 0 and 'urlhandle' in info_dict:
692 data = info_dict['urlhandle']
693 data = compat_urllib_request.urlopen(request)
695 except (compat_urllib_error.HTTPError, ) as err:
696 if (err.code < 500 or err.code >= 600) and err.code != 416:
697 # Unexpected HTTP error
699 elif err.code == 416:
700 # Unable to resume (requested range not satisfiable)
702 # Open the connection again without the range header
703 data = compat_urllib_request.urlopen(basic_request)
704 content_length = data.info()['Content-Length']
705 except (compat_urllib_error.HTTPError, ) as err:
706 if err.code < 500 or err.code >= 600:
709 # Examine the reported length
710 if (content_length is not None and
711 (resume_len - 100 < int(content_length) < resume_len + 100)):
712 # The file had already been fully downloaded.
713 # Explanation to the above condition: in issue #175 it was revealed that
714 # YouTube sometimes adds or removes a few bytes from the end of the file,
715 # changing the file size slightly and causing problems for some users. So
716 # I decided to implement a suggested change and consider the file
717 # completely downloaded if the file size differs less than 100 bytes from
718 # the one in the hard drive.
719 self.report_file_already_downloaded(filename)
720 self.try_rename(tmpfilename, filename)
721 self._hook_progress({
722 'filename': filename,
723 'status': 'finished',
727 # The length does not match, we start the download over
728 self.report_unable_to_resume()
734 self.report_retry(count, retries)
737 self.trouble(u'ERROR: giving up after %s retries' % retries)
740 data_len = data.info().get('Content-length', None)
741 if data_len is not None:
742 data_len = int(data_len) + resume_len
743 min_data_len = self.params.get("min_filesize", None)
744 max_data_len = self.params.get("max_filesize", None)
745 if min_data_len is not None and data_len < min_data_len:
746 self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
748 if max_data_len is not None and data_len > max_data_len:
749 self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
752 data_len_str = self.format_bytes(data_len)
753 byte_counter = 0 + resume_len
754 block_size = self.params.get('buffersize', 1024)
759 data_block = data.read(block_size)
761 if len(data_block) == 0:
763 byte_counter += len(data_block)
765 # Open file just in time
768 (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
769 assert stream is not None
770 filename = self.undo_temp_name(tmpfilename)
771 self.report_destination(filename)
772 except (OSError, IOError) as err:
773 self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
776 stream.write(data_block)
777 except (IOError, OSError) as err:
778 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
780 if not self.params.get('noresizebuffer', False):
781 block_size = self.best_block_size(after - before, len(data_block))
784 speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
786 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
788 percent_str = self.calc_percent(byte_counter, data_len)
789 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
790 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
792 self._hook_progress({
793 'downloaded_bytes': byte_counter,
794 'total_bytes': data_len,
795 'tmpfilename': tmpfilename,
796 'filename': filename,
797 'status': 'downloading',
801 self.slow_down(start, byte_counter - resume_len)
804 self.trouble(u'\nERROR: Did not get any data blocks')
808 if data_len is not None and byte_counter != data_len:
809 raise ContentTooShortError(byte_counter, int(data_len))
810 self.try_rename(tmpfilename, filename)
812 # Update file modification time
813 if self.params.get('updatetime', True):
814 info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
816 self._hook_progress({
817 'downloaded_bytes': byte_counter,
818 'total_bytes': byte_counter,
819 'filename': filename,
820 'status': 'finished',
825 def _hook_progress(self, status):
826 for ph in self._progress_hooks:
829 def add_progress_hook(self, ph):
830 """ ph gets called on download progress, with a dictionary with the entries
831 * filename: The final filename
832 * status: One of "downloading" and "finished"
834 It can also have some of the following entries:
836 * downloaded_bytes: Bytes on disks
837 * total_bytes: Total bytes, None if unknown
838 * tmpfilename: The filename we're currently writing to
840 Hooks are guaranteed to be called at least once (with status "finished")
841 if the download is successful.
843 self._progress_hooks.append(ph)