2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import
22 class FileDownloader(object):
23 """File Downloader class.
25 File downloader objects are the ones responsible of downloading the
26 actual video file and writing it to disk if the user has requested
27 it, among some other tasks. In most cases there should be one per
28 program. As, given a video URL, the downloader doesn't know how to
29 extract all the needed information, task that InfoExtractors do, it
30 has to pass the URL to one of them.
32 For this, file downloader objects have a method that allows
33 InfoExtractors to be registered in a given order. When it is passed
34 a URL, the file downloader handles it to the first InfoExtractor it
35 finds that reports being able to handle it. The InfoExtractor extracts
36 all the information about the video or videos the URL refers to, and
37 asks the FileDownloader to process the video information, possibly
38 downloading the video.
40 File downloaders accept a lot of parameters. In order not to saturate
41 the object constructor with arguments, it receives a dictionary of
42 options instead. These options are available through the params
43 attribute for the InfoExtractors to use. The FileDownloader also
44 registers itself as the downloader in charge for the InfoExtractors
45 that are added to it, so this is a "mutual registration".
49 username: Username for authentication purposes.
50 password: Password for authentication purposes.
51 usenetrc: Use netrc for authentication instead.
52 quiet: Do not print messages to stdout.
53 forceurl: Force printing final URL.
54 forcetitle: Force printing title.
55 forcethumbnail: Force printing thumbnail URL.
56 forcedescription: Force printing description.
57 forcefilename: Force printing final filename.
58 simulate: Do not download the video files.
59 format: Video format code.
60 format_limit: Highest quality format to try.
61 outtmpl: Template for output names.
62 restrictfilenames: Do not allow "&" and spaces in file names
63 ignoreerrors: Do not stop on download errors.
64 ratelimit: Download speed limit, in bytes/sec.
65 nooverwrites: Prevent overwriting files.
66 retries: Number of times to retry for HTTP error 5xx
67 buffersize: Size of download buffer in bytes.
68 noresizebuffer: Do not automatically resize the download buffer.
69 continuedl: Try to continue downloads if possible.
70 noprogress: Do not print the progress bar.
71 playliststart: Playlist item to start at.
72 playlistend: Playlist item to end at.
73 matchtitle: Download only matching titles.
74 rejecttitle: Reject downloads for matching titles.
75 logtostderr: Log messages to stderr instead of stdout.
76 consoletitle: Display progress in console window's titlebar.
77 nopart: Do not use temporary .part files.
78 updatetime: Use the Last-modified header to set output file timestamps.
79 writedescription: Write the video description to a .description file
80 writeinfojson: Write the video description to a .info.json file
81 writesubtitles: Write the video subtitles to a .srt file
82 subtitleslang: Language of the subtitles to download
83 test: Download only first bytes to test the downloader.
84 keepvideo: Keep the video file after post-processing
85 min_filesize: Skip files smaller than this size
86 max_filesize: Skip files larger than this size
92 _download_retcode = None
96 def __init__(self, params):
97 """Create a FileDownloader object with the given options."""
100 self._progress_hooks = []
101 self._download_retcode = 0
102 self._num_downloads = 0
103 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
106 if '%(stitle)s' in self.params['outtmpl']:
107 self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
110 def format_bytes(bytes):
113 if type(bytes) is str:
118 exponent = int(math.log(bytes, 1024.0))
119 suffix = 'bkMGTPEZY'[exponent]
120 converted = float(bytes) / float(1024 ** exponent)
121 return '%.2f%s' % (converted, suffix)
124 def calc_percent(byte_counter, data_len):
127 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
130 def calc_eta(start, now, total, current):
134 if current == 0 or dif < 0.001: # One millisecond
136 rate = float(current) / dif
137 eta = int((float(total) - float(current)) / rate)
138 (eta_mins, eta_secs) = divmod(eta, 60)
141 return '%02d:%02d' % (eta_mins, eta_secs)
144 def calc_speed(start, now, bytes):
146 if bytes == 0 or dif < 0.001: # One millisecond
147 return '%10s' % '---b/s'
148 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
151 def best_block_size(elapsed_time, bytes):
152 new_min = max(bytes / 2.0, 1.0)
153 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
154 if elapsed_time < 0.001:
156 rate = bytes / elapsed_time
164 def parse_bytes(bytestr):
165 """Parse a string indicating a byte quantity into an integer."""
166 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
169 number = float(matchobj.group(1))
170 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
171 return int(round(number * multiplier))
173 def add_info_extractor(self, ie):
174 """Add an InfoExtractor object to the end of the list."""
176 ie.set_downloader(self)
178 def add_post_processor(self, pp):
179 """Add a PostProcessor object to the end of the chain."""
181 pp.set_downloader(self)
183 def to_screen(self, message, skip_eol=False):
184 """Print message to stdout if not in quiet mode."""
185 assert type(message) == type(u'')
186 if not self.params.get('quiet', False):
187 terminator = [u'\n', u''][skip_eol]
188 output = message + terminator
189 if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
190 output = output.encode(preferredencoding(), 'ignore')
191 self._screen_file.write(output)
192 self._screen_file.flush()
194 def to_stderr(self, message):
195 """Print message to stderr."""
196 assert type(message) == type(u'')
197 output = message + u'\n'
198 if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
199 output = output.encode(preferredencoding())
200 sys.stderr.write(output)
202 def to_cons_title(self, message):
203 """Set console/terminal window title to message."""
204 if not self.params.get('consoletitle', False):
206 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
207 # c_wchar_p() might not be necessary if `message` is
208 # already of type unicode()
209 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
210 elif 'TERM' in os.environ:
211 self.to_screen('\033]0;%s\007' % message, skip_eol=True)
213 def fixed_template(self):
214 """Checks if the output template is fixed."""
215 return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
217 def trouble(self, message=None, tb=None):
218 """Determine action to take when a download problem appears.
220 Depending on if the downloader has been configured to ignore
221 download errors or not, this method may throw an exception or
222 not when errors are found, after printing the message.
224 tb, if given, is additional traceback information.
226 if message is not None:
227 self.to_stderr(message)
228 if self.params.get('verbose'):
230 tb_data = traceback.format_list(traceback.extract_stack())
231 tb = u''.join(tb_data)
233 if not self.params.get('ignoreerrors', False):
234 raise DownloadError(message)
235 self._download_retcode = 1
237 def report_warning(self, message):
239 Print the message to stderr, it will be prefixed with 'WARNING:'
240 If stderr is a tty file the 'WARNING:' will be colored
242 if sys.stderr.isatty():
243 _msg_header=u'\033[0;33mWARNING:\033[0m'
245 _msg_header=u'WARNING:'
246 warning_message=u'%s %s' % (_msg_header,message)
247 self.to_stderr(warning_message)
249 def report_error(self, message, tb=None):
251 Do the same as trouble, but prefixes the message with 'ERROR:', colored
252 in red if stderr is a tty file.
254 if sys.stderr.isatty():
255 _msg_header = u'\033[0;31mERROR:\033[0m'
257 _msg_header = u'ERROR:'
258 error_message = u'%s %s' % (_msg_header, message)
259 self.trouble(error_message, tb)
261 def slow_down(self, start_time, byte_counter):
262 """Sleep if the download speed is over the rate limit."""
263 rate_limit = self.params.get('ratelimit', None)
264 if rate_limit is None or byte_counter == 0:
267 elapsed = now - start_time
270 speed = float(byte_counter) / elapsed
271 if speed > rate_limit:
272 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
274 def temp_name(self, filename):
275 """Returns a temporary filename for the given filename."""
276 if self.params.get('nopart', False) or filename == u'-' or \
277 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
279 return filename + u'.part'
281 def undo_temp_name(self, filename):
282 if filename.endswith(u'.part'):
283 return filename[:-len(u'.part')]
286 def try_rename(self, old_filename, new_filename):
288 if old_filename == new_filename:
290 os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
291 except (IOError, OSError) as err:
292 self.trouble(u'ERROR: unable to rename file')
294 def try_utime(self, filename, last_modified_hdr):
295 """Try to set the last-modified time of the given file."""
296 if last_modified_hdr is None:
298 if not os.path.isfile(encodeFilename(filename)):
300 timestr = last_modified_hdr
303 filetime = timeconvert(timestr)
307 os.utime(filename, (time.time(), filetime))
312 def report_writedescription(self, descfn):
313 """ Report that the description file is being written """
314 self.to_screen(u'[info] Writing video description to: ' + descfn)
316 def report_writesubtitles(self, srtfn):
317 """ Report that the subtitles file is being written """
318 self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
320 def report_writeinfojson(self, infofn):
321 """ Report that the metadata file has been written """
322 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
324 def report_destination(self, filename):
325 """Report destination filename."""
326 self.to_screen(u'[download] Destination: ' + filename)
328 def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
329 """Report download progress."""
330 if self.params.get('noprogress', False):
332 if self.params.get('progress_with_newline', False):
333 self.to_screen(u'[download] %s of %s at %s ETA %s' %
334 (percent_str, data_len_str, speed_str, eta_str))
336 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
337 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
338 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
339 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
341 def report_resuming_byte(self, resume_len):
342 """Report attempt to resume at given byte."""
343 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
345 def report_retry(self, count, retries):
346 """Report retry in case of HTTP error 5xx"""
347 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
349 def report_file_already_downloaded(self, file_name):
350 """Report file has already been fully downloaded."""
352 self.to_screen(u'[download] %s has already been downloaded' % file_name)
353 except (UnicodeEncodeError) as err:
354 self.to_screen(u'[download] The file has already been downloaded')
356 def report_unable_to_resume(self):
357 """Report it was impossible to resume download."""
358 self.to_screen(u'[download] Unable to resume')
360 def report_finish(self):
361 """Report download finished."""
362 if self.params.get('noprogress', False):
363 self.to_screen(u'[download] Download completed')
367 def increment_downloads(self):
368 """Increment the ordinal that assigns a number to each file."""
369 self._num_downloads += 1
371 def prepare_filename(self, info_dict):
372 """Generate the output filename."""
374 template_dict = dict(info_dict)
376 template_dict['epoch'] = int(time.time())
377 template_dict['autonumber'] = u'%05d' % self._num_downloads
379 sanitize = lambda k,v: sanitize_filename(
380 u'NA' if v is None else compat_str(v),
381 restricted=self.params.get('restrictfilenames'),
383 template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
385 filename = self.params['outtmpl'] % template_dict
387 except (ValueError, KeyError) as err:
388 self.trouble(u'ERROR: invalid system charset or erroneous output template')
391 def _match_entry(self, info_dict):
392 """ Returns None iff the file should be downloaded """
394 title = info_dict['title']
395 matchtitle = self.params.get('matchtitle', False)
397 if not re.search(matchtitle, title, re.IGNORECASE):
398 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
399 rejecttitle = self.params.get('rejecttitle', False)
401 if re.search(rejecttitle, title, re.IGNORECASE):
402 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
405 def process_info(self, info_dict):
406 """Process a single dictionary returned by an InfoExtractor."""
408 # Keep for backwards compatibility
409 info_dict['stitle'] = info_dict['title']
411 if not 'format' in info_dict:
412 info_dict['format'] = info_dict['ext']
414 reason = self._match_entry(info_dict)
415 if reason is not None:
416 self.to_screen(u'[download] ' + reason)
419 max_downloads = self.params.get('max_downloads')
420 if max_downloads is not None:
421 if self._num_downloads > int(max_downloads):
422 raise MaxDownloadsReached()
424 filename = self.prepare_filename(info_dict)
427 if self.params.get('forcetitle', False):
428 compat_print(info_dict['title'])
429 if self.params.get('forceurl', False):
430 compat_print(info_dict['url'])
431 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
432 compat_print(info_dict['thumbnail'])
433 if self.params.get('forcedescription', False) and 'description' in info_dict:
434 compat_print(info_dict['description'])
435 if self.params.get('forcefilename', False) and filename is not None:
436 compat_print(filename)
437 if self.params.get('forceformat', False):
438 compat_print(info_dict['format'])
440 # Do nothing else if in simulate mode
441 if self.params.get('simulate', False):
448 dn = os.path.dirname(encodeFilename(filename))
449 if dn != '' and not os.path.exists(dn): # dn is already encoded
451 except (OSError, IOError) as err:
452 self.trouble(u'ERROR: unable to create directory ' + compat_str(err))
455 if self.params.get('writedescription', False):
457 descfn = filename + u'.description'
458 self.report_writedescription(descfn)
459 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
460 descfile.write(info_dict['description'])
461 except (OSError, IOError):
462 self.trouble(u'ERROR: Cannot write description file ' + descfn)
465 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
466 # subtitles download errors are already managed as troubles in relevant IE
467 # that way it will silently go on when used with unsupporting IE
469 srtfn = filename.rsplit('.', 1)[0] + u'.srt'
470 self.report_writesubtitles(srtfn)
471 with io.open(encodeFilename(srtfn), 'w', encoding='utf-8') as srtfile:
472 srtfile.write(info_dict['subtitles'])
473 except (OSError, IOError):
474 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
477 if self.params.get('writeinfojson', False):
478 infofn = filename + u'.info.json'
479 self.report_writeinfojson(infofn)
481 json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
482 write_json_file(json_info_dict, encodeFilename(infofn))
483 except (OSError, IOError):
484 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
487 if not self.params.get('skip_download', False):
488 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
492 success = self._do_download(filename, info_dict)
493 except (OSError, IOError) as err:
494 raise UnavailableVideoError()
495 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
496 self.trouble(u'ERROR: unable to download video data: %s' % str(err))
498 except (ContentTooShortError, ) as err:
499 self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
504 self.post_process(filename, info_dict)
505 except (PostProcessingError) as err:
506 self.trouble(u'ERROR: postprocessing: %s' % str(err))
509 def download(self, url_list):
510 """Download a given list of URLs."""
511 if len(url_list) > 1 and self.fixed_template():
512 raise SameFileError(self.params['outtmpl'])
515 suitable_found = False
517 # Go to next InfoExtractor if not suitable
518 if not ie.suitable(url):
521 # Warn if the _WORKING attribute is False
523 self.report_warning(u'the program functionality for this site has been marked as broken, '
524 u'and will probably not work. If you want to go on, use the -i option.')
526 # Suitable InfoExtractor found
527 suitable_found = True
529 # Extract information from URL and process it
531 videos = ie.extract(url)
532 except ExtractorError as de: # An error we somewhat expected
533 self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback())
535 except Exception as e:
536 if self.params.get('ignoreerrors', False):
537 self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc()))
542 if len(videos or []) > 1 and self.fixed_template():
543 raise SameFileError(self.params['outtmpl'])
545 for video in videos or []:
546 video['extractor'] = ie.IE_NAME
548 self.increment_downloads()
549 self.process_info(video)
550 except UnavailableVideoError:
551 self.trouble(u'\nERROR: unable to download video')
553 # Suitable InfoExtractor had been found; go to next URL
556 if not suitable_found:
557 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
559 return self._download_retcode
561 def post_process(self, filename, ie_info):
562 """Run all the postprocessors on the given file."""
564 info['filepath'] = filename
568 keep_video_wish,new_info = pp.run(info)
569 if keep_video_wish is not None:
571 keep_video = keep_video_wish
572 elif keep_video is None:
573 # No clear decision yet, let IE decide
574 keep_video = keep_video_wish
575 except PostProcessingError as e:
576 self.to_stderr(u'ERROR: ' + e.msg)
577 if keep_video is False and not self.params.get('keepvideo', False):
579 self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename)
580 os.remove(encodeFilename(filename))
581 except (IOError, OSError):
582 self.report_warning(u'Unable to remove downloaded video file')
584 def _download_with_rtmpdump(self, filename, url, player_url, page_url):
585 self.report_destination(filename)
586 tmpfilename = self.temp_name(filename)
588 # Check for rtmpdump first
590 subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
591 except (OSError, IOError):
592 self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
595 # Download using rtmpdump. rtmpdump returns exit code 2 when
596 # the connection was interrumpted and resuming appears to be
597 # possible. This is part of rtmpdump's normal usage, AFAIK.
598 basic_args = ['rtmpdump', '-q', '-r', url, '-o', tmpfilename]
599 if player_url is not None:
600 basic_args += ['-W', player_url]
601 if page_url is not None:
602 basic_args += ['--pageUrl', page_url]
603 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
604 if self.params.get('verbose', False):
607 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
610 self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
611 retval = subprocess.call(args)
612 while retval == 2 or retval == 1:
613 prevsize = os.path.getsize(encodeFilename(tmpfilename))
614 self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
615 time.sleep(5.0) # This seems to be needed
616 retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
617 cursize = os.path.getsize(encodeFilename(tmpfilename))
618 if prevsize == cursize and retval == 1:
620 # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
621 if prevsize == cursize and retval == 2 and cursize > 1024:
622 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
626 fsize = os.path.getsize(encodeFilename(tmpfilename))
627 self.to_screen(u'\r[rtmpdump] %s bytes' % fsize)
628 self.try_rename(tmpfilename, filename)
629 self._hook_progress({
630 'downloaded_bytes': fsize,
631 'total_bytes': fsize,
632 'filename': filename,
633 'status': 'finished',
637 self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
640 def _do_download(self, filename, info_dict):
641 url = info_dict['url']
643 # Check file already present
644 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
645 self.report_file_already_downloaded(filename)
646 self._hook_progress({
647 'filename': filename,
648 'status': 'finished',
652 # Attempt to download using rtmpdump
653 if url.startswith('rtmp'):
654 return self._download_with_rtmpdump(filename, url,
655 info_dict.get('player_url', None),
656 info_dict.get('page_url', None))
658 tmpfilename = self.temp_name(filename)
661 # Do not include the Accept-Encoding header
662 headers = {'Youtubedl-no-compression': 'True'}
663 if 'user_agent' in info_dict:
664 headers['Youtubedl-user-agent'] = info_dict['user_agent']
665 basic_request = compat_urllib_request.Request(url, None, headers)
666 request = compat_urllib_request.Request(url, None, headers)
668 if self.params.get('test', False):
669 request.add_header('Range','bytes=0-10240')
671 # Establish possible resume length
672 if os.path.isfile(encodeFilename(tmpfilename)):
673 resume_len = os.path.getsize(encodeFilename(tmpfilename))
679 if self.params.get('continuedl', False):
680 self.report_resuming_byte(resume_len)
681 request.add_header('Range','bytes=%d-' % resume_len)
687 retries = self.params.get('retries', 0)
688 while count <= retries:
689 # Establish connection
691 if count == 0 and 'urlhandle' in info_dict:
692 data = info_dict['urlhandle']
693 data = compat_urllib_request.urlopen(request)
695 except (compat_urllib_error.HTTPError, ) as err:
696 if (err.code < 500 or err.code >= 600) and err.code != 416:
697 # Unexpected HTTP error
699 elif err.code == 416:
700 # Unable to resume (requested range not satisfiable)
702 # Open the connection again without the range header
703 data = compat_urllib_request.urlopen(basic_request)
704 content_length = data.info()['Content-Length']
705 except (compat_urllib_error.HTTPError, ) as err:
706 if err.code < 500 or err.code >= 600:
709 # Examine the reported length
710 if (content_length is not None and
711 (resume_len - 100 < int(content_length) < resume_len + 100)):
712 # The file had already been fully downloaded.
713 # Explanation to the above condition: in issue #175 it was revealed that
714 # YouTube sometimes adds or removes a few bytes from the end of the file,
715 # changing the file size slightly and causing problems for some users. So
716 # I decided to implement a suggested change and consider the file
717 # completely downloaded if the file size differs less than 100 bytes from
718 # the one in the hard drive.
719 self.report_file_already_downloaded(filename)
720 self.try_rename(tmpfilename, filename)
721 self._hook_progress({
722 'filename': filename,
723 'status': 'finished',
727 # The length does not match, we start the download over
728 self.report_unable_to_resume()
734 self.report_retry(count, retries)
737 self.trouble(u'ERROR: giving up after %s retries' % retries)
740 data_len = data.info().get('Content-length', None)
741 if data_len is not None:
742 data_len = int(data_len) + resume_len
743 min_data_len = self.params.get("min_filesize", None)
744 max_data_len = self.params.get("max_filesize", None)
745 if min_data_len is not None and data_len < min_data_len:
746 self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
748 if max_data_len is not None and data_len > max_data_len:
749 self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
752 data_len_str = self.format_bytes(data_len)
753 byte_counter = 0 + resume_len
754 block_size = self.params.get('buffersize', 1024)
759 data_block = data.read(block_size)
761 if len(data_block) == 0:
763 byte_counter += len(data_block)
765 # Open file just in time
768 (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
769 assert stream is not None
770 filename = self.undo_temp_name(tmpfilename)
771 self.report_destination(filename)
772 except (OSError, IOError) as err:
773 self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
776 stream.write(data_block)
777 except (IOError, OSError) as err:
778 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
780 if not self.params.get('noresizebuffer', False):
781 block_size = self.best_block_size(after - before, len(data_block))
784 speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
786 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
788 percent_str = self.calc_percent(byte_counter, data_len)
789 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
790 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
792 self._hook_progress({
793 'downloaded_bytes': byte_counter,
794 'total_bytes': data_len,
795 'tmpfilename': tmpfilename,
796 'filename': filename,
797 'status': 'downloading',
801 self.slow_down(start, byte_counter - resume_len)
804 self.trouble(u'\nERROR: Did not get any data blocks')
808 if data_len is not None and byte_counter != data_len:
809 raise ContentTooShortError(byte_counter, int(data_len))
810 self.try_rename(tmpfilename, filename)
812 # Update file modification time
813 if self.params.get('updatetime', True):
814 info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
816 self._hook_progress({
817 'downloaded_bytes': byte_counter,
818 'total_bytes': byte_counter,
819 'filename': filename,
820 'status': 'finished',
825 def _hook_progress(self, status):
826 for ph in self._progress_hooks:
829 def add_progress_hook(self, ph):
830 """ ph gets called on download progress, with a dictionary with the entries
831 * filename: The final filename
832 * status: One of "downloading" and "finished"
834 It can also have some of the following entries:
836 * downloaded_bytes: Bytes on disks
837 * total_bytes: Total bytes, None if unknown
838 * tmpfilename: The filename we're currently writing to
840 Hooks are guaranteed to be called at least once (with status "finished")
841 if the download is successful.
843 self._progress_hooks.append(ph)