+from __future__ import unicode_literals
+
import os
import re
import sys
Subclasses of this one must re-define the real_download method.
"""
+ _TEST_FILE_SIZE = 10241
params = None
def __init__(self, ydl, params):
def calc_eta(start, now, total, current):
if total is None:
return None
+ if now is None:
+ now = time.time()
dif = now - start
if current == 0 or dif < 0.001: # One millisecond
return None
def report_error(self, *args, **kargs):
self.ydl.report_error(*args, **kargs)
- def slow_down(self, start_time, byte_counter):
+ def slow_down(self, start_time, now, byte_counter):
"""Sleep if the download speed is over the rate limit."""
rate_limit = self.params.get('ratelimit', None)
if rate_limit is None or byte_counter == 0:
return
- now = time.time()
+ if now is None:
+ now = time.time()
elapsed = now - start_time
if elapsed <= 0.0:
return
speed = float(byte_counter) / elapsed
if speed > rate_limit:
- time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
+ time.sleep(max((byte_counter / rate_limit) - elapsed, 0))
def temp_name(self, filename):
"""Returns a temporary filename for the given filename."""
- if self.params.get('nopart', False) or filename == u'-' or \
+ if self.params.get('nopart', False) or filename == '-' or \
(os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
return filename
- return filename + u'.part'
+ return filename + '.part'
def undo_temp_name(self, filename):
- if filename.endswith(u'.part'):
- return filename[:-len(u'.part')]
+ if filename.endswith('.part'):
+ return filename[:-len('.part')]
return filename
def try_rename(self, old_filename, new_filename):
return
os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
except (IOError, OSError) as err:
- self.report_error(u'unable to rename file: %s' % compat_str(err))
+ self.report_error('unable to rename file: %s' % compat_str(err))
def try_utime(self, filename, last_modified_hdr):
"""Try to set the last-modified time of the given file."""
def report_destination(self, filename):
"""Report destination filename."""
- self.to_screen(u'[download] Destination: ' + filename)
+ self.to_screen('[download] Destination: ' + filename)
def _report_progress_status(self, msg, is_last_line=False):
- fullmsg = u'[download] ' + msg
+ fullmsg = '[download] ' + msg
if self.params.get('progress_with_newline', False):
self.to_screen(fullmsg)
else:
prev_len = getattr(self, '_report_progress_prev_line_length',
0)
if prev_len > len(fullmsg):
- fullmsg += u' ' * (prev_len - len(fullmsg))
+ fullmsg += ' ' * (prev_len - len(fullmsg))
self._report_progress_prev_line_length = len(fullmsg)
- clear_line = u'\r'
+ clear_line = '\r'
else:
- clear_line = (u'\r\x1b[K' if sys.stderr.isatty() else u'\r')
+ clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r')
self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
- self.to_console_title(u'youtube-dl ' + msg)
+ self.to_console_title('youtube-dl ' + msg)
def report_progress(self, percent, data_len_str, speed, eta):
"""Report download progress."""
percent_str = 'Unknown %'
speed_str = self.format_speed(speed)
- msg = (u'%s of %s at %s ETA %s' %
+ msg = ('%s of %s at %s ETA %s' %
(percent_str, data_len_str, speed_str, eta_str))
self._report_progress_status(msg)
downloaded_str = format_bytes(downloaded_data_len)
speed_str = self.format_speed(speed)
elapsed_str = FileDownloader.format_seconds(elapsed)
- msg = u'%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str)
+ msg = '%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str)
self._report_progress_status(msg)
def report_finish(self, data_len_str, tot_time):
"""Report download finished."""
if self.params.get('noprogress', False):
- self.to_screen(u'[download] Download completed')
+ self.to_screen('[download] Download completed')
else:
self._report_progress_status(
- (u'100%% of %s in %s' %
+ ('100%% of %s in %s' %
(data_len_str, self.format_seconds(tot_time))),
is_last_line=True)
def report_resuming_byte(self, resume_len):
"""Report attempt to resume at given byte."""
- self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
+ self.to_screen('[download] Resuming download at byte %s' % resume_len)
def report_retry(self, count, retries):
"""Report retry in case of HTTP error 5xx"""
- self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
+ self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
def report_file_already_downloaded(self, file_name):
"""Report file has already been fully downloaded."""
try:
- self.to_screen(u'[download] %s has already been downloaded' % file_name)
+ self.to_screen('[download] %s has already been downloaded' % file_name)
except UnicodeEncodeError:
- self.to_screen(u'[download] The file has already been downloaded')
+ self.to_screen('[download] The file has already been downloaded')
def report_unable_to_resume(self):
"""Report it was impossible to resume download."""
- self.to_screen(u'[download] Unable to resume')
+ self.to_screen('[download] Unable to resume')
def download(self, filename, info_dict):
"""Download to a filename using the info from info_dict
def real_download(self, filename, info_dict):
"""Real download process. Redefine in subclasses."""
- raise NotImplementedError(u'This method must be implemented by sublcasses')
+ raise NotImplementedError('This method must be implemented by subclasses')
def _hook_progress(self, status):
for ph in self._progress_hooks:
+from __future__ import unicode_literals
+
import os
import time
class HttpFD(FileDownloader):
- _TEST_FILE_SIZE = 10241
-
def real_download(self, filename, info_dict):
url = info_dict['url']
tmpfilename = self.temp_name(filename)
headers['Youtubedl-user-agent'] = info_dict['user_agent']
if 'http_referer' in info_dict:
headers['Referer'] = info_dict['http_referer']
- basic_request = compat_urllib_request.Request(url, None, headers)
- request = compat_urllib_request.Request(url, None, headers)
+ add_headers = info_dict.get('http_headers')
+ if add_headers:
+ headers.update(add_headers)
+ data = info_dict.get('http_post_data')
+ http_method = info_dict.get('http_method')
+ basic_request = compat_urllib_request.Request(url, data, headers)
+ request = compat_urllib_request.Request(url, data, headers)
+ if http_method is not None:
+ basic_request.get_method = lambda: http_method
+ request.get_method = lambda: http_method
is_test = self.params.get('test', False)
self.report_retry(count, retries)
if count > retries:
- self.report_error(u'giving up after %s retries' % retries)
+ self.report_error('giving up after %s retries' % retries)
return False
data_len = data.info().get('Content-length', None)
min_data_len = self.params.get("min_filesize", None)
max_data_len = self.params.get("max_filesize", None)
if min_data_len is not None and data_len < min_data_len:
- self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
+ self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
return False
if max_data_len is not None and data_len > max_data_len:
- self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
+ self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
return False
data_len_str = format_bytes(data_len)
byte_counter = 0 + resume_len
block_size = self.params.get('buffersize', 1024)
start = time.time()
+
+ # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
+ now = None # needed for slow_down() in the first loop run
+ before = start # start measuring
while True:
+
# Download and write
- before = time.time()
data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
- after = time.time()
+ byte_counter += len(data_block)
+
+ # exit loop when download is finished
if len(data_block) == 0:
break
- byte_counter += len(data_block)
- # Open file just in time
+ # Open destination file just in time
if stream is None:
try:
(stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
filename = self.undo_temp_name(tmpfilename)
self.report_destination(filename)
except (OSError, IOError) as err:
- self.report_error(u'unable to open for writing: %s' % str(err))
+ self.report_error('unable to open for writing: %s' % str(err))
return False
try:
stream.write(data_block)
except (IOError, OSError) as err:
- self.to_stderr(u"\n")
- self.report_error(u'unable to write data: %s' % str(err))
+ self.to_stderr('\n')
+ self.report_error('unable to write data: %s' % str(err))
return False
+
+ # Apply rate limit
+ self.slow_down(start, now, byte_counter - resume_len)
+
+ # end measuring of one loop run
+ now = time.time()
+ after = now
+
+ # Adjust block size
if not self.params.get('noresizebuffer', False):
block_size = self.best_block_size(after - before, len(data_block))
+ before = after
+
# Progress message
- speed = self.calc_speed(start, time.time(), byte_counter - resume_len)
+ speed = self.calc_speed(start, now, byte_counter - resume_len)
if data_len is None:
eta = percent = None
else:
if is_test and byte_counter == data_len:
break
- # Apply rate limit
- self.slow_down(start, byte_counter - resume_len)
-
if stream is None:
- self.to_stderr(u"\n")
- self.report_error(u'Did not get any data blocks')
+ self.to_stderr('\n')
+ self.report_error('Did not get any data blocks')
return False
- stream.close()
+ if tmpfilename != '-':
+ stream.close()
self.report_finish(data_len_str, (time.time() - start))
if data_len is not None and byte_counter != data_len:
raise ContentTooShortError(byte_counter, int(data_len))
projects / youtube-dl / commitdiff