if numeric_limit is None:
parser.error('invalid max_filesize specified')
opts.max_filesize = numeric_limit
- if opts.retries is not None:
- if opts.retries in ('inf', 'infinite'):
- opts_retries = float('inf')
+
+ def parse_retries(retries):
+ if retries in ('inf', 'infinite'):
+ parsed_retries = float('inf')
else:
try:
- opts_retries = int(opts.retries)
+ parsed_retries = int(retries)
except (TypeError, ValueError):
parser.error('invalid retry count specified')
+ return parsed_retries
+ if opts.retries is not None:
+ opts.retries = parse_retries(opts.retries)
+ if opts.fragment_retries is not None:
+ opts.fragment_retries = parse_retries(opts.fragment_retries)
if opts.buffersize is not None:
numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
if numeric_buffersize is None:
'force_generic_extractor': opts.force_generic_extractor,
'ratelimit': opts.ratelimit,
'nooverwrites': opts.nooverwrites,
- 'retries': opts_retries,
+ 'retries': opts.retries,
+ 'fragment_retries': opts.fragment_retries,
'buffersize': opts.buffersize,
'noresizebuffer': opts.noresizebuffer,
'continuedl': opts.continue_dl,
return '%10s' % '---b/s'
return '%10s' % ('%s/s' % format_bytes(speed))
+ @staticmethod
+ def format_retries(retries):
+ return 'inf' if retries == float('inf') else '%.0f' % retries
+
@staticmethod
def best_block_size(elapsed_time, bytes):
new_min = max(bytes / 2.0, 1.0)
def report_retry(self, count, retries):
"""Report retry in case of HTTP error 5xx"""
- self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %.0f)...' % (count, retries))
+ self.to_screen(
+ '[download] Got server HTTP error. Retrying (attempt %d of %s)...'
+ % (count, self.format_retries(retries)))
def report_file_already_downloaded(self, file_name):
"""Report file has already been fully downloaded."""
import re
from .fragment import FragmentFD
+from ..compat import compat_urllib_error
from ..utils import (
sanitize_open,
encodeFilename,
segments_filenames = []
- def append_url_to_file(target_url, target_filename):
- success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
- if not success:
+ fragment_retries = self.params.get('fragment_retries', 0)
+
+ def append_url_to_file(target_url, tmp_filename, segment_name):
+ target_filename = '%s-%s' % (tmp_filename, segment_name)
+ count = 0
+ while count <= fragment_retries:
+ try:
+ success = ctx['dl'].download(target_filename, {'url': combine_url(base_url, target_url)})
+ if not success:
+ return False
+ down, target_sanitized = sanitize_open(target_filename, 'rb')
+ ctx['dest_stream'].write(down.read())
+ down.close()
+ segments_filenames.append(target_sanitized)
+ break
+ except (compat_urllib_error.HTTPError, ) as err:
+ # YouTube may often return 404 HTTP error for a fragment causing the
+ # whole download to fail. However if the same fragment is immediately
+ # retried with the same request data this usually succeeds (1-2 attemps
+ # is usually enough) thus allowing to download the whole file successfully.
+ # So, we will retry all fragments that fail with 404 HTTP error for now.
+ if err.code != 404:
+ raise
+ # Retry fragment
+ count += 1
+ if count <= fragment_retries:
+ self.report_retry_fragment(segment_name, count, fragment_retries)
+ if count > fragment_retries:
+ self.report_error('giving up after %s fragment retries' % fragment_retries)
return False
- down, target_sanitized = sanitize_open(target_filename, 'rb')
- ctx['dest_stream'].write(down.read())
- down.close()
- segments_filenames.append(target_sanitized)
if initialization_url:
- append_url_to_file(initialization_url, ctx['tmpfilename'] + '-Init')
+ append_url_to_file(initialization_url, ctx['tmpfilename'], 'Init')
for i, segment_url in enumerate(segment_urls):
- segment_filename = '%s-Seg%d' % (ctx['tmpfilename'], i)
- append_url_to_file(segment_url, segment_filename)
+ append_url_to_file(segment_url, ctx['tmpfilename'], 'Seg%d' % i)
self._finish_frag_download(ctx)
class FragmentFD(FileDownloader):
"""
A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).
+
+ Available options:
+
+ fragment_retries: Number of times to retry a fragment for HTTP error (DASH only)
"""
+ def report_retry_fragment(self, fragment_name, count, retries):
+ self.to_screen(
+ '[download] Got server HTTP error. Retrying fragment %s (attempt %d of %s)...'
+ % (fragment_name, count, self.format_retries(retries)))
+
def _prepare_and_start_frag_download(self, ctx):
self._prepare_frag_download(ctx)
self._start_frag_download(ctx)
'-R', '--retries',
dest='retries', metavar='RETRIES', default=10,
help='Number of retries (default is %default), or "infinite".')
+ downloader.add_option(
+ '--fragment-retries',
+ dest='fragment_retries', metavar='RETRIES', default=10,
+ help='Number of retries for a fragment (default is %default), or "infinite" (DASH only)')
downloader.add_option(
'--buffer-size',
dest='buffersize', metavar='SIZE', default='1024',