_ Git - youtube-dl/blob - youtube_dl/downloader/fragment.py

   1 from __future__ import division, unicode_literals
   2
   3 import os
   4 import time
   5 import io
   6
   7 from .common import FileDownloader
   8 from .http import HttpFD
   9 from ..utils import (
  10     error_to_compat_str,
  11     encodeFilename,
  12     sanitize_open,
  13     sanitized_Request,
  14     compat_str,
  15 )
  16
  17
  18 class HttpQuietDownloader(HttpFD):
  19     def to_screen(self, *args, **kargs):
  20         pass
  21
  22
  23 class FragmentFD(FileDownloader):
  24     """
  25     A base file downloader class for fragmented media (e.g. f4m/m3u8 manifests).
  26
  27     Available options:
  28
  29     fragment_retries:   Number of times to retry a fragment for HTTP error (DASH
  30                         and hlsnative only)
  31     skip_unavailable_fragments:
  32                         Skip unavailable fragments (DASH and hlsnative only)
  33     """
  34
  35     def report_retry_fragment(self, err, frag_index, count, retries):
  36         self.to_screen(
  37             '[download] Got server HTTP error: %s. Retrying fragment %d (attempt %d of %s)...'
  38             % (error_to_compat_str(err), frag_index, count, self.format_retries(retries)))
  39
  40     def report_skip_fragment(self, frag_index):
  41         self.to_screen('[download] Skipping fragment %d...' % frag_index)
  42
  43     def _prepare_url(self, info_dict, url):
  44         headers = info_dict.get('http_headers')
  45         return sanitized_Request(url, None, headers) if headers else url
  46
  47     def _prepare_and_start_frag_download(self, ctx):
  48         self._prepare_frag_download(ctx)
  49         self._start_frag_download(ctx)
  50
  51     def _download_fragment(self, ctx, frag_url, info_dict, headers=None):
  52         down = io.BytesIO()
  53         success = ctx['dl'].download(down, {
  54             'url': frag_url,
  55             'http_headers': headers or info_dict.get('http_headers'),
  56         })
  57         if not success:
  58             return False, None
  59         frag_content = down.getvalue()
  60         down.close()
  61         return True, frag_content
  62
  63     def _append_fragment(self, ctx, frag_content):
  64         ctx['dest_stream'].write(frag_content)
  65         if not (ctx.get('live') or ctx['tmpfilename'] == '-'):
  66             frag_index_stream, _ = sanitize_open(ctx['tmpfilename'] + '.fragindex', 'w')
  67             frag_index_stream.write(compat_str(ctx['frag_index']))
  68             frag_index_stream.close()
  69
  70     def _prepare_frag_download(self, ctx):
  71         if 'live' not in ctx:
  72             ctx['live'] = False
  73         self.to_screen(
  74             '[%s] Total fragments: %s'
  75             % (self.FD_NAME, ctx['total_frags'] if not ctx['live'] else 'unknown (live)'))
  76         self.report_destination(ctx['filename'])
  77         dl = HttpQuietDownloader(
  78             self.ydl,
  79             {
  80                 'continuedl': True,
  81                 'quiet': True,
  82                 'noprogress': True,
  83                 'ratelimit': self.params.get('ratelimit'),
  84                 'retries': self.params.get('retries', 0),
  85                 'nopart': self.params.get('nopart', False),
  86                 'test': self.params.get('test', False),
  87             }
  88         )
  89         tmpfilename = self.temp_name(ctx['filename'])
  90         open_mode = 'wb'
  91         resume_len = 0
  92         frag_index = 0
  93         # Establish possible resume length
  94         if os.path.isfile(encodeFilename(tmpfilename)):
  95             open_mode = 'ab'
  96             resume_len = os.path.getsize(encodeFilename(tmpfilename))
  97             if os.path.isfile(encodeFilename(tmpfilename + '.fragindex')):
  98                 frag_index_stream, _ = sanitize_open(tmpfilename + '.fragindex', 'r')
  99                 frag_index = int(frag_index_stream.read())
 100                 frag_index_stream.close()
 101         dest_stream, tmpfilename = sanitize_open(tmpfilename, open_mode)
 102
 103         ctx.update({
 104             'dl': dl,
 105             'dest_stream': dest_stream,
 106             'tmpfilename': tmpfilename,
 107             'frag_index': frag_index,
 108             # Total complete fragments downloaded so far in bytes
 109             'complete_frags_downloaded_bytes': resume_len,
 110         })
 111
 112     def _start_frag_download(self, ctx):
 113         total_frags = ctx['total_frags']
 114         # This dict stores the download progress, it's updated by the progress
 115         # hook
 116         state = {
 117             'status': 'downloading',
 118             'downloaded_bytes': ctx['complete_frags_downloaded_bytes'],
 119             'frag_index': ctx['frag_index'],
 120             'frag_count': total_frags,
 121             'filename': ctx['filename'],
 122             'tmpfilename': ctx['tmpfilename'],
 123         }
 124
 125         start = time.time()
 126         ctx.update({
 127             'started': start,
 128             # Amount of fragment's bytes downloaded by the time of the previous
 129             # frag progress hook invocation
 130             'prev_frag_downloaded_bytes': 0,
 131         })
 132
 133         def frag_progress_hook(s):
 134             if s['status'] not in ('downloading', 'finished'):
 135                 return
 136
 137             time_now = time.time()
 138             state['elapsed'] = time_now - start
 139             frag_total_bytes = s.get('total_bytes') or 0
 140             if not ctx['live']:
 141                 estimated_size = (
 142                     (ctx['complete_frags_downloaded_bytes'] + frag_total_bytes) /
 143                     (state['frag_index'] + 1) * total_frags)
 144                 state['total_bytes_estimate'] = estimated_size
 145
 146             if s['status'] == 'finished':
 147                 state['frag_index'] += 1
 148                 ctx['frag_index'] = state['frag_index']
 149                 state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
 150                 ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
 151                 ctx['prev_frag_downloaded_bytes'] = 0
 152             else:
 153                 frag_downloaded_bytes = s['downloaded_bytes']
 154                 state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
 155                 if not ctx['live']:
 156                     state['eta'] = self.calc_eta(
 157                         start, time_now, estimated_size,
 158                         state['downloaded_bytes'])
 159                 state['speed'] = s.get('speed') or ctx.get('speed')
 160                 ctx['speed'] = state['speed']
 161                 ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
 162             self._hook_progress(state)
 163
 164         ctx['dl'].add_progress_hook(frag_progress_hook)
 165
 166         return start
 167
 168     def _finish_frag_download(self, ctx):
 169         ctx['dest_stream'].close()
 170         if os.path.isfile(encodeFilename(ctx['tmpfilename'] + '.fragindex')):
 171             os.remove(encodeFilename(ctx['tmpfilename'] + '.fragindex'))
 172         elapsed = time.time() - ctx['started']
 173         self.try_rename(ctx['tmpfilename'], ctx['filename'])
 174         fsize = os.path.getsize(encodeFilename(ctx['filename']))
 175
 176         self._hook_progress({
 177             'downloaded_bytes': fsize,
 178             'total_bytes': fsize,
 179             'filename': ctx['filename'],
 180             'status': 'finished',
 181             'elapsed': elapsed,
 182         })