_ Git - youtube-dl/blob - youtube_dl/FileDownloader.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import httplib
   5 import math
   6 import os
   7 import re
   8 import socket
   9 import subprocess
  10 import sys
  11 import time
  12 import urllib2
  13
  14 if os.name == 'nt':
  15         import ctypes
  16
  17 from utils import *
  18
  19
  20 class FileDownloader(object):
  21         """File Downloader class.
  22
  23         File downloader objects are the ones responsible of downloading the
  24         actual video file and writing it to disk if the user has requested
  25         it, among some other tasks. In most cases there should be one per
  26         program. As, given a video URL, the downloader doesn't know how to
  27         extract all the needed information, task that InfoExtractors do, it
  28         has to pass the URL to one of them.
  29
  30         For this, file downloader objects have a method that allows
  31         InfoExtractors to be registered in a given order. When it is passed
  32         a URL, the file downloader handles it to the first InfoExtractor it
  33         finds that reports being able to handle it. The InfoExtractor extracts
  34         all the information about the video or videos the URL refers to, and
  35         asks the FileDownloader to process the video information, possibly
  36         downloading the video.
  37
  38         File downloaders accept a lot of parameters. In order not to saturate
  39         the object constructor with arguments, it receives a dictionary of
  40         options instead. These options are available through the params
  41         attribute for the InfoExtractors to use. The FileDownloader also
  42         registers itself as the downloader in charge for the InfoExtractors
  43         that are added to it, so this is a "mutual registration".
  44
  45         Available options:
  46
  47         username:          Username for authentication purposes.
  48         password:          Password for authentication purposes.
  49         usenetrc:          Use netrc for authentication instead.
  50         quiet:             Do not print messages to stdout.
  51         forceurl:          Force printing final URL.
  52         forcetitle:        Force printing title.
  53         forcethumbnail:    Force printing thumbnail URL.
  54         forcedescription:  Force printing description.
  55         forcefilename:     Force printing final filename.
  56         simulate:          Do not download the video files.
  57         format:            Video format code.
  58         format_limit:      Highest quality format to try.
  59         outtmpl:           Template for output names.
  60         restrictfilenames: Do not allow "&" and spaces in file names
  61         ignoreerrors:      Do not stop on download errors.
  62         ratelimit:         Download speed limit, in bytes/sec.
  63         nooverwrites:      Prevent overwriting files.
  64         retries:           Number of times to retry for HTTP error 5xx
  65         continuedl:        Try to continue downloads if possible.
  66         noprogress:        Do not print the progress bar.
  67         playliststart:     Playlist item to start at.
  68         playlistend:       Playlist item to end at.
  69         matchtitle:        Download only matching titles.
  70         rejecttitle:       Reject downloads for matching titles.
  71         logtostderr:       Log messages to stderr instead of stdout.
  72         consoletitle:      Display progress in console window's titlebar.
  73         nopart:            Do not use temporary .part files.
  74         updatetime:        Use the Last-modified header to set output file timestamps.
  75         writedescription:  Write the video description to a .description file
  76         writeinfojson:     Write the video description to a .info.json file
  77         writesubtitles:    Write the video subtitles to a .srt file
  78         subtitleslang:     Language of the subtitles to download
  79         """
  80
  81         params = None
  82         _ies = []
  83         _pps = []
  84         _download_retcode = None
  85         _num_downloads = None
  86         _screen_file = None
  87
  88         def __init__(self, params):
  89                 """Create a FileDownloader object with the given options."""
  90                 self._ies = []
  91                 self._pps = []
  92                 self._download_retcode = 0
  93                 self._num_downloads = 0
  94                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
  95                 self.params = params
  96
  97                 if '%(stitle)s' in self.params['outtmpl']:
  98                         self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
  99
 100         @staticmethod
 101         def format_bytes(bytes):
 102                 if bytes is None:
 103                         return 'N/A'
 104                 if type(bytes) is str:
 105                         bytes = float(bytes)
 106                 if bytes == 0.0:
 107                         exponent = 0
 108                 else:
 109                         exponent = long(math.log(bytes, 1024.0))
 110                 suffix = 'bkMGTPEZY'[exponent]
 111                 converted = float(bytes) / float(1024 ** exponent)
 112                 return '%.2f%s' % (converted, suffix)
 113
 114         @staticmethod
 115         def calc_percent(byte_counter, data_len):
 116                 if data_len is None:
 117                         return '---.-%'
 118                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 119
 120         @staticmethod
 121         def calc_eta(start, now, total, current):
 122                 if total is None:
 123                         return '--:--'
 124                 dif = now - start
 125                 if current == 0 or dif < 0.001: # One millisecond
 126                         return '--:--'
 127                 rate = float(current) / dif
 128                 eta = long((float(total) - float(current)) / rate)
 129                 (eta_mins, eta_secs) = divmod(eta, 60)
 130                 if eta_mins > 99:
 131                         return '--:--'
 132                 return '%02d:%02d' % (eta_mins, eta_secs)
 133
 134         @staticmethod
 135         def calc_speed(start, now, bytes):
 136                 dif = now - start
 137                 if bytes == 0 or dif < 0.001: # One millisecond
 138                         return '%10s' % '---b/s'
 139                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 140
 141         @staticmethod
 142         def best_block_size(elapsed_time, bytes):
 143                 new_min = max(bytes / 2.0, 1.0)
 144                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 145                 if elapsed_time < 0.001:
 146                         return int(new_max)
 147                 rate = bytes / elapsed_time
 148                 if rate > new_max:
 149                         return int(new_max)
 150                 if rate < new_min:
 151                         return int(new_min)
 152                 return int(rate)
 153
 154         @staticmethod
 155         def parse_bytes(bytestr):
 156                 """Parse a string indicating a byte quantity into an integer."""
 157                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 158                 if matchobj is None:
 159                         return None
 160                 number = float(matchobj.group(1))
 161                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 162                 return int(round(number * multiplier))
 163
 164         def add_info_extractor(self, ie):
 165                 """Add an InfoExtractor object to the end of the list."""
 166                 self._ies.append(ie)
 167                 ie.set_downloader(self)
 168
 169         def add_post_processor(self, pp):
 170                 """Add a PostProcessor object to the end of the chain."""
 171                 self._pps.append(pp)
 172                 pp.set_downloader(self)
 173
 174         def to_screen(self, message, skip_eol=False):
 175                 """Print message to stdout if not in quiet mode."""
 176                 assert type(message) == type(u'')
 177                 if not self.params.get('quiet', False):
 178                         terminator = [u'\n', u''][skip_eol]
 179                         output = message + terminator
 180                         if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
 181                                 output = output.encode(preferredencoding(), 'ignore')
 182                         self._screen_file.write(output)
 183                         self._screen_file.flush()
 184
 185         def to_stderr(self, message):
 186                 """Print message to stderr."""
 187                 assert type(message) == type(u'')
 188                 sys.stderr.write((message + u'\n').encode(preferredencoding()))
 189
 190         def to_cons_title(self, message):
 191                 """Set console/terminal window title to message."""
 192                 if not self.params.get('consoletitle', False):
 193                         return
 194                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 195                         # c_wchar_p() might not be necessary if `message` is
 196                         # already of type unicode()
 197                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 198                 elif 'TERM' in os.environ:
 199                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
 200
 201         def fixed_template(self):
 202                 """Checks if the output template is fixed."""
 203                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 204
 205         def trouble(self, message=None):
 206                 """Determine action to take when a download problem appears.
 207
 208                 Depending on if the downloader has been configured to ignore
 209                 download errors or not, this method may throw an exception or
 210                 not when errors are found, after printing the message.
 211                 """
 212                 if message is not None:
 213                         self.to_stderr(message)
 214                 if not self.params.get('ignoreerrors', False):
 215                         raise DownloadError(message)
 216                 self._download_retcode = 1
 217
 218         def slow_down(self, start_time, byte_counter):
 219                 """Sleep if the download speed is over the rate limit."""
 220                 rate_limit = self.params.get('ratelimit', None)
 221                 if rate_limit is None or byte_counter == 0:
 222                         return
 223                 now = time.time()
 224                 elapsed = now - start_time
 225                 if elapsed <= 0.0:
 226                         return
 227                 speed = float(byte_counter) / elapsed
 228                 if speed > rate_limit:
 229                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 230
 231         def temp_name(self, filename):
 232                 """Returns a temporary filename for the given filename."""
 233                 if self.params.get('nopart', False) or filename == u'-' or \
 234                                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 235                         return filename
 236                 return filename + u'.part'
 237
 238         def undo_temp_name(self, filename):
 239                 if filename.endswith(u'.part'):
 240                         return filename[:-len(u'.part')]
 241                 return filename
 242
 243         def try_rename(self, old_filename, new_filename):
 244                 try:
 245                         if old_filename == new_filename:
 246                                 return
 247                         os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
 248                 except (IOError, OSError), err:
 249                         self.trouble(u'ERROR: unable to rename file')
 250
 251         def try_utime(self, filename, last_modified_hdr):
 252                 """Try to set the last-modified time of the given file."""
 253                 if last_modified_hdr is None:
 254                         return
 255                 if not os.path.isfile(encodeFilename(filename)):
 256                         return
 257                 timestr = last_modified_hdr
 258                 if timestr is None:
 259                         return
 260                 filetime = timeconvert(timestr)
 261                 if filetime is None:
 262                         return filetime
 263                 try:
 264                         os.utime(filename, (time.time(), filetime))
 265                 except:
 266                         pass
 267                 return filetime
 268
 269         def report_writedescription(self, descfn):
 270                 """ Report that the description file is being written """
 271                 self.to_screen(u'[info] Writing video description to: ' + descfn)
 272
 273         def report_writesubtitles(self, srtfn):
 274                 """ Report that the subtitles file is being written """
 275                 self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
 276
 277         def report_writeinfojson(self, infofn):
 278                 """ Report that the metadata file has been written """
 279                 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
 280
 281         def report_destination(self, filename):
 282                 """Report destination filename."""
 283                 self.to_screen(u'[download] Destination: ' + filename)
 284
 285         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 286                 """Report download progress."""
 287                 if self.params.get('noprogress', False):
 288                         return
 289                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 290                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 291                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
 292                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 293
 294         def report_resuming_byte(self, resume_len):
 295                 """Report attempt to resume at given byte."""
 296                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 297
 298         def report_retry(self, count, retries):
 299                 """Report retry in case of HTTP error 5xx"""
 300                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 301
 302         def report_file_already_downloaded(self, file_name):
 303                 """Report file has already been fully downloaded."""
 304                 try:
 305                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 306                 except (UnicodeEncodeError), err:
 307                         self.to_screen(u'[download] The file has already been downloaded')
 308
 309         def report_unable_to_resume(self):
 310                 """Report it was impossible to resume download."""
 311                 self.to_screen(u'[download] Unable to resume')
 312
 313         def report_finish(self):
 314                 """Report download finished."""
 315                 if self.params.get('noprogress', False):
 316                         self.to_screen(u'[download] Download completed')
 317                 else:
 318                         self.to_screen(u'')
 319
 320         def increment_downloads(self):
 321                 """Increment the ordinal that assigns a number to each file."""
 322                 self._num_downloads += 1
 323
 324         def prepare_filename(self, info_dict):
 325                 """Generate the output filename."""
 326                 try:
 327                         template_dict = dict(info_dict)
 328                         template_dict['epoch'] = unicode(int(time.time()))
 329                         template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
 330                         filename = self.params['outtmpl'] % template_dict
 331                         return filename
 332                 except (ValueError, KeyError), err:
 333                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 334                         return None
 335
 336         def _match_entry(self, info_dict):
 337                 """ Returns None iff the file should be downloaded """
 338
 339                 title = info_dict['title']
 340                 matchtitle = self.params.get('matchtitle', False)
 341                 if matchtitle:
 342                         matchtitle = matchtitle.decode('utf8')
 343                         if not re.search(matchtitle, title, re.IGNORECASE):
 344                                 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
 345                 rejecttitle = self.params.get('rejecttitle', False)
 346                 if rejecttitle:
 347                         rejecttitle = rejecttitle.decode('utf8')
 348                         if re.search(rejecttitle, title, re.IGNORECASE):
 349                                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 350                 return None
 351
 352         def process_info(self, info_dict):
 353                 """Process a single dictionary returned by an InfoExtractor."""
 354
 355                 # Keep for backwards compatibility
 356                 info_dict['stitle'] = info_dict['title']
 357
 358                 reason = self._match_entry(info_dict)
 359                 if reason is not None:
 360                         self.to_screen(u'[download] ' + reason)
 361                         return
 362
 363                 max_downloads = self.params.get('max_downloads')
 364                 if max_downloads is not None:
 365                         if self._num_downloads > int(max_downloads):
 366                                 raise MaxDownloadsReached()
 367
 368                 filename = self.prepare_filename(info_dict)
 369                 filename = sanitize_filename(filename, self.params.get('restrictfilenames'))
 370
 371                 # Forced printings
 372                 if self.params.get('forcetitle', False):
 373                         print(info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace'))
 374                 if self.params.get('forceurl', False):
 375                         print(info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace'))
 376                 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 377                         print(info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace'))
 378                 if self.params.get('forcedescription', False) and 'description' in info_dict:
 379                         print(info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace'))
 380                 if self.params.get('forcefilename', False) and filename is not None:
 381                         print(filename.encode(preferredencoding(), 'xmlcharrefreplace'))
 382                 if self.params.get('forceformat', False):
 383                         print(info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace'))
 384
 385                 # Do nothing else if in simulate mode
 386                 if self.params.get('simulate', False):
 387                         return
 388
 389                 if filename is None:
 390                         return
 391
 392                 try:
 393                         dn = os.path.dirname(encodeFilename(filename))
 394                         if dn != '' and not os.path.exists(dn): # dn is already encoded
 395                                 os.makedirs(dn)
 396                 except (OSError, IOError), err:
 397                         self.trouble(u'ERROR: unable to create directory ' + unicode(err))
 398                         return
 399
 400                 if self.params.get('writedescription', False):
 401                         try:
 402                                 descfn = filename + u'.description'
 403                                 self.report_writedescription(descfn)
 404                                 descfile = open(encodeFilename(descfn), 'wb')
 405                                 try:
 406                                         descfile.write(info_dict['description'].encode('utf-8'))
 407                                 finally:
 408                                         descfile.close()
 409                         except (OSError, IOError):
 410                                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
 411                                 return
 412
 413                 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
 414                         # subtitles download errors are already managed as troubles in relevant IE
 415                         # that way it will silently go on when used with unsupporting IE
 416                         try:
 417                                 srtfn = filename.rsplit('.', 1)[0] + u'.srt'
 418                                 self.report_writesubtitles(srtfn)
 419                                 srtfile = open(encodeFilename(srtfn), 'wb')
 420                                 try:
 421                                         srtfile.write(info_dict['subtitles'].encode('utf-8'))
 422                                 finally:
 423                                         srtfile.close()
 424                         except (OSError, IOError):
 425                                 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
 426                                 return
 427
 428                 if self.params.get('writeinfojson', False):
 429                         infofn = filename + u'.info.json'
 430                         self.report_writeinfojson(infofn)
 431                         try:
 432                                 json.dump
 433                         except (NameError,AttributeError):
 434                                 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
 435                                 return
 436                         try:
 437                                 infof = open(encodeFilename(infofn), 'wb')
 438                                 try:
 439                                         json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
 440                                         json.dump(json_info_dict, infof)
 441                                 finally:
 442                                         infof.close()
 443                         except (OSError, IOError):
 444                                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
 445                                 return
 446
 447                 if not self.params.get('skip_download', False):
 448                         if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
 449                                 success = True
 450                         else:
 451                                 try:
 452                                         success = self._do_download(filename, info_dict)
 453                                 except (OSError, IOError), err:
 454                                         raise UnavailableVideoError
 455                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 456                                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 457                                         return
 458                                 except (ContentTooShortError, ), err:
 459                                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 460                                         return
 461
 462                         if success:
 463                                 try:
 464                                         self.post_process(filename, info_dict)
 465                                 except (PostProcessingError), err:
 466                                         self.trouble(u'ERROR: postprocessing: %s' % str(err))
 467                                         return
 468
 469         def download(self, url_list):
 470                 """Download a given list of URLs."""
 471                 if len(url_list) > 1 and self.fixed_template():
 472                         raise SameFileError(self.params['outtmpl'])
 473
 474                 for url in url_list:
 475                         suitable_found = False
 476                         for ie in self._ies:
 477                                 # Go to next InfoExtractor if not suitable
 478                                 if not ie.suitable(url):
 479                                         continue
 480
 481                                 # Suitable InfoExtractor found
 482                                 suitable_found = True
 483
 484                                 # Extract information from URL and process it
 485                                 videos = ie.extract(url)
 486                                 for video in videos or []:
 487                                         video['extractor'] = ie.IE_NAME
 488                                         try:
 489                                                 self.increment_downloads()
 490                                                 self.process_info(video)
 491                                         except UnavailableVideoError:
 492                                                 self.trouble(u'\nERROR: unable to download video')
 493
 494                                 # Suitable InfoExtractor had been found; go to next URL
 495                                 break
 496
 497                         if not suitable_found:
 498                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 499
 500                 return self._download_retcode
 501
 502         def post_process(self, filename, ie_info):
 503                 """Run the postprocessing chain on the given file."""
 504                 info = dict(ie_info)
 505                 info['filepath'] = filename
 506                 for pp in self._pps:
 507                         info = pp.run(info)
 508                         if info is None:
 509                                 break
 510
 511         def _download_with_rtmpdump(self, filename, url, player_url):
 512                 self.report_destination(filename)
 513                 tmpfilename = self.temp_name(filename)
 514
 515                 # Check for rtmpdump first
 516                 try:
 517                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 518                 except (OSError, IOError):
 519                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 520                         return False
 521
 522                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 523                 # the connection was interrumpted and resuming appears to be
 524                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 525                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 526                 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
 527                 if self.params.get('verbose', False):
 528                         try:
 529                                 import pipes
 530                                 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
 531                         except ImportError:
 532                                 shell_quote = repr
 533                         self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
 534                 retval = subprocess.call(args)
 535                 while retval == 2 or retval == 1:
 536                         prevsize = os.path.getsize(encodeFilename(tmpfilename))
 537                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 538                         time.sleep(5.0) # This seems to be needed
 539                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 540                         cursize = os.path.getsize(encodeFilename(tmpfilename))
 541                         if prevsize == cursize and retval == 1:
 542                                 break
 543                          # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
 544                         if prevsize == cursize and retval == 2 and cursize > 1024:
 545                                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
 546                                 retval = 0
 547                                 break
 548                 if retval == 0:
 549                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename)))
 550                         self.try_rename(tmpfilename, filename)
 551                         return True
 552                 else:
 553                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 554                         return False
 555
 556         def _do_download(self, filename, info_dict):
 557                 url = info_dict['url']
 558                 player_url = info_dict.get('player_url', None)
 559
 560                 # Check file already present
 561                 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
 562                         self.report_file_already_downloaded(filename)
 563                         return True
 564
 565                 # Attempt to download using rtmpdump
 566                 if url.startswith('rtmp'):
 567                         return self._download_with_rtmpdump(filename, url, player_url)
 568
 569                 tmpfilename = self.temp_name(filename)
 570                 stream = None
 571
 572                 # Do not include the Accept-Encoding header
 573                 headers = {'Youtubedl-no-compression': 'True'}
 574                 basic_request = urllib2.Request(url, None, headers)
 575                 request = urllib2.Request(url, None, headers)
 576
 577                 # Establish possible resume length
 578                 if os.path.isfile(encodeFilename(tmpfilename)):
 579                         resume_len = os.path.getsize(encodeFilename(tmpfilename))
 580                 else:
 581                         resume_len = 0
 582
 583                 open_mode = 'wb'
 584                 if resume_len != 0:
 585                         if self.params.get('continuedl', False):
 586                                 self.report_resuming_byte(resume_len)
 587                                 request.add_header('Range','bytes=%d-' % resume_len)
 588                                 open_mode = 'ab'
 589                         else:
 590                                 resume_len = 0
 591
 592                 count = 0
 593                 retries = self.params.get('retries', 0)
 594                 while count <= retries:
 595                         # Establish connection
 596                         try:
 597                                 if count == 0 and 'urlhandle' in info_dict:
 598                                         data = info_dict['urlhandle']
 599                                 data = urllib2.urlopen(request)
 600                                 break
 601                         except (urllib2.HTTPError, ), err:
 602                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 603                                         # Unexpected HTTP error
 604                                         raise
 605                                 elif err.code == 416:
 606                                         # Unable to resume (requested range not satisfiable)
 607                                         try:
 608                                                 # Open the connection again without the range header
 609                                                 data = urllib2.urlopen(basic_request)
 610                                                 content_length = data.info()['Content-Length']
 611                                         except (urllib2.HTTPError, ), err:
 612                                                 if err.code < 500 or err.code >= 600:
 613                                                         raise
 614                                         else:
 615                                                 # Examine the reported length
 616                                                 if (content_length is not None and
 617                                                                 (resume_len - 100 < long(content_length) < resume_len + 100)):
 618                                                         # The file had already been fully downloaded.
 619                                                         # Explanation to the above condition: in issue #175 it was revealed that
 620                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 621                                                         # changing the file size slightly and causing problems for some users. So
 622                                                         # I decided to implement a suggested change and consider the file
 623                                                         # completely downloaded if the file size differs less than 100 bytes from
 624                                                         # the one in the hard drive.
 625                                                         self.report_file_already_downloaded(filename)
 626                                                         self.try_rename(tmpfilename, filename)
 627                                                         return True
 628                                                 else:
 629                                                         # The length does not match, we start the download over
 630                                                         self.report_unable_to_resume()
 631                                                         open_mode = 'wb'
 632                                                         break
 633                         # Retry
 634                         count += 1
 635                         if count <= retries:
 636                                 self.report_retry(count, retries)
 637
 638                 if count > retries:
 639                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 640                         return False
 641
 642                 data_len = data.info().get('Content-length', None)
 643                 if data_len is not None:
 644                         data_len = long(data_len) + resume_len
 645                 data_len_str = self.format_bytes(data_len)
 646                 byte_counter = 0 + resume_len
 647                 block_size = 1024
 648                 start = time.time()
 649                 while True:
 650                         # Download and write
 651                         before = time.time()
 652                         data_block = data.read(block_size)
 653                         after = time.time()
 654                         if len(data_block) == 0:
 655                                 break
 656                         byte_counter += len(data_block)
 657
 658                         # Open file just in time
 659                         if stream is None:
 660                                 try:
 661                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 662                                         assert stream is not None
 663                                         filename = self.undo_temp_name(tmpfilename)
 664                                         self.report_destination(filename)
 665                                 except (OSError, IOError), err:
 666                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 667                                         return False
 668                         try:
 669                                 stream.write(data_block)
 670                         except (IOError, OSError), err:
 671                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 672                                 return False
 673                         block_size = self.best_block_size(after - before, len(data_block))
 674
 675                         # Progress message
 676                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
 677                         if data_len is None:
 678                                 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
 679                         else:
 680                                 percent_str = self.calc_percent(byte_counter, data_len)
 681                                 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 682                                 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 683
 684                         # Apply rate limit
 685                         self.slow_down(start, byte_counter - resume_len)
 686
 687                 if stream is None:
 688                         self.trouble(u'\nERROR: Did not get any data blocks')
 689                         return False
 690                 stream.close()
 691                 self.report_finish()
 692                 if data_len is not None and byte_counter != data_len:
 693                         raise ContentTooShortError(byte_counter, long(data_len))
 694                 self.try_rename(tmpfilename, filename)
 695
 696                 # Update file modification time
 697                 if self.params.get('updatetime', True):
 698                         info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
 699
 700                 return True