_ Git - youtube-dl/blob - youtube_dl/FileDownloader.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import httplib
   5 import math
   6 import os
   7 import re
   8 import socket
   9 import subprocess
  10 import sys
  11 import time
  12 import urllib2
  13
  14 if os.name == 'nt':
  15         import ctypes
  16
  17 from utils import *
  18
  19
  20 class FileDownloader(object):
  21         """File Downloader class.
  22
  23         File downloader objects are the ones responsible of downloading the
  24         actual video file and writing it to disk if the user has requested
  25         it, among some other tasks. In most cases there should be one per
  26         program. As, given a video URL, the downloader doesn't know how to
  27         extract all the needed information, task that InfoExtractors do, it
  28         has to pass the URL to one of them.
  29
  30         For this, file downloader objects have a method that allows
  31         InfoExtractors to be registered in a given order. When it is passed
  32         a URL, the file downloader handles it to the first InfoExtractor it
  33         finds that reports being able to handle it. The InfoExtractor extracts
  34         all the information about the video or videos the URL refers to, and
  35         asks the FileDownloader to process the video information, possibly
  36         downloading the video.
  37
  38         File downloaders accept a lot of parameters. In order not to saturate
  39         the object constructor with arguments, it receives a dictionary of
  40         options instead. These options are available through the params
  41         attribute for the InfoExtractors to use. The FileDownloader also
  42         registers itself as the downloader in charge for the InfoExtractors
  43         that are added to it, so this is a "mutual registration".
  44
  45         Available options:
  46
  47         username:         Username for authentication purposes.
  48         password:         Password for authentication purposes.
  49         usenetrc:         Use netrc for authentication instead.
  50         quiet:            Do not print messages to stdout.
  51         forceurl:         Force printing final URL.
  52         forcetitle:       Force printing title.
  53         forcethumbnail:   Force printing thumbnail URL.
  54         forcedescription: Force printing description.
  55         forcefilename:    Force printing final filename.
  56         simulate:         Do not download the video files.
  57         format:           Video format code.
  58         format_limit:     Highest quality format to try.
  59         outtmpl:          Template for output names.
  60         ignoreerrors:     Do not stop on download errors.
  61         ratelimit:        Download speed limit, in bytes/sec.
  62         nooverwrites:     Prevent overwriting files.
  63         retries:          Number of times to retry for HTTP error 5xx
  64         continuedl:       Try to continue downloads if possible.
  65         noprogress:       Do not print the progress bar.
  66         playliststart:    Playlist item to start at.
  67         playlistend:      Playlist item to end at.
  68         matchtitle:       Download only matching titles.
  69         rejecttitle:      Reject downloads for matching titles.
  70         logtostderr:      Log messages to stderr instead of stdout.
  71         consoletitle:     Display progress in console window's titlebar.
  72         nopart:           Do not use temporary .part files.
  73         updatetime:       Use the Last-modified header to set output file timestamps.
  74         writedescription: Write the video description to a .description file
  75         writeinfojson:    Write the video description to a .info.json file
  76         writesubtitles:   Write the video subtitles to a .srt file
  77         subtitleslang:    Language of the subtitles to download
  78         """
  79
  80         params = None
  81         _ies = []
  82         _pps = []
  83         _download_retcode = None
  84         _num_downloads = None
  85         _screen_file = None
  86
  87         def __init__(self, params):
  88                 """Create a FileDownloader object with the given options."""
  89                 self._ies = []
  90                 self._pps = []
  91                 self._download_retcode = 0
  92                 self._num_downloads = 0
  93                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
  94                 self.params = params
  95
  96         @staticmethod
  97         def format_bytes(bytes):
  98                 if bytes is None:
  99                         return 'N/A'
 100                 if type(bytes) is str:
 101                         bytes = float(bytes)
 102                 if bytes == 0.0:
 103                         exponent = 0
 104                 else:
 105                         exponent = long(math.log(bytes, 1024.0))
 106                 suffix = 'bkMGTPEZY'[exponent]
 107                 converted = float(bytes) / float(1024 ** exponent)
 108                 return '%.2f%s' % (converted, suffix)
 109
 110         @staticmethod
 111         def calc_percent(byte_counter, data_len):
 112                 if data_len is None:
 113                         return '---.-%'
 114                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 115
 116         @staticmethod
 117         def calc_eta(start, now, total, current):
 118                 if total is None:
 119                         return '--:--'
 120                 dif = now - start
 121                 if current == 0 or dif < 0.001: # One millisecond
 122                         return '--:--'
 123                 rate = float(current) / dif
 124                 eta = long((float(total) - float(current)) / rate)
 125                 (eta_mins, eta_secs) = divmod(eta, 60)
 126                 if eta_mins > 99:
 127                         return '--:--'
 128                 return '%02d:%02d' % (eta_mins, eta_secs)
 129
 130         @staticmethod
 131         def calc_speed(start, now, bytes):
 132                 dif = now - start
 133                 if bytes == 0 or dif < 0.001: # One millisecond
 134                         return '%10s' % '---b/s'
 135                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 136
 137         @staticmethod
 138         def best_block_size(elapsed_time, bytes):
 139                 new_min = max(bytes / 2.0, 1.0)
 140                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 141                 if elapsed_time < 0.001:
 142                         return int(new_max)
 143                 rate = bytes / elapsed_time
 144                 if rate > new_max:
 145                         return int(new_max)
 146                 if rate < new_min:
 147                         return int(new_min)
 148                 return int(rate)
 149
 150         @staticmethod
 151         def parse_bytes(bytestr):
 152                 """Parse a string indicating a byte quantity into an integer."""
 153                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 154                 if matchobj is None:
 155                         return None
 156                 number = float(matchobj.group(1))
 157                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 158                 return int(round(number * multiplier))
 159
 160         def add_info_extractor(self, ie):
 161                 """Add an InfoExtractor object to the end of the list."""
 162                 self._ies.append(ie)
 163                 ie.set_downloader(self)
 164
 165         def add_post_processor(self, pp):
 166                 """Add a PostProcessor object to the end of the chain."""
 167                 self._pps.append(pp)
 168                 pp.set_downloader(self)
 169
 170         def to_screen(self, message, skip_eol=False):
 171                 """Print message to stdout if not in quiet mode."""
 172                 assert type(message) == type(u'')
 173                 if not self.params.get('quiet', False):
 174                         terminator = [u'\n', u''][skip_eol]
 175                         output = message + terminator
 176                         if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
 177                                 output = output.encode(preferredencoding(), 'ignore')
 178                         self._screen_file.write(output)
 179                         self._screen_file.flush()
 180
 181         def to_stderr(self, message):
 182                 """Print message to stderr."""
 183                 assert type(message) == type(u'')
 184                 sys.stderr.write((message + u'\n').encode(preferredencoding()))
 185
 186         def to_cons_title(self, message):
 187                 """Set console/terminal window title to message."""
 188                 if not self.params.get('consoletitle', False):
 189                         return
 190                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 191                         # c_wchar_p() might not be necessary if `message` is
 192                         # already of type unicode()
 193                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 194                 elif 'TERM' in os.environ:
 195                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
 196
 197         def fixed_template(self):
 198                 """Checks if the output template is fixed."""
 199                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 200
 201         def trouble(self, message=None):
 202                 """Determine action to take when a download problem appears.
 203
 204                 Depending on if the downloader has been configured to ignore
 205                 download errors or not, this method may throw an exception or
 206                 not when errors are found, after printing the message.
 207                 """
 208                 if message is not None:
 209                         self.to_stderr(message)
 210                 if not self.params.get('ignoreerrors', False):
 211                         raise DownloadError(message)
 212                 self._download_retcode = 1
 213
 214         def slow_down(self, start_time, byte_counter):
 215                 """Sleep if the download speed is over the rate limit."""
 216                 rate_limit = self.params.get('ratelimit', None)
 217                 if rate_limit is None or byte_counter == 0:
 218                         return
 219                 now = time.time()
 220                 elapsed = now - start_time
 221                 if elapsed <= 0.0:
 222                         return
 223                 speed = float(byte_counter) / elapsed
 224                 if speed > rate_limit:
 225                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 226
 227         def temp_name(self, filename):
 228                 """Returns a temporary filename for the given filename."""
 229                 if self.params.get('nopart', False) or filename == u'-' or \
 230                                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 231                         return filename
 232                 return filename + u'.part'
 233
 234         def undo_temp_name(self, filename):
 235                 if filename.endswith(u'.part'):
 236                         return filename[:-len(u'.part')]
 237                 return filename
 238
 239         def try_rename(self, old_filename, new_filename):
 240                 try:
 241                         if old_filename == new_filename:
 242                                 return
 243                         os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
 244                 except (IOError, OSError), err:
 245                         self.trouble(u'ERROR: unable to rename file')
 246
 247         def try_utime(self, filename, last_modified_hdr):
 248                 """Try to set the last-modified time of the given file."""
 249                 if last_modified_hdr is None:
 250                         return
 251                 if not os.path.isfile(encodeFilename(filename)):
 252                         return
 253                 timestr = last_modified_hdr
 254                 if timestr is None:
 255                         return
 256                 filetime = timeconvert(timestr)
 257                 if filetime is None:
 258                         return filetime
 259                 try:
 260                         os.utime(filename, (time.time(), filetime))
 261                 except:
 262                         pass
 263                 return filetime
 264
 265         def report_writedescription(self, descfn):
 266                 """ Report that the description file is being written """
 267                 self.to_screen(u'[info] Writing video description to: ' + descfn)
 268
 269         def report_writesubtitles(self, srtfn):
 270                 """ Report that the subtitles file is being written """
 271                 self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
 272
 273         def report_writeinfojson(self, infofn):
 274                 """ Report that the metadata file has been written """
 275                 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
 276
 277         def report_destination(self, filename):
 278                 """Report destination filename."""
 279                 self.to_screen(u'[download] Destination: ' + filename)
 280
 281         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 282                 """Report download progress."""
 283                 if self.params.get('noprogress', False):
 284                         return
 285                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 286                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 287                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
 288                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 289
 290         def report_resuming_byte(self, resume_len):
 291                 """Report attempt to resume at given byte."""
 292                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 293
 294         def report_retry(self, count, retries):
 295                 """Report retry in case of HTTP error 5xx"""
 296                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 297
 298         def report_file_already_downloaded(self, file_name):
 299                 """Report file has already been fully downloaded."""
 300                 try:
 301                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 302                 except (UnicodeEncodeError), err:
 303                         self.to_screen(u'[download] The file has already been downloaded')
 304
 305         def report_unable_to_resume(self):
 306                 """Report it was impossible to resume download."""
 307                 self.to_screen(u'[download] Unable to resume')
 308
 309         def report_finish(self):
 310                 """Report download finished."""
 311                 if self.params.get('noprogress', False):
 312                         self.to_screen(u'[download] Download completed')
 313                 else:
 314                         self.to_screen(u'')
 315
 316         def increment_downloads(self):
 317                 """Increment the ordinal that assigns a number to each file."""
 318                 self._num_downloads += 1
 319
 320         def prepare_filename(self, info_dict):
 321                 """Generate the output filename."""
 322                 try:
 323                         template_dict = dict(info_dict)
 324                         template_dict['epoch'] = unicode(long(time.time()))
 325                         template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
 326                         template_dict['title'] = template_dict['stitle'] # Keep both for backwards compatibility
 327                         filename = self.params['outtmpl'] % template_dict
 328                         return filename
 329                 except (ValueError, KeyError), err:
 330                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 331                         return None
 332
 333         def _match_entry(self, info_dict):
 334                 """ Returns None iff the file should be downloaded """
 335
 336                 title = info_dict['title']
 337                 matchtitle = self.params.get('matchtitle', False)
 338                 if matchtitle:
 339                         matchtitle = matchtitle.decode('utf8')
 340                         if not re.search(matchtitle, title, re.IGNORECASE):
 341                                 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
 342                 rejecttitle = self.params.get('rejecttitle', False)
 343                 if rejecttitle:
 344                         rejecttitle = rejecttitle.decode('utf8')
 345                         if re.search(rejecttitle, title, re.IGNORECASE):
 346                                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 347                 return None
 348
 349         def process_info(self, info_dict):
 350                 """Process a single dictionary returned by an InfoExtractor."""
 351
 352                 info_dict['stitle'] = sanitize_filename(info_dict['title'])
 353
 354                 reason = self._match_entry(info_dict)
 355                 if reason is not None:
 356                         self.to_screen(u'[download] ' + reason)
 357                         return
 358
 359                 max_downloads = self.params.get('max_downloads')
 360                 if max_downloads is not None:
 361                         if self._num_downloads > int(max_downloads):
 362                                 raise MaxDownloadsReached()
 363
 364                 filename = self.prepare_filename(info_dict)
 365
 366                 # Forced printings
 367                 if self.params.get('forcetitle', False):
 368                         print(info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace'))
 369                 if self.params.get('forceurl', False):
 370                         print(info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace'))
 371                 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 372                         print(info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace'))
 373                 if self.params.get('forcedescription', False) and 'description' in info_dict:
 374                         print(info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace'))
 375                 if self.params.get('forcefilename', False) and filename is not None:
 376                         print(filename.encode(preferredencoding(), 'xmlcharrefreplace'))
 377                 if self.params.get('forceformat', False):
 378                         print(info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace'))
 379
 380                 # Do nothing else if in simulate mode
 381                 if self.params.get('simulate', False):
 382                         return
 383
 384                 if filename is None:
 385                         return
 386
 387                 try:
 388                         dn = os.path.dirname(encodeFilename(filename))
 389                         if dn != '' and not os.path.exists(dn): # dn is already encoded
 390                                 os.makedirs(dn)
 391                 except (OSError, IOError), err:
 392                         self.trouble(u'ERROR: unable to create directory ' + unicode(err))
 393                         return
 394
 395                 if self.params.get('writedescription', False):
 396                         try:
 397                                 descfn = filename + u'.description'
 398                                 self.report_writedescription(descfn)
 399                                 descfile = open(encodeFilename(descfn), 'wb')
 400                                 try:
 401                                         descfile.write(info_dict['description'].encode('utf-8'))
 402                                 finally:
 403                                         descfile.close()
 404                         except (OSError, IOError):
 405                                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
 406                                 return
 407
 408                 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
 409                         # subtitles download errors are already managed as troubles in relevant IE
 410                         # that way it will silently go on when used with unsupporting IE
 411                         try:
 412                                 srtfn = filename.rsplit('.', 1)[0] + u'.srt'
 413                                 self.report_writesubtitles(srtfn)
 414                                 srtfile = open(encodeFilename(srtfn), 'wb')
 415                                 try:
 416                                         srtfile.write(info_dict['subtitles'].encode('utf-8'))
 417                                 finally:
 418                                         srtfile.close()
 419                         except (OSError, IOError):
 420                                 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
 421                                 return
 422
 423                 if self.params.get('writeinfojson', False):
 424                         infofn = filename + u'.info.json'
 425                         self.report_writeinfojson(infofn)
 426                         try:
 427                                 json.dump
 428                         except (NameError,AttributeError):
 429                                 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
 430                                 return
 431                         try:
 432                                 infof = open(encodeFilename(infofn), 'wb')
 433                                 try:
 434                                         json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
 435                                         json.dump(json_info_dict, infof)
 436                                 finally:
 437                                         infof.close()
 438                         except (OSError, IOError):
 439                                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
 440                                 return
 441
 442                 if not self.params.get('skip_download', False):
 443                         if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
 444                                 success = True
 445                         else:
 446                                 try:
 447                                         success = self._do_download(filename, info_dict)
 448                                 except (OSError, IOError), err:
 449                                         raise UnavailableVideoError
 450                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 451                                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 452                                         return
 453                                 except (ContentTooShortError, ), err:
 454                                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 455                                         return
 456
 457                         if success:
 458                                 try:
 459                                         self.post_process(filename, info_dict)
 460                                 except (PostProcessingError), err:
 461                                         self.trouble(u'ERROR: postprocessing: %s' % str(err))
 462                                         return
 463
 464         def download(self, url_list):
 465                 """Download a given list of URLs."""
 466                 if len(url_list) > 1 and self.fixed_template():
 467                         raise SameFileError(self.params['outtmpl'])
 468
 469                 for url in url_list:
 470                         suitable_found = False
 471                         for ie in self._ies:
 472                                 # Go to next InfoExtractor if not suitable
 473                                 if not ie.suitable(url):
 474                                         continue
 475
 476                                 # Suitable InfoExtractor found
 477                                 suitable_found = True
 478
 479                                 # Extract information from URL and process it
 480                                 videos = ie.extract(url)
 481                                 for video in videos or []:
 482                                         video['extractor'] = ie.IE_NAME
 483                                         try:
 484                                                 self.increment_downloads()
 485                                                 self.process_info(video)
 486                                         except UnavailableVideoError:
 487                                                 self.trouble(u'\nERROR: unable to download video')
 488
 489                                 # Suitable InfoExtractor had been found; go to next URL
 490                                 break
 491
 492                         if not suitable_found:
 493                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 494
 495                 return self._download_retcode
 496
 497         def post_process(self, filename, ie_info):
 498                 """Run the postprocessing chain on the given file."""
 499                 info = dict(ie_info)
 500                 info['filepath'] = filename
 501                 for pp in self._pps:
 502                         info = pp.run(info)
 503                         if info is None:
 504                                 break
 505
 506         def _download_with_rtmpdump(self, filename, url, player_url):
 507                 self.report_destination(filename)
 508                 tmpfilename = self.temp_name(filename)
 509
 510                 # Check for rtmpdump first
 511                 try:
 512                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 513                 except (OSError, IOError):
 514                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 515                         return False
 516
 517                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 518                 # the connection was interrumpted and resuming appears to be
 519                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 520                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 521                 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
 522                 if self.params.get('verbose', False):
 523                         try:
 524                                 import pipes
 525                                 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
 526                         except ImportError:
 527                                 shell_quote = repr
 528                         self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
 529                 retval = subprocess.call(args)
 530                 while retval == 2 or retval == 1:
 531                         prevsize = os.path.getsize(encodeFilename(tmpfilename))
 532                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 533                         time.sleep(5.0) # This seems to be needed
 534                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 535                         cursize = os.path.getsize(encodeFilename(tmpfilename))
 536                         if prevsize == cursize and retval == 1:
 537                                 break
 538                          # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
 539                         if prevsize == cursize and retval == 2 and cursize > 1024:
 540                                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
 541                                 retval = 0
 542                                 break
 543                 if retval == 0:
 544                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename)))
 545                         self.try_rename(tmpfilename, filename)
 546                         return True
 547                 else:
 548                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 549                         return False
 550
 551         def _do_download(self, filename, info_dict):
 552                 url = info_dict['url']
 553                 player_url = info_dict.get('player_url', None)
 554
 555                 # Check file already present
 556                 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
 557                         self.report_file_already_downloaded(filename)
 558                         return True
 559
 560                 # Attempt to download using rtmpdump
 561                 if url.startswith('rtmp'):
 562                         return self._download_with_rtmpdump(filename, url, player_url)
 563
 564                 tmpfilename = self.temp_name(filename)
 565                 stream = None
 566
 567                 # Do not include the Accept-Encoding header
 568                 headers = {'Youtubedl-no-compression': 'True'}
 569                 basic_request = urllib2.Request(url, None, headers)
 570                 request = urllib2.Request(url, None, headers)
 571
 572                 # Establish possible resume length
 573                 if os.path.isfile(encodeFilename(tmpfilename)):
 574                         resume_len = os.path.getsize(encodeFilename(tmpfilename))
 575                 else:
 576                         resume_len = 0
 577
 578                 open_mode = 'wb'
 579                 if resume_len != 0:
 580                         if self.params.get('continuedl', False):
 581                                 self.report_resuming_byte(resume_len)
 582                                 request.add_header('Range','bytes=%d-' % resume_len)
 583                                 open_mode = 'ab'
 584                         else:
 585                                 resume_len = 0
 586
 587                 count = 0
 588                 retries = self.params.get('retries', 0)
 589                 while count <= retries:
 590                         # Establish connection
 591                         try:
 592                                 if count == 0 and 'urlhandle' in info_dict:
 593                                         data = info_dict['urlhandle']
 594                                 data = urllib2.urlopen(request)
 595                                 break
 596                         except (urllib2.HTTPError, ), err:
 597                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 598                                         # Unexpected HTTP error
 599                                         raise
 600                                 elif err.code == 416:
 601                                         # Unable to resume (requested range not satisfiable)
 602                                         try:
 603                                                 # Open the connection again without the range header
 604                                                 data = urllib2.urlopen(basic_request)
 605                                                 content_length = data.info()['Content-Length']
 606                                         except (urllib2.HTTPError, ), err:
 607                                                 if err.code < 500 or err.code >= 600:
 608                                                         raise
 609                                         else:
 610                                                 # Examine the reported length
 611                                                 if (content_length is not None and
 612                                                                 (resume_len - 100 < long(content_length) < resume_len + 100)):
 613                                                         # The file had already been fully downloaded.
 614                                                         # Explanation to the above condition: in issue #175 it was revealed that
 615                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 616                                                         # changing the file size slightly and causing problems for some users. So
 617                                                         # I decided to implement a suggested change and consider the file
 618                                                         # completely downloaded if the file size differs less than 100 bytes from
 619                                                         # the one in the hard drive.
 620                                                         self.report_file_already_downloaded(filename)
 621                                                         self.try_rename(tmpfilename, filename)
 622                                                         return True
 623                                                 else:
 624                                                         # The length does not match, we start the download over
 625                                                         self.report_unable_to_resume()
 626                                                         open_mode = 'wb'
 627                                                         break
 628                         # Retry
 629                         count += 1
 630                         if count <= retries:
 631                                 self.report_retry(count, retries)
 632
 633                 if count > retries:
 634                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 635                         return False
 636
 637                 data_len = data.info().get('Content-length', None)
 638                 if data_len is not None:
 639                         data_len = long(data_len) + resume_len
 640                 data_len_str = self.format_bytes(data_len)
 641                 byte_counter = 0 + resume_len
 642                 block_size = 1024
 643                 start = time.time()
 644                 while True:
 645                         # Download and write
 646                         before = time.time()
 647                         data_block = data.read(block_size)
 648                         after = time.time()
 649                         if len(data_block) == 0:
 650                                 break
 651                         byte_counter += len(data_block)
 652
 653                         # Open file just in time
 654                         if stream is None:
 655                                 try:
 656                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 657                                         assert stream is not None
 658                                         filename = self.undo_temp_name(tmpfilename)
 659                                         self.report_destination(filename)
 660                                 except (OSError, IOError), err:
 661                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 662                                         return False
 663                         try:
 664                                 stream.write(data_block)
 665                         except (IOError, OSError), err:
 666                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 667                                 return False
 668                         block_size = self.best_block_size(after - before, len(data_block))
 669
 670                         # Progress message
 671                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
 672                         if data_len is None:
 673                                 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
 674                         else:
 675                                 percent_str = self.calc_percent(byte_counter, data_len)
 676                                 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 677                                 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 678
 679                         # Apply rate limit
 680                         self.slow_down(start, byte_counter - resume_len)
 681
 682                 if stream is None:
 683                         self.trouble(u'\nERROR: Did not get any data blocks')
 684                         return False
 685                 stream.close()
 686                 self.report_finish()
 687                 if data_len is not None and byte_counter != data_len:
 688                         raise ContentTooShortError(byte_counter, long(data_len))
 689                 self.try_rename(tmpfilename, filename)
 690
 691                 # Update file modification time
 692                 if self.params.get('updatetime', True):
 693                         info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
 694
 695                 return True