_ Git - youtube-dl/blob - youtube_dl/FileDownloader.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import httplib
   5 import math
   6 import os
   7 import re
   8 import socket
   9 import subprocess
  10 import sys
  11 import time
  12 import urllib2
  13
  14 if os.name == 'nt':
  15         import ctypes
  16
  17 from utils import *
  18
  19
  20 class FileDownloader(object):
  21         """File Downloader class.
  22
  23         File downloader objects are the ones responsible of downloading the
  24         actual video file and writing it to disk if the user has requested
  25         it, among some other tasks. In most cases there should be one per
  26         program. As, given a video URL, the downloader doesn't know how to
  27         extract all the needed information, task that InfoExtractors do, it
  28         has to pass the URL to one of them.
  29
  30         For this, file downloader objects have a method that allows
  31         InfoExtractors to be registered in a given order. When it is passed
  32         a URL, the file downloader handles it to the first InfoExtractor it
  33         finds that reports being able to handle it. The InfoExtractor extracts
  34         all the information about the video or videos the URL refers to, and
  35         asks the FileDownloader to process the video information, possibly
  36         downloading the video.
  37
  38         File downloaders accept a lot of parameters. In order not to saturate
  39         the object constructor with arguments, it receives a dictionary of
  40         options instead. These options are available through the params
  41         attribute for the InfoExtractors to use. The FileDownloader also
  42         registers itself as the downloader in charge for the InfoExtractors
  43         that are added to it, so this is a "mutual registration".
  44
  45         Available options:
  46
  47         username:         Username for authentication purposes.
  48         password:         Password for authentication purposes.
  49         usenetrc:         Use netrc for authentication instead.
  50         quiet:            Do not print messages to stdout.
  51         forceurl:         Force printing final URL.
  52         forcetitle:       Force printing title.
  53         forcethumbnail:   Force printing thumbnail URL.
  54         forcedescription: Force printing description.
  55         forcefilename:    Force printing final filename.
  56         simulate:         Do not download the video files.
  57         format:           Video format code.
  58         format_limit:     Highest quality format to try.
  59         outtmpl:          Template for output names.
  60         ignoreerrors:     Do not stop on download errors.
  61         ratelimit:        Download speed limit, in bytes/sec.
  62         nooverwrites:     Prevent overwriting files.
  63         retries:          Number of times to retry for HTTP error 5xx
  64         buffersize:       Size of download buffer in bytes.
  65         noresizebuffer:   Do not automatically resize the download buffer.
  66         continuedl:       Try to continue downloads if possible.
  67         noprogress:       Do not print the progress bar.
  68         playliststart:    Playlist item to start at.
  69         playlistend:      Playlist item to end at.
  70         matchtitle:       Download only matching titles.
  71         rejecttitle:      Reject downloads for matching titles.
  72         logtostderr:      Log messages to stderr instead of stdout.
  73         consoletitle:     Display progress in console window's titlebar.
  74         nopart:           Do not use temporary .part files.
  75         updatetime:       Use the Last-modified header to set output file timestamps.
  76         writedescription: Write the video description to a .description file
  77         writeinfojson:    Write the video description to a .info.json file
  78         writesubtitles:   Write the video subtitles to a .srt file
  79         subtitleslang:    Language of the subtitles to download
  80         """
  81
  82         params = None
  83         _ies = []
  84         _pps = []
  85         _download_retcode = None
  86         _num_downloads = None
  87         _screen_file = None
  88
  89         def __init__(self, params):
  90                 """Create a FileDownloader object with the given options."""
  91                 self._ies = []
  92                 self._pps = []
  93                 self._download_retcode = 0
  94                 self._num_downloads = 0
  95                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
  96                 self.params = params
  97
  98         @staticmethod
  99         def format_bytes(bytes):
 100                 if bytes is None:
 101                         return 'N/A'
 102                 if type(bytes) is str:
 103                         bytes = float(bytes)
 104                 if bytes == 0.0:
 105                         exponent = 0
 106                 else:
 107                         exponent = long(math.log(bytes, 1024.0))
 108                 suffix = 'bkMGTPEZY'[exponent]
 109                 converted = float(bytes) / float(1024 ** exponent)
 110                 return '%.2f%s' % (converted, suffix)
 111
 112         @staticmethod
 113         def calc_percent(byte_counter, data_len):
 114                 if data_len is None:
 115                         return '---.-%'
 116                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 117
 118         @staticmethod
 119         def calc_eta(start, now, total, current):
 120                 if total is None:
 121                         return '--:--'
 122                 dif = now - start
 123                 if current == 0 or dif < 0.001: # One millisecond
 124                         return '--:--'
 125                 rate = float(current) / dif
 126                 eta = long((float(total) - float(current)) / rate)
 127                 (eta_mins, eta_secs) = divmod(eta, 60)
 128                 if eta_mins > 99:
 129                         return '--:--'
 130                 return '%02d:%02d' % (eta_mins, eta_secs)
 131
 132         @staticmethod
 133         def calc_speed(start, now, bytes):
 134                 dif = now - start
 135                 if bytes == 0 or dif < 0.001: # One millisecond
 136                         return '%10s' % '---b/s'
 137                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 138
 139         @staticmethod
 140         def best_block_size(elapsed_time, bytes):
 141                 new_min = max(bytes / 2.0, 1.0)
 142                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 143                 if elapsed_time < 0.001:
 144                         return long(new_max)
 145                 rate = bytes / elapsed_time
 146                 if rate > new_max:
 147                         return long(new_max)
 148                 if rate < new_min:
 149                         return long(new_min)
 150                 return long(rate)
 151
 152         @staticmethod
 153         def parse_bytes(bytestr):
 154                 """Parse a string indicating a byte quantity into a long integer."""
 155                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 156                 if matchobj is None:
 157                         return None
 158                 number = float(matchobj.group(1))
 159                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 160                 return long(round(number * multiplier))
 161
 162         def add_info_extractor(self, ie):
 163                 """Add an InfoExtractor object to the end of the list."""
 164                 self._ies.append(ie)
 165                 ie.set_downloader(self)
 166
 167         def add_post_processor(self, pp):
 168                 """Add a PostProcessor object to the end of the chain."""
 169                 self._pps.append(pp)
 170                 pp.set_downloader(self)
 171
 172         def to_screen(self, message, skip_eol=False):
 173                 """Print message to stdout if not in quiet mode."""
 174                 assert type(message) == type(u'')
 175                 if not self.params.get('quiet', False):
 176                         terminator = [u'\n', u''][skip_eol]
 177                         output = message + terminator
 178
 179                         if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
 180                                 output = output.encode(preferredencoding(), 'ignore')
 181                         self._screen_file.write(output)
 182                         self._screen_file.flush()
 183
 184         def to_stderr(self, message):
 185                 """Print message to stderr."""
 186                 print >>sys.stderr, message.encode(preferredencoding())
 187
 188         def to_cons_title(self, message):
 189                 """Set console/terminal window title to message."""
 190                 if not self.params.get('consoletitle', False):
 191                         return
 192                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 193                         # c_wchar_p() might not be necessary if `message` is
 194                         # already of type unicode()
 195                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 196                 elif 'TERM' in os.environ:
 197                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
 198
 199         def fixed_template(self):
 200                 """Checks if the output template is fixed."""
 201                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 202
 203         def trouble(self, message=None):
 204                 """Determine action to take when a download problem appears.
 205
 206                 Depending on if the downloader has been configured to ignore
 207                 download errors or not, this method may throw an exception or
 208                 not when errors are found, after printing the message.
 209                 """
 210                 if message is not None:
 211                         self.to_stderr(message)
 212                 if not self.params.get('ignoreerrors', False):
 213                         raise DownloadError(message)
 214                 self._download_retcode = 1
 215
 216         def slow_down(self, start_time, byte_counter):
 217                 """Sleep if the download speed is over the rate limit."""
 218                 rate_limit = self.params.get('ratelimit', None)
 219                 if rate_limit is None or byte_counter == 0:
 220                         return
 221                 now = time.time()
 222                 elapsed = now - start_time
 223                 if elapsed <= 0.0:
 224                         return
 225                 speed = float(byte_counter) / elapsed
 226                 if speed > rate_limit:
 227                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 228
 229         def temp_name(self, filename):
 230                 """Returns a temporary filename for the given filename."""
 231                 if self.params.get('nopart', False) or filename == u'-' or \
 232                                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 233                         return filename
 234                 return filename + u'.part'
 235
 236         def undo_temp_name(self, filename):
 237                 if filename.endswith(u'.part'):
 238                         return filename[:-len(u'.part')]
 239                 return filename
 240
 241         def try_rename(self, old_filename, new_filename):
 242                 try:
 243                         if old_filename == new_filename:
 244                                 return
 245                         os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
 246                 except (IOError, OSError), err:
 247                         self.trouble(u'ERROR: unable to rename file')
 248
 249         def try_utime(self, filename, last_modified_hdr):
 250                 """Try to set the last-modified time of the given file."""
 251                 if last_modified_hdr is None:
 252                         return
 253                 if not os.path.isfile(encodeFilename(filename)):
 254                         return
 255                 timestr = last_modified_hdr
 256                 if timestr is None:
 257                         return
 258                 filetime = timeconvert(timestr)
 259                 if filetime is None:
 260                         return filetime
 261                 try:
 262                         os.utime(filename, (time.time(), filetime))
 263                 except:
 264                         pass
 265                 return filetime
 266
 267         def report_writedescription(self, descfn):
 268                 """ Report that the description file is being written """
 269                 self.to_screen(u'[info] Writing video description to: ' + descfn)
 270
 271         def report_writesubtitles(self, srtfn):
 272                 """ Report that the subtitles file is being written """
 273                 self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
 274
 275         def report_writeinfojson(self, infofn):
 276                 """ Report that the metadata file has been written """
 277                 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
 278
 279         def report_destination(self, filename):
 280                 """Report destination filename."""
 281                 self.to_screen(u'[download] Destination: ' + filename)
 282
 283         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 284                 """Report download progress."""
 285                 if self.params.get('noprogress', False):
 286                         return
 287                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 288                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 289                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
 290                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 291
 292         def report_resuming_byte(self, resume_len):
 293                 """Report attempt to resume at given byte."""
 294                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 295
 296         def report_retry(self, count, retries):
 297                 """Report retry in case of HTTP error 5xx"""
 298                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 299
 300         def report_file_already_downloaded(self, file_name):
 301                 """Report file has already been fully downloaded."""
 302                 try:
 303                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 304                 except (UnicodeEncodeError), err:
 305                         self.to_screen(u'[download] The file has already been downloaded')
 306
 307         def report_unable_to_resume(self):
 308                 """Report it was impossible to resume download."""
 309                 self.to_screen(u'[download] Unable to resume')
 310
 311         def report_finish(self):
 312                 """Report download finished."""
 313                 if self.params.get('noprogress', False):
 314                         self.to_screen(u'[download] Download completed')
 315                 else:
 316                         self.to_screen(u'')
 317
 318         def increment_downloads(self):
 319                 """Increment the ordinal that assigns a number to each file."""
 320                 self._num_downloads += 1
 321
 322         def prepare_filename(self, info_dict):
 323                 """Generate the output filename."""
 324                 try:
 325                         template_dict = dict(info_dict)
 326                         template_dict['epoch'] = unicode(long(time.time()))
 327                         template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
 328                         filename = self.params['outtmpl'] % template_dict
 329                         return filename
 330                 except (ValueError, KeyError), err:
 331                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 332                         return None
 333
 334         def _match_entry(self, info_dict):
 335                 """ Returns None iff the file should be downloaded """
 336
 337                 title = info_dict['title']
 338                 matchtitle = self.params.get('matchtitle', False)
 339                 if matchtitle and not re.search(matchtitle, title, re.IGNORECASE):
 340                         return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
 341                 rejecttitle = self.params.get('rejecttitle', False)
 342                 if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE):
 343                         return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 344                 return None
 345
 346         def process_info(self, info_dict):
 347                 """Process a single dictionary returned by an InfoExtractor."""
 348
 349                 info_dict['stitle'] = sanitize_filename(info_dict['title'])
 350
 351                 reason = self._match_entry(info_dict)
 352                 if reason is not None:
 353                         self.to_screen(u'[download] ' + reason)
 354                         return
 355
 356                 max_downloads = self.params.get('max_downloads')
 357                 if max_downloads is not None:
 358                         if self._num_downloads > int(max_downloads):
 359                                 raise MaxDownloadsReached()
 360
 361                 filename = self.prepare_filename(info_dict)
 362
 363                 # Forced printings
 364                 if self.params.get('forcetitle', False):
 365                         print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
 366                 if self.params.get('forceurl', False):
 367                         print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
 368                 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 369                         print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
 370                 if self.params.get('forcedescription', False) and 'description' in info_dict:
 371                         print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
 372                 if self.params.get('forcefilename', False) and filename is not None:
 373                         print filename.encode(preferredencoding(), 'xmlcharrefreplace')
 374                 if self.params.get('forceformat', False):
 375                         print info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace')
 376
 377                 # Do nothing else if in simulate mode
 378                 if self.params.get('simulate', False):
 379                         return
 380
 381                 if filename is None:
 382                         return
 383
 384                 try:
 385                         dn = os.path.dirname(encodeFilename(filename))
 386                         if dn != '' and not os.path.exists(dn): # dn is already encoded
 387                                 os.makedirs(dn)
 388                 except (OSError, IOError), err:
 389                         self.trouble(u'ERROR: unable to create directory ' + unicode(err))
 390                         return
 391
 392                 if self.params.get('writedescription', False):
 393                         try:
 394                                 descfn = filename + u'.description'
 395                                 self.report_writedescription(descfn)
 396                                 descfile = open(encodeFilename(descfn), 'wb')
 397                                 try:
 398                                         descfile.write(info_dict['description'].encode('utf-8'))
 399                                 finally:
 400                                         descfile.close()
 401                         except (OSError, IOError):
 402                                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
 403                                 return
 404
 405                 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
 406                         # subtitles download errors are already managed as troubles in relevant IE
 407                         # that way it will silently go on when used with unsupporting IE
 408                         try:
 409                                 srtfn = filename.rsplit('.', 1)[0] + u'.srt'
 410                                 self.report_writesubtitles(srtfn)
 411                                 srtfile = open(encodeFilename(srtfn), 'wb')
 412                                 try:
 413                                         srtfile.write(info_dict['subtitles'].encode('utf-8'))
 414                                 finally:
 415                                         srtfile.close()
 416                         except (OSError, IOError):
 417                                 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
 418                                 return
 419
 420                 if self.params.get('writeinfojson', False):
 421                         infofn = filename + u'.info.json'
 422                         self.report_writeinfojson(infofn)
 423                         try:
 424                                 json.dump
 425                         except (NameError,AttributeError):
 426                                 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
 427                                 return
 428                         try:
 429                                 infof = open(encodeFilename(infofn), 'wb')
 430                                 try:
 431                                         json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
 432                                         json.dump(json_info_dict, infof)
 433                                 finally:
 434                                         infof.close()
 435                         except (OSError, IOError):
 436                                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
 437                                 return
 438
 439                 if not self.params.get('skip_download', False):
 440                         if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
 441                                 success = True
 442                         else:
 443                                 try:
 444                                         success = self._do_download(filename, info_dict)
 445                                 except (OSError, IOError), err:
 446                                         raise UnavailableVideoError
 447                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 448                                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 449                                         return
 450                                 except (ContentTooShortError, ), err:
 451                                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 452                                         return
 453
 454                         if success:
 455                                 try:
 456                                         self.post_process(filename, info_dict)
 457                                 except (PostProcessingError), err:
 458                                         self.trouble(u'ERROR: postprocessing: %s' % str(err))
 459                                         return
 460
 461         def download(self, url_list):
 462                 """Download a given list of URLs."""
 463                 if len(url_list) > 1 and self.fixed_template():
 464                         raise SameFileError(self.params['outtmpl'])
 465
 466                 for url in url_list:
 467                         suitable_found = False
 468                         for ie in self._ies:
 469                                 # Go to next InfoExtractor if not suitable
 470                                 if not ie.suitable(url):
 471                                         continue
 472
 473                                 # Suitable InfoExtractor found
 474                                 suitable_found = True
 475
 476                                 # Extract information from URL and process it
 477                                 videos = ie.extract(url)
 478                                 for video in videos or []:
 479                                         try:
 480                                                 self.increment_downloads()
 481                                                 self.process_info(video)
 482                                         except UnavailableVideoError:
 483                                                 self.trouble(u'\nERROR: unable to download video')
 484
 485                                 # Suitable InfoExtractor had been found; go to next URL
 486                                 break
 487
 488                         if not suitable_found:
 489                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 490
 491                 return self._download_retcode
 492
 493         def post_process(self, filename, ie_info):
 494                 """Run the postprocessing chain on the given file."""
 495                 info = dict(ie_info)
 496                 info['filepath'] = filename
 497                 for pp in self._pps:
 498                         info = pp.run(info)
 499                         if info is None:
 500                                 break
 501
 502         def _download_with_rtmpdump(self, filename, url, player_url):
 503                 self.report_destination(filename)
 504                 tmpfilename = self.temp_name(filename)
 505
 506                 # Check for rtmpdump first
 507                 try:
 508                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 509                 except (OSError, IOError):
 510                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 511                         return False
 512
 513                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 514                 # the connection was interrumpted and resuming appears to be
 515                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 516                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 517                 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
 518                 if self.params.get('verbose', False):
 519                         try:
 520                                 import pipes
 521                                 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
 522                         except ImportError:
 523                                 shell_quote = repr
 524                         self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
 525                 retval = subprocess.call(args)
 526                 while retval == 2 or retval == 1:
 527                         prevsize = os.path.getsize(encodeFilename(tmpfilename))
 528                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 529                         time.sleep(5.0) # This seems to be needed
 530                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 531                         cursize = os.path.getsize(encodeFilename(tmpfilename))
 532                         if prevsize == cursize and retval == 1:
 533                                 break
 534                          # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
 535                         if prevsize == cursize and retval == 2 and cursize > 1024:
 536                                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
 537                                 retval = 0
 538                                 break
 539                 if retval == 0:
 540                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename)))
 541                         self.try_rename(tmpfilename, filename)
 542                         return True
 543                 else:
 544                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 545                         return False
 546
 547         def _do_download(self, filename, info_dict):
 548                 url = info_dict['url']
 549                 player_url = info_dict.get('player_url', None)
 550
 551                 # Check file already present
 552                 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
 553                         self.report_file_already_downloaded(filename)
 554                         return True
 555
 556                 # Attempt to download using rtmpdump
 557                 if url.startswith('rtmp'):
 558                         return self._download_with_rtmpdump(filename, url, player_url)
 559
 560                 tmpfilename = self.temp_name(filename)
 561                 stream = None
 562
 563                 # Do not include the Accept-Encoding header
 564                 headers = {'Youtubedl-no-compression': 'True'}
 565                 basic_request = urllib2.Request(url, None, headers)
 566                 request = urllib2.Request(url, None, headers)
 567
 568                 # Establish possible resume length
 569                 if os.path.isfile(encodeFilename(tmpfilename)):
 570                         resume_len = os.path.getsize(encodeFilename(tmpfilename))
 571                 else:
 572                         resume_len = 0
 573
 574                 open_mode = 'wb'
 575                 if resume_len != 0:
 576                         if self.params.get('continuedl', False):
 577                                 self.report_resuming_byte(resume_len)
 578                                 request.add_header('Range','bytes=%d-' % resume_len)
 579                                 open_mode = 'ab'
 580                         else:
 581                                 resume_len = 0
 582
 583                 count = 0
 584                 retries = self.params.get('retries', 0)
 585                 while count <= retries:
 586                         # Establish connection
 587                         try:
 588                                 if count == 0 and 'urlhandle' in info_dict:
 589                                         data = info_dict['urlhandle']
 590                                 data = urllib2.urlopen(request)
 591                                 break
 592                         except (urllib2.HTTPError, ), err:
 593                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 594                                         # Unexpected HTTP error
 595                                         raise
 596                                 elif err.code == 416:
 597                                         # Unable to resume (requested range not satisfiable)
 598                                         try:
 599                                                 # Open the connection again without the range header
 600                                                 data = urllib2.urlopen(basic_request)
 601                                                 content_length = data.info()['Content-Length']
 602                                         except (urllib2.HTTPError, ), err:
 603                                                 if err.code < 500 or err.code >= 600:
 604                                                         raise
 605                                         else:
 606                                                 # Examine the reported length
 607                                                 if (content_length is not None and
 608                                                                 (resume_len - 100 < long(content_length) < resume_len + 100)):
 609                                                         # The file had already been fully downloaded.
 610                                                         # Explanation to the above condition: in issue #175 it was revealed that
 611                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 612                                                         # changing the file size slightly and causing problems for some users. So
 613                                                         # I decided to implement a suggested change and consider the file
 614                                                         # completely downloaded if the file size differs less than 100 bytes from
 615                                                         # the one in the hard drive.
 616                                                         self.report_file_already_downloaded(filename)
 617                                                         self.try_rename(tmpfilename, filename)
 618                                                         return True
 619                                                 else:
 620                                                         # The length does not match, we start the download over
 621                                                         self.report_unable_to_resume()
 622                                                         open_mode = 'wb'
 623                                                         break
 624                         # Retry
 625                         count += 1
 626                         if count <= retries:
 627                                 self.report_retry(count, retries)
 628
 629                 if count > retries:
 630                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 631                         return False
 632
 633                 data_len = data.info().get('Content-length', None)
 634                 if data_len is not None:
 635                         data_len = long(data_len) + resume_len
 636                 data_len_str = self.format_bytes(data_len)
 637                 byte_counter = 0 + resume_len
 638                 block_size = self.params.get('buffersize', 1024)
 639                 start = time.time()
 640                 while True:
 641                         # Download and write
 642                         before = time.time()
 643                         data_block = data.read(block_size)
 644                         after = time.time()
 645                         if len(data_block) == 0:
 646                                 break
 647                         byte_counter += len(data_block)
 648
 649                         # Open file just in time
 650                         if stream is None:
 651                                 try:
 652                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 653                                         assert stream is not None
 654                                         filename = self.undo_temp_name(tmpfilename)
 655                                         self.report_destination(filename)
 656                                 except (OSError, IOError), err:
 657                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 658                                         return False
 659                         try:
 660                                 stream.write(data_block)
 661                         except (IOError, OSError), err:
 662                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 663                                 return False
 664                         if not self.params.get('noresizebuffer', False):
 665                                 block_size = self.best_block_size(after - before, len(data_block))
 666
 667                         # Progress message
 668                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
 669                         if data_len is None:
 670                                 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
 671                         else:
 672                                 percent_str = self.calc_percent(byte_counter, data_len)
 673                                 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 674                                 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 675
 676                         # Apply rate limit
 677                         self.slow_down(start, byte_counter - resume_len)
 678
 679                 if stream is None:
 680                         self.trouble(u'\nERROR: Did not get any data blocks')
 681                         return False
 682                 stream.close()
 683                 self.report_finish()
 684                 if data_len is not None and byte_counter != data_len:
 685                         raise ContentTooShortError(byte_counter, long(data_len))
 686                 self.try_rename(tmpfilename, filename)
 687
 688                 # Update file modification time
 689                 if self.params.get('updatetime', True):
 690                         info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
 691
 692                 return True