_ Git - youtube-dl/blob - youtube_dl/FileDownloader.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import httplib
   5 import math
   6 import os
   7 import re
   8 import socket
   9 import subprocess
  10 import sys
  11 import time
  12 import urllib2
  13
  14 if os.name == 'nt':
  15         import ctypes
  16
  17 from utils import *
  18
  19
  20 class FileDownloader(object):
  21         """File Downloader class.
  22
  23         File downloader objects are the ones responsible of downloading the
  24         actual video file and writing it to disk if the user has requested
  25         it, among some other tasks. In most cases there should be one per
  26         program. As, given a video URL, the downloader doesn't know how to
  27         extract all the needed information, task that InfoExtractors do, it
  28         has to pass the URL to one of them.
  29
  30         For this, file downloader objects have a method that allows
  31         InfoExtractors to be registered in a given order. When it is passed
  32         a URL, the file downloader handles it to the first InfoExtractor it
  33         finds that reports being able to handle it. The InfoExtractor extracts
  34         all the information about the video or videos the URL refers to, and
  35         asks the FileDownloader to process the video information, possibly
  36         downloading the video.
  37
  38         File downloaders accept a lot of parameters. In order not to saturate
  39         the object constructor with arguments, it receives a dictionary of
  40         options instead. These options are available through the params
  41         attribute for the InfoExtractors to use. The FileDownloader also
  42         registers itself as the downloader in charge for the InfoExtractors
  43         that are added to it, so this is a "mutual registration".
  44
  45         Available options:
  46
  47         username:          Username for authentication purposes.
  48         password:          Password for authentication purposes.
  49         usenetrc:          Use netrc for authentication instead.
  50         quiet:             Do not print messages to stdout.
  51         forceurl:          Force printing final URL.
  52         forcetitle:        Force printing title.
  53         forcethumbnail:    Force printing thumbnail URL.
  54         forcedescription:  Force printing description.
  55         forcefilename:     Force printing final filename.
  56         simulate:          Do not download the video files.
  57         format:            Video format code.
  58         format_limit:      Highest quality format to try.
  59         outtmpl:           Template for output names.
  60         restrictfilenames: Do not allow "&" and spaces in file names
  61         ignoreerrors:      Do not stop on download errors.
  62         ratelimit:         Download speed limit, in bytes/sec.
  63         nooverwrites:      Prevent overwriting files.
  64         retries:           Number of times to retry for HTTP error 5xx
  65         continuedl:        Try to continue downloads if possible.
  66         noprogress:        Do not print the progress bar.
  67         playliststart:     Playlist item to start at.
  68         playlistend:       Playlist item to end at.
  69         matchtitle:        Download only matching titles.
  70         rejecttitle:       Reject downloads for matching titles.
  71         logtostderr:       Log messages to stderr instead of stdout.
  72         consoletitle:      Display progress in console window's titlebar.
  73         nopart:            Do not use temporary .part files.
  74         updatetime:        Use the Last-modified header to set output file timestamps.
  75         writedescription:  Write the video description to a .description file
  76         writeinfojson:     Write the video description to a .info.json file
  77         writesubtitles:    Write the video subtitles to a .srt file
  78         subtitleslang:     Language of the subtitles to download
  79         """
  80
  81         params = None
  82         _ies = []
  83         _pps = []
  84         _download_retcode = None
  85         _num_downloads = None
  86         _screen_file = None
  87
  88         def __init__(self, params):
  89                 """Create a FileDownloader object with the given options."""
  90                 self._ies = []
  91                 self._pps = []
  92                 self._download_retcode = 0
  93                 self._num_downloads = 0
  94                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
  95                 self.params = params
  96
  97                 if '%(stitle)s' in self.params['outtmpl']:
  98                         self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
  99
 100         @staticmethod
 101         def format_bytes(bytes):
 102                 if bytes is None:
 103                         return 'N/A'
 104                 if type(bytes) is str:
 105                         bytes = float(bytes)
 106                 if bytes == 0.0:
 107                         exponent = 0
 108                 else:
 109                         exponent = long(math.log(bytes, 1024.0))
 110                 suffix = 'bkMGTPEZY'[exponent]
 111                 converted = float(bytes) / float(1024 ** exponent)
 112                 return '%.2f%s' % (converted, suffix)
 113
 114         @staticmethod
 115         def calc_percent(byte_counter, data_len):
 116                 if data_len is None:
 117                         return '---.-%'
 118                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 119
 120         @staticmethod
 121         def calc_eta(start, now, total, current):
 122                 if total is None:
 123                         return '--:--'
 124                 dif = now - start
 125                 if current == 0 or dif < 0.001: # One millisecond
 126                         return '--:--'
 127                 rate = float(current) / dif
 128                 eta = long((float(total) - float(current)) / rate)
 129                 (eta_mins, eta_secs) = divmod(eta, 60)
 130                 if eta_mins > 99:
 131                         return '--:--'
 132                 return '%02d:%02d' % (eta_mins, eta_secs)
 133
 134         @staticmethod
 135         def calc_speed(start, now, bytes):
 136                 dif = now - start
 137                 if bytes == 0 or dif < 0.001: # One millisecond
 138                         return '%10s' % '---b/s'
 139                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 140
 141         @staticmethod
 142         def best_block_size(elapsed_time, bytes):
 143                 new_min = max(bytes / 2.0, 1.0)
 144                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 145                 if elapsed_time < 0.001:
 146                         return int(new_max)
 147                 rate = bytes / elapsed_time
 148                 if rate > new_max:
 149                         return int(new_max)
 150                 if rate < new_min:
 151                         return int(new_min)
 152                 return int(rate)
 153
 154         @staticmethod
 155         def parse_bytes(bytestr):
 156                 """Parse a string indicating a byte quantity into an integer."""
 157                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 158                 if matchobj is None:
 159                         return None
 160                 number = float(matchobj.group(1))
 161                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 162                 return int(round(number * multiplier))
 163
 164         def add_info_extractor(self, ie):
 165                 """Add an InfoExtractor object to the end of the list."""
 166                 self._ies.append(ie)
 167                 ie.set_downloader(self)
 168
 169         def add_post_processor(self, pp):
 170                 """Add a PostProcessor object to the end of the chain."""
 171                 self._pps.append(pp)
 172                 pp.set_downloader(self)
 173
 174         def to_screen(self, message, skip_eol=False):
 175                 """Print message to stdout if not in quiet mode."""
 176                 assert type(message) == type(u'')
 177                 if not self.params.get('quiet', False):
 178                         terminator = [u'\n', u''][skip_eol]
 179                         output = message + terminator
 180                         if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
 181                                 output = output.encode(preferredencoding(), 'ignore')
 182                         self._screen_file.write(output)
 183                         self._screen_file.flush()
 184
 185         def to_stderr(self, message):
 186                 """Print message to stderr."""
 187                 assert type(message) == type(u'')
 188                 sys.stderr.write((message + u'\n').encode(preferredencoding()))
 189
 190         def to_cons_title(self, message):
 191                 """Set console/terminal window title to message."""
 192                 if not self.params.get('consoletitle', False):
 193                         return
 194                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 195                         # c_wchar_p() might not be necessary if `message` is
 196                         # already of type unicode()
 197                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 198                 elif 'TERM' in os.environ:
 199                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
 200
 201         def fixed_template(self):
 202                 """Checks if the output template is fixed."""
 203                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 204
 205         def trouble(self, message=None):
 206                 """Determine action to take when a download problem appears.
 207
 208                 Depending on if the downloader has been configured to ignore
 209                 download errors or not, this method may throw an exception or
 210                 not when errors are found, after printing the message.
 211                 """
 212                 if message is not None:
 213                         self.to_stderr(message)
 214                 if not self.params.get('ignoreerrors', False):
 215                         raise DownloadError(message)
 216                 self._download_retcode = 1
 217
 218         def slow_down(self, start_time, byte_counter):
 219                 """Sleep if the download speed is over the rate limit."""
 220                 rate_limit = self.params.get('ratelimit', None)
 221                 if rate_limit is None or byte_counter == 0:
 222                         return
 223                 now = time.time()
 224                 elapsed = now - start_time
 225                 if elapsed <= 0.0:
 226                         return
 227                 speed = float(byte_counter) / elapsed
 228                 if speed > rate_limit:
 229                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 230
 231         def temp_name(self, filename):
 232                 """Returns a temporary filename for the given filename."""
 233                 if self.params.get('nopart', False) or filename == u'-' or \
 234                                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 235                         return filename
 236                 return filename + u'.part'
 237
 238         def undo_temp_name(self, filename):
 239                 if filename.endswith(u'.part'):
 240                         return filename[:-len(u'.part')]
 241                 return filename
 242
 243         def try_rename(self, old_filename, new_filename):
 244                 try:
 245                         if old_filename == new_filename:
 246                                 return
 247                         os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
 248                 except (IOError, OSError), err:
 249                         self.trouble(u'ERROR: unable to rename file')
 250
 251         def try_utime(self, filename, last_modified_hdr):
 252                 """Try to set the last-modified time of the given file."""
 253                 if last_modified_hdr is None:
 254                         return
 255                 if not os.path.isfile(encodeFilename(filename)):
 256                         return
 257                 timestr = last_modified_hdr
 258                 if timestr is None:
 259                         return
 260                 filetime = timeconvert(timestr)
 261                 if filetime is None:
 262                         return filetime
 263                 try:
 264                         os.utime(filename, (time.time(), filetime))
 265                 except:
 266                         pass
 267                 return filetime
 268
 269         def report_writedescription(self, descfn):
 270                 """ Report that the description file is being written """
 271                 self.to_screen(u'[info] Writing video description to: ' + descfn)
 272
 273         def report_writesubtitles(self, srtfn):
 274                 """ Report that the subtitles file is being written """
 275                 self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
 276
 277         def report_writeinfojson(self, infofn):
 278                 """ Report that the metadata file has been written """
 279                 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
 280
 281         def report_destination(self, filename):
 282                 """Report destination filename."""
 283                 self.to_screen(u'[download] Destination: ' + filename)
 284
 285         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 286                 """Report download progress."""
 287                 if self.params.get('noprogress', False):
 288                         return
 289                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 290                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 291                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
 292                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 293
 294         def report_resuming_byte(self, resume_len):
 295                 """Report attempt to resume at given byte."""
 296                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 297
 298         def report_retry(self, count, retries):
 299                 """Report retry in case of HTTP error 5xx"""
 300                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 301
 302         def report_file_already_downloaded(self, file_name):
 303                 """Report file has already been fully downloaded."""
 304                 try:
 305                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 306                 except (UnicodeEncodeError), err:
 307                         self.to_screen(u'[download] The file has already been downloaded')
 308
 309         def report_unable_to_resume(self):
 310                 """Report it was impossible to resume download."""
 311                 self.to_screen(u'[download] Unable to resume')
 312
 313         def report_finish(self):
 314                 """Report download finished."""
 315                 if self.params.get('noprogress', False):
 316                         self.to_screen(u'[download] Download completed')
 317                 else:
 318                         self.to_screen(u'')
 319
 320         def increment_downloads(self):
 321                 """Increment the ordinal that assigns a number to each file."""
 322                 self._num_downloads += 1
 323
 324         def prepare_filename(self, info_dict):
 325                 """Generate the output filename."""
 326                 try:
 327                         template_dict = dict(info_dict)
 328                         template_dict['epoch'] = unicode(int(time.time()))
 329                         template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
 330                         filename = self.params['outtmpl'] % template_dict
 331                         return filename
 332                 except (ValueError, KeyError), err:
 333                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 334                         return None
 335
 336         def _match_entry(self, info_dict):
 337                 """ Returns None iff the file should be downloaded """
 338
 339                 title = info_dict['title']
 340                 matchtitle = self.params.get('matchtitle', False)
 341                 if matchtitle:
 342                         matchtitle = matchtitle.decode('utf8')
 343                         if not re.search(matchtitle, title, re.IGNORECASE):
 344                                 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
 345                 rejecttitle = self.params.get('rejecttitle', False)
 346                 if rejecttitle:
 347                         rejecttitle = rejecttitle.decode('utf8')
 348                         if re.search(rejecttitle, title, re.IGNORECASE):
 349                                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 350                 return None
 351
 352         def process_info(self, info_dict):
 353                 """Process a single dictionary returned by an InfoExtractor."""
 354
 355                 # Keep for backwards compatibility
 356                 info_dict['stitle'] = info_dict['title']
 357
 358                 if not 'format' in info_dict:
 359                         info_dict['format'] = info_dict['ext']
 360
 361                 reason = self._match_entry(info_dict)
 362                 if reason is not None:
 363                         self.to_screen(u'[download] ' + reason)
 364                         return
 365
 366                 max_downloads = self.params.get('max_downloads')
 367                 if max_downloads is not None:
 368                         if self._num_downloads > int(max_downloads):
 369                                 raise MaxDownloadsReached()
 370
 371                 filename = self.prepare_filename(info_dict)
 372                 filename = sanitize_filename(filename, self.params.get('restrictfilenames'))
 373
 374                 # Forced printings
 375                 if self.params.get('forcetitle', False):
 376                         print(info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace'))
 377                 if self.params.get('forceurl', False):
 378                         print(info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace'))
 379                 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 380                         print(info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace'))
 381                 if self.params.get('forcedescription', False) and 'description' in info_dict:
 382                         print(info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace'))
 383                 if self.params.get('forcefilename', False) and filename is not None:
 384                         print(filename.encode(preferredencoding(), 'xmlcharrefreplace'))
 385                 if self.params.get('forceformat', False):
 386                         print(info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace'))
 387
 388                 # Do nothing else if in simulate mode
 389                 if self.params.get('simulate', False):
 390                         return
 391
 392                 if filename is None:
 393                         return
 394
 395                 try:
 396                         dn = os.path.dirname(encodeFilename(filename))
 397                         if dn != '' and not os.path.exists(dn): # dn is already encoded
 398                                 os.makedirs(dn)
 399                 except (OSError, IOError), err:
 400                         self.trouble(u'ERROR: unable to create directory ' + unicode(err))
 401                         return
 402
 403                 if self.params.get('writedescription', False):
 404                         try:
 405                                 descfn = filename + u'.description'
 406                                 self.report_writedescription(descfn)
 407                                 descfile = open(encodeFilename(descfn), 'wb')
 408                                 try:
 409                                         descfile.write(info_dict['description'].encode('utf-8'))
 410                                 finally:
 411                                         descfile.close()
 412                         except (OSError, IOError):
 413                                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
 414                                 return
 415
 416                 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
 417                         # subtitles download errors are already managed as troubles in relevant IE
 418                         # that way it will silently go on when used with unsupporting IE
 419                         try:
 420                                 srtfn = filename.rsplit('.', 1)[0] + u'.srt'
 421                                 self.report_writesubtitles(srtfn)
 422                                 srtfile = open(encodeFilename(srtfn), 'wb')
 423                                 try:
 424                                         srtfile.write(info_dict['subtitles'].encode('utf-8'))
 425                                 finally:
 426                                         srtfile.close()
 427                         except (OSError, IOError):
 428                                 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
 429                                 return
 430
 431                 if self.params.get('writeinfojson', False):
 432                         infofn = filename + u'.info.json'
 433                         self.report_writeinfojson(infofn)
 434                         try:
 435                                 json.dump
 436                         except (NameError,AttributeError):
 437                                 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
 438                                 return
 439                         try:
 440                                 infof = open(encodeFilename(infofn), 'wb')
 441                                 try:
 442                                         json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
 443                                         json.dump(json_info_dict, infof)
 444                                 finally:
 445                                         infof.close()
 446                         except (OSError, IOError):
 447                                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
 448                                 return
 449
 450                 if not self.params.get('skip_download', False):
 451                         if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
 452                                 success = True
 453                         else:
 454                                 try:
 455                                         success = self._do_download(filename, info_dict)
 456                                 except (OSError, IOError), err:
 457                                         raise UnavailableVideoError
 458                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 459                                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 460                                         return
 461                                 except (ContentTooShortError, ), err:
 462                                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 463                                         return
 464
 465                         if success:
 466                                 try:
 467                                         self.post_process(filename, info_dict)
 468                                 except (PostProcessingError), err:
 469                                         self.trouble(u'ERROR: postprocessing: %s' % str(err))
 470                                         return
 471
 472         def download(self, url_list):
 473                 """Download a given list of URLs."""
 474                 if len(url_list) > 1 and self.fixed_template():
 475                         raise SameFileError(self.params['outtmpl'])
 476
 477                 for url in url_list:
 478                         suitable_found = False
 479                         for ie in self._ies:
 480                                 # Go to next InfoExtractor if not suitable
 481                                 if not ie.suitable(url):
 482                                         continue
 483
 484                                 # Suitable InfoExtractor found
 485                                 suitable_found = True
 486
 487                                 # Extract information from URL and process it
 488                                 videos = ie.extract(url)
 489                                 for video in videos or []:
 490                                         video['extractor'] = ie.IE_NAME
 491                                         try:
 492                                                 self.increment_downloads()
 493                                                 self.process_info(video)
 494                                         except UnavailableVideoError:
 495                                                 self.trouble(u'\nERROR: unable to download video')
 496
 497                                 # Suitable InfoExtractor had been found; go to next URL
 498                                 break
 499
 500                         if not suitable_found:
 501                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 502
 503                 return self._download_retcode
 504
 505         def post_process(self, filename, ie_info):
 506                 """Run the postprocessing chain on the given file."""
 507                 info = dict(ie_info)
 508                 info['filepath'] = filename
 509                 for pp in self._pps:
 510                         info = pp.run(info)
 511                         if info is None:
 512                                 break
 513
 514         def _download_with_rtmpdump(self, filename, url, player_url):
 515                 self.report_destination(filename)
 516                 tmpfilename = self.temp_name(filename)
 517
 518                 # Check for rtmpdump first
 519                 try:
 520                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 521                 except (OSError, IOError):
 522                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 523                         return False
 524
 525                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 526                 # the connection was interrumpted and resuming appears to be
 527                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 528                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 529                 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
 530                 if self.params.get('verbose', False):
 531                         try:
 532                                 import pipes
 533                                 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
 534                         except ImportError:
 535                                 shell_quote = repr
 536                         self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
 537                 retval = subprocess.call(args)
 538                 while retval == 2 or retval == 1:
 539                         prevsize = os.path.getsize(encodeFilename(tmpfilename))
 540                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 541                         time.sleep(5.0) # This seems to be needed
 542                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 543                         cursize = os.path.getsize(encodeFilename(tmpfilename))
 544                         if prevsize == cursize and retval == 1:
 545                                 break
 546                          # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
 547                         if prevsize == cursize and retval == 2 and cursize > 1024:
 548                                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
 549                                 retval = 0
 550                                 break
 551                 if retval == 0:
 552                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename)))
 553                         self.try_rename(tmpfilename, filename)
 554                         return True
 555                 else:
 556                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 557                         return False
 558
 559         def _do_download(self, filename, info_dict):
 560                 url = info_dict['url']
 561                 player_url = info_dict.get('player_url', None)
 562
 563                 # Check file already present
 564                 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
 565                         self.report_file_already_downloaded(filename)
 566                         return True
 567
 568                 # Attempt to download using rtmpdump
 569                 if url.startswith('rtmp'):
 570                         return self._download_with_rtmpdump(filename, url, player_url)
 571
 572                 tmpfilename = self.temp_name(filename)
 573                 stream = None
 574
 575                 # Do not include the Accept-Encoding header
 576                 headers = {'Youtubedl-no-compression': 'True'}
 577                 basic_request = urllib2.Request(url, None, headers)
 578                 request = urllib2.Request(url, None, headers)
 579
 580                 # Establish possible resume length
 581                 if os.path.isfile(encodeFilename(tmpfilename)):
 582                         resume_len = os.path.getsize(encodeFilename(tmpfilename))
 583                 else:
 584                         resume_len = 0
 585
 586                 open_mode = 'wb'
 587                 if resume_len != 0:
 588                         if self.params.get('continuedl', False):
 589                                 self.report_resuming_byte(resume_len)
 590                                 request.add_header('Range','bytes=%d-' % resume_len)
 591                                 open_mode = 'ab'
 592                         else:
 593                                 resume_len = 0
 594
 595                 count = 0
 596                 retries = self.params.get('retries', 0)
 597                 while count <= retries:
 598                         # Establish connection
 599                         try:
 600                                 if count == 0 and 'urlhandle' in info_dict:
 601                                         data = info_dict['urlhandle']
 602                                 data = urllib2.urlopen(request)
 603                                 break
 604                         except (urllib2.HTTPError, ), err:
 605                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 606                                         # Unexpected HTTP error
 607                                         raise
 608                                 elif err.code == 416:
 609                                         # Unable to resume (requested range not satisfiable)
 610                                         try:
 611                                                 # Open the connection again without the range header
 612                                                 data = urllib2.urlopen(basic_request)
 613                                                 content_length = data.info()['Content-Length']
 614                                         except (urllib2.HTTPError, ), err:
 615                                                 if err.code < 500 or err.code >= 600:
 616                                                         raise
 617                                         else:
 618                                                 # Examine the reported length
 619                                                 if (content_length is not None and
 620                                                                 (resume_len - 100 < long(content_length) < resume_len + 100)):
 621                                                         # The file had already been fully downloaded.
 622                                                         # Explanation to the above condition: in issue #175 it was revealed that
 623                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 624                                                         # changing the file size slightly and causing problems for some users. So
 625                                                         # I decided to implement a suggested change and consider the file
 626                                                         # completely downloaded if the file size differs less than 100 bytes from
 627                                                         # the one in the hard drive.
 628                                                         self.report_file_already_downloaded(filename)
 629                                                         self.try_rename(tmpfilename, filename)
 630                                                         return True
 631                                                 else:
 632                                                         # The length does not match, we start the download over
 633                                                         self.report_unable_to_resume()
 634                                                         open_mode = 'wb'
 635                                                         break
 636                         # Retry
 637                         count += 1
 638                         if count <= retries:
 639                                 self.report_retry(count, retries)
 640
 641                 if count > retries:
 642                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 643                         return False
 644
 645                 data_len = data.info().get('Content-length', None)
 646                 if data_len is not None:
 647                         data_len = long(data_len) + resume_len
 648                 data_len_str = self.format_bytes(data_len)
 649                 byte_counter = 0 + resume_len
 650                 block_size = 1024
 651                 start = time.time()
 652                 while True:
 653                         # Download and write
 654                         before = time.time()
 655                         data_block = data.read(block_size)
 656                         after = time.time()
 657                         if len(data_block) == 0:
 658                                 break
 659                         byte_counter += len(data_block)
 660
 661                         # Open file just in time
 662                         if stream is None:
 663                                 try:
 664                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 665                                         assert stream is not None
 666                                         filename = self.undo_temp_name(tmpfilename)
 667                                         self.report_destination(filename)
 668                                 except (OSError, IOError), err:
 669                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 670                                         return False
 671                         try:
 672                                 stream.write(data_block)
 673                         except (IOError, OSError), err:
 674                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 675                                 return False
 676                         block_size = self.best_block_size(after - before, len(data_block))
 677
 678                         # Progress message
 679                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
 680                         if data_len is None:
 681                                 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
 682                         else:
 683                                 percent_str = self.calc_percent(byte_counter, data_len)
 684                                 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 685                                 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 686
 687                         # Apply rate limit
 688                         self.slow_down(start, byte_counter - resume_len)
 689
 690                 if stream is None:
 691                         self.trouble(u'\nERROR: Did not get any data blocks')
 692                         return False
 693                 stream.close()
 694                 self.report_finish()
 695                 if data_len is not None and byte_counter != data_len:
 696                         raise ContentTooShortError(byte_counter, long(data_len))
 697                 self.try_rename(tmpfilename, filename)
 698
 699                 # Update file modification time
 700                 if self.params.get('updatetime', True):
 701                         info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
 702
 703                 return True