_ Git - youtube-dl/blob - youtube_dl/FileDownloader.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import httplib
   5 import math
   6 import os
   7 import re
   8 import socket
   9 import subprocess
  10 import sys
  11 import time
  12
  13 if os.name == 'nt':
  14         import ctypes
  15
  16 from utils import *
  17
  18
  19 class FileDownloader(object):
  20         """File Downloader class.
  21
  22         File downloader objects are the ones responsible of downloading the
  23         actual video file and writing it to disk if the user has requested
  24         it, among some other tasks. In most cases there should be one per
  25         program. As, given a video URL, the downloader doesn't know how to
  26         extract all the needed information, task that InfoExtractors do, it
  27         has to pass the URL to one of them.
  28
  29         For this, file downloader objects have a method that allows
  30         InfoExtractors to be registered in a given order. When it is passed
  31         a URL, the file downloader handles it to the first InfoExtractor it
  32         finds that reports being able to handle it. The InfoExtractor extracts
  33         all the information about the video or videos the URL refers to, and
  34         asks the FileDownloader to process the video information, possibly
  35         downloading the video.
  36
  37         File downloaders accept a lot of parameters. In order not to saturate
  38         the object constructor with arguments, it receives a dictionary of
  39         options instead. These options are available through the params
  40         attribute for the InfoExtractors to use. The FileDownloader also
  41         registers itself as the downloader in charge for the InfoExtractors
  42         that are added to it, so this is a "mutual registration".
  43
  44         Available options:
  45
  46         username:          Username for authentication purposes.
  47         password:          Password for authentication purposes.
  48         usenetrc:          Use netrc for authentication instead.
  49         quiet:             Do not print messages to stdout.
  50         forceurl:          Force printing final URL.
  51         forcetitle:        Force printing title.
  52         forcethumbnail:    Force printing thumbnail URL.
  53         forcedescription:  Force printing description.
  54         forcefilename:     Force printing final filename.
  55         simulate:          Do not download the video files.
  56         format:            Video format code.
  57         format_limit:      Highest quality format to try.
  58         outtmpl:           Template for output names.
  59         restrictfilenames: Do not allow "&" and spaces in file names
  60         ignoreerrors:      Do not stop on download errors.
  61         ratelimit:         Download speed limit, in bytes/sec.
  62         nooverwrites:      Prevent overwriting files.
  63         retries:           Number of times to retry for HTTP error 5xx
  64         buffersize:        Size of download buffer in bytes.
  65         noresizebuffer:    Do not automatically resize the download buffer.
  66         continuedl:        Try to continue downloads if possible.
  67         noprogress:        Do not print the progress bar.
  68         playliststart:     Playlist item to start at.
  69         playlistend:       Playlist item to end at.
  70         matchtitle:        Download only matching titles.
  71         rejecttitle:       Reject downloads for matching titles.
  72         logtostderr:       Log messages to stderr instead of stdout.
  73         consoletitle:      Display progress in console window's titlebar.
  74         nopart:            Do not use temporary .part files.
  75         updatetime:        Use the Last-modified header to set output file timestamps.
  76         writedescription:  Write the video description to a .description file
  77         writeinfojson:     Write the video description to a .info.json file
  78         writesubtitles:    Write the video subtitles to a .srt file
  79         subtitleslang:     Language of the subtitles to download
  80         """
  81
  82         params = None
  83         _ies = []
  84         _pps = []
  85         _download_retcode = None
  86         _num_downloads = None
  87         _screen_file = None
  88
  89         def __init__(self, params):
  90                 """Create a FileDownloader object with the given options."""
  91                 self._ies = []
  92                 self._pps = []
  93                 self._download_retcode = 0
  94                 self._num_downloads = 0
  95                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
  96                 self.params = params
  97
  98                 if '%(stitle)s' in self.params['outtmpl']:
  99                         self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
 100
 101         @staticmethod
 102         def format_bytes(bytes):
 103                 if bytes is None:
 104                         return 'N/A'
 105                 if type(bytes) is str:
 106                         bytes = float(bytes)
 107                 if bytes == 0.0:
 108                         exponent = 0
 109                 else:
 110                         exponent = int(math.log(bytes, 1024.0))
 111                 suffix = 'bkMGTPEZY'[exponent]
 112                 converted = float(bytes) / float(1024 ** exponent)
 113                 return '%.2f%s' % (converted, suffix)
 114
 115         @staticmethod
 116         def calc_percent(byte_counter, data_len):
 117                 if data_len is None:
 118                         return '---.-%'
 119                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 120
 121         @staticmethod
 122         def calc_eta(start, now, total, current):
 123                 if total is None:
 124                         return '--:--'
 125                 dif = now - start
 126                 if current == 0 or dif < 0.001: # One millisecond
 127                         return '--:--'
 128                 rate = float(current) / dif
 129                 eta = int((float(total) - float(current)) / rate)
 130                 (eta_mins, eta_secs) = divmod(eta, 60)
 131                 if eta_mins > 99:
 132                         return '--:--'
 133                 return '%02d:%02d' % (eta_mins, eta_secs)
 134
 135         @staticmethod
 136         def calc_speed(start, now, bytes):
 137                 dif = now - start
 138                 if bytes == 0 or dif < 0.001: # One millisecond
 139                         return '%10s' % '---b/s'
 140                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 141
 142         @staticmethod
 143         def best_block_size(elapsed_time, bytes):
 144                 new_min = max(bytes / 2.0, 1.0)
 145                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 146                 if elapsed_time < 0.001:
 147                         return int(new_max)
 148                 rate = bytes / elapsed_time
 149                 if rate > new_max:
 150                         return int(new_max)
 151                 if rate < new_min:
 152                         return int(new_min)
 153                 return int(rate)
 154
 155         @staticmethod
 156         def parse_bytes(bytestr):
 157                 """Parse a string indicating a byte quantity into an integer."""
 158                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 159                 if matchobj is None:
 160                         return None
 161                 number = float(matchobj.group(1))
 162                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 163                 return int(round(number * multiplier))
 164
 165         def add_info_extractor(self, ie):
 166                 """Add an InfoExtractor object to the end of the list."""
 167                 self._ies.append(ie)
 168                 ie.set_downloader(self)
 169
 170         def add_post_processor(self, pp):
 171                 """Add a PostProcessor object to the end of the chain."""
 172                 self._pps.append(pp)
 173                 pp.set_downloader(self)
 174
 175         def to_screen(self, message, skip_eol=False):
 176                 """Print message to stdout if not in quiet mode."""
 177                 assert type(message) == type(u'')
 178                 if not self.params.get('quiet', False):
 179                         terminator = [u'\n', u''][skip_eol]
 180                         output = message + terminator
 181                         if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
 182                                 output = output.encode(preferredencoding(), 'ignore')
 183                         self._screen_file.write(output)
 184                         self._screen_file.flush()
 185
 186         def to_stderr(self, message):
 187                 """Print message to stderr."""
 188                 assert type(message) == type(u'')
 189                 sys.stderr.write((message + u'\n').encode(preferredencoding()))
 190
 191         def to_cons_title(self, message):
 192                 """Set console/terminal window title to message."""
 193                 if not self.params.get('consoletitle', False):
 194                         return
 195                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 196                         # c_wchar_p() might not be necessary if `message` is
 197                         # already of type unicode()
 198                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 199                 elif 'TERM' in os.environ:
 200                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
 201
 202         def fixed_template(self):
 203                 """Checks if the output template is fixed."""
 204                 return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
 205
 206         def trouble(self, message=None):
 207                 """Determine action to take when a download problem appears.
 208
 209                 Depending on if the downloader has been configured to ignore
 210                 download errors or not, this method may throw an exception or
 211                 not when errors are found, after printing the message.
 212                 """
 213                 if message is not None:
 214                         self.to_stderr(message)
 215                 if not self.params.get('ignoreerrors', False):
 216                         raise DownloadError(message)
 217                 self._download_retcode = 1
 218
 219         def slow_down(self, start_time, byte_counter):
 220                 """Sleep if the download speed is over the rate limit."""
 221                 rate_limit = self.params.get('ratelimit', None)
 222                 if rate_limit is None or byte_counter == 0:
 223                         return
 224                 now = time.time()
 225                 elapsed = now - start_time
 226                 if elapsed <= 0.0:
 227                         return
 228                 speed = float(byte_counter) / elapsed
 229                 if speed > rate_limit:
 230                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 231
 232         def temp_name(self, filename):
 233                 """Returns a temporary filename for the given filename."""
 234                 if self.params.get('nopart', False) or filename == u'-' or \
 235                                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 236                         return filename
 237                 return filename + u'.part'
 238
 239         def undo_temp_name(self, filename):
 240                 if filename.endswith(u'.part'):
 241                         return filename[:-len(u'.part')]
 242                 return filename
 243
 244         def try_rename(self, old_filename, new_filename):
 245                 try:
 246                         if old_filename == new_filename:
 247                                 return
 248                         os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
 249                 except (IOError, OSError) as err:
 250                         self.trouble(u'ERROR: unable to rename file')
 251
 252         def try_utime(self, filename, last_modified_hdr):
 253                 """Try to set the last-modified time of the given file."""
 254                 if last_modified_hdr is None:
 255                         return
 256                 if not os.path.isfile(encodeFilename(filename)):
 257                         return
 258                 timestr = last_modified_hdr
 259                 if timestr is None:
 260                         return
 261                 filetime = timeconvert(timestr)
 262                 if filetime is None:
 263                         return filetime
 264                 try:
 265                         os.utime(filename, (time.time(), filetime))
 266                 except:
 267                         pass
 268                 return filetime
 269
 270         def report_writedescription(self, descfn):
 271                 """ Report that the description file is being written """
 272                 self.to_screen(u'[info] Writing video description to: ' + descfn)
 273
 274         def report_writesubtitles(self, srtfn):
 275                 """ Report that the subtitles file is being written """
 276                 self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
 277
 278         def report_writeinfojson(self, infofn):
 279                 """ Report that the metadata file has been written """
 280                 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
 281
 282         def report_destination(self, filename):
 283                 """Report destination filename."""
 284                 self.to_screen(u'[download] Destination: ' + filename)
 285
 286         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 287                 """Report download progress."""
 288                 if self.params.get('noprogress', False):
 289                         return
 290                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 291                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 292                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
 293                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 294
 295         def report_resuming_byte(self, resume_len):
 296                 """Report attempt to resume at given byte."""
 297                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 298
 299         def report_retry(self, count, retries):
 300                 """Report retry in case of HTTP error 5xx"""
 301                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 302
 303         def report_file_already_downloaded(self, file_name):
 304                 """Report file has already been fully downloaded."""
 305                 try:
 306                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 307                 except (UnicodeEncodeError) as err:
 308                         self.to_screen(u'[download] The file has already been downloaded')
 309
 310         def report_unable_to_resume(self):
 311                 """Report it was impossible to resume download."""
 312                 self.to_screen(u'[download] Unable to resume')
 313
 314         def report_finish(self):
 315                 """Report download finished."""
 316                 if self.params.get('noprogress', False):
 317                         self.to_screen(u'[download] Download completed')
 318                 else:
 319                         self.to_screen(u'')
 320
 321         def increment_downloads(self):
 322                 """Increment the ordinal that assigns a number to each file."""
 323                 self._num_downloads += 1
 324
 325         def prepare_filename(self, info_dict):
 326                 """Generate the output filename."""
 327                 try:
 328                         template_dict = dict(info_dict)
 329
 330                         template_dict['epoch'] = int(time.time())
 331                         template_dict['autonumber'] = u'%05d' % self._num_downloads
 332
 333                         template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items())
 334                         template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
 335
 336                         filename = self.params['outtmpl'] % template_dict
 337                         return filename
 338                 except (ValueError, KeyError) as err:
 339                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 340                         return None
 341
 342         def _match_entry(self, info_dict):
 343                 """ Returns None iff the file should be downloaded """
 344
 345                 title = info_dict['title']
 346                 matchtitle = self.params.get('matchtitle', False)
 347                 if matchtitle:
 348                         matchtitle = matchtitle.decode('utf8')
 349                         if not re.search(matchtitle, title, re.IGNORECASE):
 350                                 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
 351                 rejecttitle = self.params.get('rejecttitle', False)
 352                 if rejecttitle:
 353                         rejecttitle = rejecttitle.decode('utf8')
 354                         if re.search(rejecttitle, title, re.IGNORECASE):
 355                                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 356                 return None
 357
 358         def process_info(self, info_dict):
 359                 """Process a single dictionary returned by an InfoExtractor."""
 360
 361                 # Keep for backwards compatibility
 362                 info_dict['stitle'] = info_dict['title']
 363
 364                 if not 'format' in info_dict:
 365                         info_dict['format'] = info_dict['ext']
 366
 367                 reason = self._match_entry(info_dict)
 368                 if reason is not None:
 369                         self.to_screen(u'[download] ' + reason)
 370                         return
 371
 372                 max_downloads = self.params.get('max_downloads')
 373                 if max_downloads is not None:
 374                         if self._num_downloads > int(max_downloads):
 375                                 raise MaxDownloadsReached()
 376
 377                 filename = self.prepare_filename(info_dict)
 378
 379                 # Forced printings
 380                 if self.params.get('forcetitle', False):
 381                         print(info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace'))
 382                 if self.params.get('forceurl', False):
 383                         print(info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace'))
 384                 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 385                         print(info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace'))
 386                 if self.params.get('forcedescription', False) and 'description' in info_dict:
 387                         print(info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace'))
 388                 if self.params.get('forcefilename', False) and filename is not None:
 389                         print(filename.encode(preferredencoding(), 'xmlcharrefreplace'))
 390                 if self.params.get('forceformat', False):
 391                         print(info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace'))
 392
 393                 # Do nothing else if in simulate mode
 394                 if self.params.get('simulate', False):
 395                         return
 396
 397                 if filename is None:
 398                         return
 399
 400                 try:
 401                         dn = os.path.dirname(encodeFilename(filename))
 402                         if dn != '' and not os.path.exists(dn): # dn is already encoded
 403                                 os.makedirs(dn)
 404                 except (OSError, IOError) as err:
 405                         self.trouble(u'ERROR: unable to create directory ' + compat_str(err))
 406                         return
 407
 408                 if self.params.get('writedescription', False):
 409                         try:
 410                                 descfn = filename + u'.description'
 411                                 self.report_writedescription(descfn)
 412                                 descfile = open(encodeFilename(descfn), 'wb')
 413                                 try:
 414                                         descfile.write(info_dict['description'].encode('utf-8'))
 415                                 finally:
 416                                         descfile.close()
 417                         except (OSError, IOError):
 418                                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
 419                                 return
 420
 421                 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
 422                         # subtitles download errors are already managed as troubles in relevant IE
 423                         # that way it will silently go on when used with unsupporting IE
 424                         try:
 425                                 srtfn = filename.rsplit('.', 1)[0] + u'.srt'
 426                                 self.report_writesubtitles(srtfn)
 427                                 srtfile = open(encodeFilename(srtfn), 'wb')
 428                                 try:
 429                                         srtfile.write(info_dict['subtitles'].encode('utf-8'))
 430                                 finally:
 431                                         srtfile.close()
 432                         except (OSError, IOError):
 433                                 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
 434                                 return
 435
 436                 if self.params.get('writeinfojson', False):
 437                         infofn = filename + u'.info.json'
 438                         self.report_writeinfojson(infofn)
 439                         try:
 440                                 json.dump
 441                         except (NameError,AttributeError):
 442                                 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
 443                                 return
 444                         try:
 445                                 infof = open(encodeFilename(infofn), 'wb')
 446                                 try:
 447                                         json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
 448                                         json.dump(json_info_dict, infof)
 449                                 finally:
 450                                         infof.close()
 451                         except (OSError, IOError):
 452                                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
 453                                 return
 454
 455                 if not self.params.get('skip_download', False):
 456                         if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
 457                                 success = True
 458                         else:
 459                                 try:
 460                                         success = self._do_download(filename, info_dict)
 461                                 except (OSError, IOError) as err:
 462                                         raise UnavailableVideoError
 463                                 except (compat_urllib_error.URLError, httplib.HTTPException, socket.error) as err:
 464                                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 465                                         return
 466                                 except (ContentTooShortError, ) as err:
 467                                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 468                                         return
 469
 470                         if success:
 471                                 try:
 472                                         self.post_process(filename, info_dict)
 473                                 except (PostProcessingError) as err:
 474                                         self.trouble(u'ERROR: postprocessing: %s' % str(err))
 475                                         return
 476
 477         def download(self, url_list):
 478                 """Download a given list of URLs."""
 479                 if len(url_list) > 1 and self.fixed_template():
 480                         raise SameFileError(self.params['outtmpl'])
 481
 482                 for url in url_list:
 483                         suitable_found = False
 484                         for ie in self._ies:
 485                                 # Go to next InfoExtractor if not suitable
 486                                 if not ie.suitable(url):
 487                                         continue
 488
 489                                 # Warn if the _WORKING attribute is False
 490                                 if not ie.working():
 491                                         self.trouble(u'WARNING: the program functionality for this site has been marked as broken, '
 492                                                          u'and will probably not work. If you want to go on, use the -i option.')
 493
 494                                 # Suitable InfoExtractor found
 495                                 suitable_found = True
 496
 497                                 # Extract information from URL and process it
 498                                 videos = ie.extract(url)
 499                                 for video in videos or []:
 500                                         video['extractor'] = ie.IE_NAME
 501                                         try:
 502                                                 self.increment_downloads()
 503                                                 self.process_info(video)
 504                                         except UnavailableVideoError:
 505                                                 self.trouble(u'\nERROR: unable to download video')
 506
 507                                 # Suitable InfoExtractor had been found; go to next URL
 508                                 break
 509
 510                         if not suitable_found:
 511                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 512
 513                 return self._download_retcode
 514
 515         def post_process(self, filename, ie_info):
 516                 """Run the postprocessing chain on the given file."""
 517                 info = dict(ie_info)
 518                 info['filepath'] = filename
 519                 for pp in self._pps:
 520                         info = pp.run(info)
 521                         if info is None:
 522                                 break
 523
 524         def _download_with_rtmpdump(self, filename, url, player_url):
 525                 self.report_destination(filename)
 526                 tmpfilename = self.temp_name(filename)
 527
 528                 # Check for rtmpdump first
 529                 try:
 530                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 531                 except (OSError, IOError):
 532                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 533                         return False
 534
 535                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 536                 # the connection was interrumpted and resuming appears to be
 537                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 538                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 539                 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
 540                 if self.params.get('verbose', False):
 541                         try:
 542                                 import pipes
 543                                 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
 544                         except ImportError:
 545                                 shell_quote = repr
 546                         self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
 547                 retval = subprocess.call(args)
 548                 while retval == 2 or retval == 1:
 549                         prevsize = os.path.getsize(encodeFilename(tmpfilename))
 550                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 551                         time.sleep(5.0) # This seems to be needed
 552                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 553                         cursize = os.path.getsize(encodeFilename(tmpfilename))
 554                         if prevsize == cursize and retval == 1:
 555                                 break
 556                          # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
 557                         if prevsize == cursize and retval == 2 and cursize > 1024:
 558                                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
 559                                 retval = 0
 560                                 break
 561                 if retval == 0:
 562                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename)))
 563                         self.try_rename(tmpfilename, filename)
 564                         return True
 565                 else:
 566                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 567                         return False
 568
 569         def _do_download(self, filename, info_dict):
 570                 url = info_dict['url']
 571                 player_url = info_dict.get('player_url', None)
 572
 573                 # Check file already present
 574                 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
 575                         self.report_file_already_downloaded(filename)
 576                         return True
 577
 578                 # Attempt to download using rtmpdump
 579                 if url.startswith('rtmp'):
 580                         return self._download_with_rtmpdump(filename, url, player_url)
 581
 582                 tmpfilename = self.temp_name(filename)
 583                 stream = None
 584
 585                 # Do not include the Accept-Encoding header
 586                 headers = {'Youtubedl-no-compression': 'True'}
 587                 basic_request = compat_urllib_request.Request(url, None, headers)
 588                 request = compat_urllib_request.Request(url, None, headers)
 589
 590                 # Establish possible resume length
 591                 if os.path.isfile(encodeFilename(tmpfilename)):
 592                         resume_len = os.path.getsize(encodeFilename(tmpfilename))
 593                 else:
 594                         resume_len = 0
 595
 596                 open_mode = 'wb'
 597                 if resume_len != 0:
 598                         if self.params.get('continuedl', False):
 599                                 self.report_resuming_byte(resume_len)
 600                                 request.add_header('Range','bytes=%d-' % resume_len)
 601                                 open_mode = 'ab'
 602                         else:
 603                                 resume_len = 0
 604
 605                 count = 0
 606                 retries = self.params.get('retries', 0)
 607                 while count <= retries:
 608                         # Establish connection
 609                         try:
 610                                 if count == 0 and 'urlhandle' in info_dict:
 611                                         data = info_dict['urlhandle']
 612                                 data = compat_urllib_request.urlopen(request)
 613                                 break
 614                         except (compat_urllib_error.HTTPError, ) as err:
 615                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 616                                         # Unexpected HTTP error
 617                                         raise
 618                                 elif err.code == 416:
 619                                         # Unable to resume (requested range not satisfiable)
 620                                         try:
 621                                                 # Open the connection again without the range header
 622                                                 data = compat_urllib_request.urlopen(basic_request)
 623                                                 content_length = data.info()['Content-Length']
 624                                         except (compat_urllib_error.HTTPError, ) as err:
 625                                                 if err.code < 500 or err.code >= 600:
 626                                                         raise
 627                                         else:
 628                                                 # Examine the reported length
 629                                                 if (content_length is not None and
 630                                                                 (resume_len - 100 < int(content_length) < resume_len + 100)):
 631                                                         # The file had already been fully downloaded.
 632                                                         # Explanation to the above condition: in issue #175 it was revealed that
 633                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 634                                                         # changing the file size slightly and causing problems for some users. So
 635                                                         # I decided to implement a suggested change and consider the file
 636                                                         # completely downloaded if the file size differs less than 100 bytes from
 637                                                         # the one in the hard drive.
 638                                                         self.report_file_already_downloaded(filename)
 639                                                         self.try_rename(tmpfilename, filename)
 640                                                         return True
 641                                                 else:
 642                                                         # The length does not match, we start the download over
 643                                                         self.report_unable_to_resume()
 644                                                         open_mode = 'wb'
 645                                                         break
 646                         # Retry
 647                         count += 1
 648                         if count <= retries:
 649                                 self.report_retry(count, retries)
 650
 651                 if count > retries:
 652                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 653                         return False
 654
 655                 data_len = data.info().get('Content-length', None)
 656                 if data_len is not None:
 657                         data_len = int(data_len) + resume_len
 658                 data_len_str = self.format_bytes(data_len)
 659                 byte_counter = 0 + resume_len
 660                 block_size = self.params.get('buffersize', 1024)
 661                 start = time.time()
 662                 while True:
 663                         # Download and write
 664                         before = time.time()
 665                         data_block = data.read(block_size)
 666                         after = time.time()
 667                         if len(data_block) == 0:
 668                                 break
 669                         byte_counter += len(data_block)
 670
 671                         # Open file just in time
 672                         if stream is None:
 673                                 try:
 674                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 675                                         assert stream is not None
 676                                         filename = self.undo_temp_name(tmpfilename)
 677                                         self.report_destination(filename)
 678                                 except (OSError, IOError) as err:
 679                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 680                                         return False
 681                         try:
 682                                 stream.write(data_block)
 683                         except (IOError, OSError) as err:
 684                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 685                                 return False
 686                         if not self.params.get('noresizebuffer', False):
 687                                 block_size = self.best_block_size(after - before, len(data_block))
 688
 689                         # Progress message
 690                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
 691                         if data_len is None:
 692                                 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
 693                         else:
 694                                 percent_str = self.calc_percent(byte_counter, data_len)
 695                                 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 696                                 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 697
 698                         # Apply rate limit
 699                         self.slow_down(start, byte_counter - resume_len)
 700
 701                 if stream is None:
 702                         self.trouble(u'\nERROR: Did not get any data blocks')
 703                         return False
 704                 stream.close()
 705                 self.report_finish()
 706                 if data_len is not None and byte_counter != data_len:
 707                         raise ContentTooShortError(byte_counter, int(data_len))
 708                 self.try_rename(tmpfilename, filename)
 709
 710                 # Update file modification time
 711                 if self.params.get('updatetime', True):
 712                         info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
 713
 714                 return True