_ Git - youtube-dl/blob - youtube_dl/FileDownloader.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import math
   5 import os
   6 import re
   7 import socket
   8 import subprocess
   9 import sys
  10 import time
  11
  12 if os.name == 'nt':
  13         import ctypes
  14
  15 from utils import *
  16
  17
  18 class FileDownloader(object):
  19         """File Downloader class.
  20
  21         File downloader objects are the ones responsible of downloading the
  22         actual video file and writing it to disk if the user has requested
  23         it, among some other tasks. In most cases there should be one per
  24         program. As, given a video URL, the downloader doesn't know how to
  25         extract all the needed information, task that InfoExtractors do, it
  26         has to pass the URL to one of them.
  27
  28         For this, file downloader objects have a method that allows
  29         InfoExtractors to be registered in a given order. When it is passed
  30         a URL, the file downloader handles it to the first InfoExtractor it
  31         finds that reports being able to handle it. The InfoExtractor extracts
  32         all the information about the video or videos the URL refers to, and
  33         asks the FileDownloader to process the video information, possibly
  34         downloading the video.
  35
  36         File downloaders accept a lot of parameters. In order not to saturate
  37         the object constructor with arguments, it receives a dictionary of
  38         options instead. These options are available through the params
  39         attribute for the InfoExtractors to use. The FileDownloader also
  40         registers itself as the downloader in charge for the InfoExtractors
  41         that are added to it, so this is a "mutual registration".
  42
  43         Available options:
  44
  45         username:          Username for authentication purposes.
  46         password:          Password for authentication purposes.
  47         usenetrc:          Use netrc for authentication instead.
  48         quiet:             Do not print messages to stdout.
  49         forceurl:          Force printing final URL.
  50         forcetitle:        Force printing title.
  51         forcethumbnail:    Force printing thumbnail URL.
  52         forcedescription:  Force printing description.
  53         forcefilename:     Force printing final filename.
  54         simulate:          Do not download the video files.
  55         format:            Video format code.
  56         format_limit:      Highest quality format to try.
  57         outtmpl:           Template for output names.
  58         restrictfilenames: Do not allow "&" and spaces in file names
  59         ignoreerrors:      Do not stop on download errors.
  60         ratelimit:         Download speed limit, in bytes/sec.
  61         nooverwrites:      Prevent overwriting files.
  62         retries:           Number of times to retry for HTTP error 5xx
  63         buffersize:        Size of download buffer in bytes.
  64         noresizebuffer:    Do not automatically resize the download buffer.
  65         continuedl:        Try to continue downloads if possible.
  66         noprogress:        Do not print the progress bar.
  67         playliststart:     Playlist item to start at.
  68         playlistend:       Playlist item to end at.
  69         matchtitle:        Download only matching titles.
  70         rejecttitle:       Reject downloads for matching titles.
  71         logtostderr:       Log messages to stderr instead of stdout.
  72         consoletitle:      Display progress in console window's titlebar.
  73         nopart:            Do not use temporary .part files.
  74         updatetime:        Use the Last-modified header to set output file timestamps.
  75         writedescription:  Write the video description to a .description file
  76         writeinfojson:     Write the video description to a .info.json file
  77         writesubtitles:    Write the video subtitles to a .srt file
  78         subtitleslang:     Language of the subtitles to download
  79         """
  80
  81         params = None
  82         _ies = []
  83         _pps = []
  84         _download_retcode = None
  85         _num_downloads = None
  86         _screen_file = None
  87
  88         def __init__(self, params):
  89                 """Create a FileDownloader object with the given options."""
  90                 self._ies = []
  91                 self._pps = []
  92                 self._download_retcode = 0
  93                 self._num_downloads = 0
  94                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
  95                 self.params = params
  96
  97                 if '%(stitle)s' in self.params['outtmpl']:
  98                         self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
  99
 100         @staticmethod
 101         def format_bytes(bytes):
 102                 if bytes is None:
 103                         return 'N/A'
 104                 if type(bytes) is str:
 105                         bytes = float(bytes)
 106                 if bytes == 0.0:
 107                         exponent = 0
 108                 else:
 109                         exponent = int(math.log(bytes, 1024.0))
 110                 suffix = 'bkMGTPEZY'[exponent]
 111                 converted = float(bytes) / float(1024 ** exponent)
 112                 return '%.2f%s' % (converted, suffix)
 113
 114         @staticmethod
 115         def calc_percent(byte_counter, data_len):
 116                 if data_len is None:
 117                         return '---.-%'
 118                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 119
 120         @staticmethod
 121         def calc_eta(start, now, total, current):
 122                 if total is None:
 123                         return '--:--'
 124                 dif = now - start
 125                 if current == 0 or dif < 0.001: # One millisecond
 126                         return '--:--'
 127                 rate = float(current) / dif
 128                 eta = int((float(total) - float(current)) / rate)
 129                 (eta_mins, eta_secs) = divmod(eta, 60)
 130                 if eta_mins > 99:
 131                         return '--:--'
 132                 return '%02d:%02d' % (eta_mins, eta_secs)
 133
 134         @staticmethod
 135         def calc_speed(start, now, bytes):
 136                 dif = now - start
 137                 if bytes == 0 or dif < 0.001: # One millisecond
 138                         return '%10s' % '---b/s'
 139                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 140
 141         @staticmethod
 142         def best_block_size(elapsed_time, bytes):
 143                 new_min = max(bytes / 2.0, 1.0)
 144                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 145                 if elapsed_time < 0.001:
 146                         return int(new_max)
 147                 rate = bytes / elapsed_time
 148                 if rate > new_max:
 149                         return int(new_max)
 150                 if rate < new_min:
 151                         return int(new_min)
 152                 return int(rate)
 153
 154         @staticmethod
 155         def parse_bytes(bytestr):
 156                 """Parse a string indicating a byte quantity into an integer."""
 157                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 158                 if matchobj is None:
 159                         return None
 160                 number = float(matchobj.group(1))
 161                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 162                 return int(round(number * multiplier))
 163
 164         def add_info_extractor(self, ie):
 165                 """Add an InfoExtractor object to the end of the list."""
 166                 self._ies.append(ie)
 167                 ie.set_downloader(self)
 168
 169         def add_post_processor(self, pp):
 170                 """Add a PostProcessor object to the end of the chain."""
 171                 self._pps.append(pp)
 172                 pp.set_downloader(self)
 173
 174         def to_screen(self, message, skip_eol=False):
 175                 """Print message to stdout if not in quiet mode."""
 176                 assert type(message) == type(u'')
 177                 if not self.params.get('quiet', False):
 178                         terminator = [u'\n', u''][skip_eol]
 179                         output = message + terminator
 180                         if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
 181                                 output = output.encode(preferredencoding(), 'ignore')
 182                         self._screen_file.write(output)
 183                         self._screen_file.flush()
 184
 185         def to_stderr(self, message):
 186                 """Print message to stderr."""
 187                 assert type(message) == type(u'')
 188                 sys.stderr.write((message + u'\n').encode(preferredencoding()))
 189
 190         def to_cons_title(self, message):
 191                 """Set console/terminal window title to message."""
 192                 if not self.params.get('consoletitle', False):
 193                         return
 194                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 195                         # c_wchar_p() might not be necessary if `message` is
 196                         # already of type unicode()
 197                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 198                 elif 'TERM' in os.environ:
 199                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
 200
 201         def fixed_template(self):
 202                 """Checks if the output template is fixed."""
 203                 return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
 204
 205         def trouble(self, message=None):
 206                 """Determine action to take when a download problem appears.
 207
 208                 Depending on if the downloader has been configured to ignore
 209                 download errors or not, this method may throw an exception or
 210                 not when errors are found, after printing the message.
 211                 """
 212                 if message is not None:
 213                         self.to_stderr(message)
 214                 if not self.params.get('ignoreerrors', False):
 215                         raise DownloadError(message)
 216                 self._download_retcode = 1
 217
 218         def slow_down(self, start_time, byte_counter):
 219                 """Sleep if the download speed is over the rate limit."""
 220                 rate_limit = self.params.get('ratelimit', None)
 221                 if rate_limit is None or byte_counter == 0:
 222                         return
 223                 now = time.time()
 224                 elapsed = now - start_time
 225                 if elapsed <= 0.0:
 226                         return
 227                 speed = float(byte_counter) / elapsed
 228                 if speed > rate_limit:
 229                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 230
 231         def temp_name(self, filename):
 232                 """Returns a temporary filename for the given filename."""
 233                 if self.params.get('nopart', False) or filename == u'-' or \
 234                                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 235                         return filename
 236                 return filename + u'.part'
 237
 238         def undo_temp_name(self, filename):
 239                 if filename.endswith(u'.part'):
 240                         return filename[:-len(u'.part')]
 241                 return filename
 242
 243         def try_rename(self, old_filename, new_filename):
 244                 try:
 245                         if old_filename == new_filename:
 246                                 return
 247                         os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
 248                 except (IOError, OSError) as err:
 249                         self.trouble(u'ERROR: unable to rename file')
 250
 251         def try_utime(self, filename, last_modified_hdr):
 252                 """Try to set the last-modified time of the given file."""
 253                 if last_modified_hdr is None:
 254                         return
 255                 if not os.path.isfile(encodeFilename(filename)):
 256                         return
 257                 timestr = last_modified_hdr
 258                 if timestr is None:
 259                         return
 260                 filetime = timeconvert(timestr)
 261                 if filetime is None:
 262                         return filetime
 263                 try:
 264                         os.utime(filename, (time.time(), filetime))
 265                 except:
 266                         pass
 267                 return filetime
 268
 269         def report_writedescription(self, descfn):
 270                 """ Report that the description file is being written """
 271                 self.to_screen(u'[info] Writing video description to: ' + descfn)
 272
 273         def report_writesubtitles(self, srtfn):
 274                 """ Report that the subtitles file is being written """
 275                 self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
 276
 277         def report_writeinfojson(self, infofn):
 278                 """ Report that the metadata file has been written """
 279                 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
 280
 281         def report_destination(self, filename):
 282                 """Report destination filename."""
 283                 self.to_screen(u'[download] Destination: ' + filename)
 284
 285         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 286                 """Report download progress."""
 287                 if self.params.get('noprogress', False):
 288                         return
 289                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 290                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 291                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
 292                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 293
 294         def report_resuming_byte(self, resume_len):
 295                 """Report attempt to resume at given byte."""
 296                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 297
 298         def report_retry(self, count, retries):
 299                 """Report retry in case of HTTP error 5xx"""
 300                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 301
 302         def report_file_already_downloaded(self, file_name):
 303                 """Report file has already been fully downloaded."""
 304                 try:
 305                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 306                 except (UnicodeEncodeError) as err:
 307                         self.to_screen(u'[download] The file has already been downloaded')
 308
 309         def report_unable_to_resume(self):
 310                 """Report it was impossible to resume download."""
 311                 self.to_screen(u'[download] Unable to resume')
 312
 313         def report_finish(self):
 314                 """Report download finished."""
 315                 if self.params.get('noprogress', False):
 316                         self.to_screen(u'[download] Download completed')
 317                 else:
 318                         self.to_screen(u'')
 319
 320         def increment_downloads(self):
 321                 """Increment the ordinal that assigns a number to each file."""
 322                 self._num_downloads += 1
 323
 324         def prepare_filename(self, info_dict):
 325                 """Generate the output filename."""
 326                 try:
 327                         template_dict = dict(info_dict)
 328
 329                         template_dict['epoch'] = int(time.time())
 330                         template_dict['autonumber'] = u'%05d' % self._num_downloads
 331
 332                         template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items())
 333                         template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
 334
 335                         filename = self.params['outtmpl'] % template_dict
 336                         return filename
 337                 except (ValueError, KeyError) as err:
 338                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 339                         return None
 340
 341         def _match_entry(self, info_dict):
 342                 """ Returns None iff the file should be downloaded """
 343
 344                 title = info_dict['title']
 345                 matchtitle = self.params.get('matchtitle', False)
 346                 if matchtitle:
 347                         matchtitle = matchtitle.decode('utf8')
 348                         if not re.search(matchtitle, title, re.IGNORECASE):
 349                                 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
 350                 rejecttitle = self.params.get('rejecttitle', False)
 351                 if rejecttitle:
 352                         rejecttitle = rejecttitle.decode('utf8')
 353                         if re.search(rejecttitle, title, re.IGNORECASE):
 354                                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 355                 return None
 356
 357         def process_info(self, info_dict):
 358                 """Process a single dictionary returned by an InfoExtractor."""
 359
 360                 # Keep for backwards compatibility
 361                 info_dict['stitle'] = info_dict['title']
 362
 363                 if not 'format' in info_dict:
 364                         info_dict['format'] = info_dict['ext']
 365
 366                 reason = self._match_entry(info_dict)
 367                 if reason is not None:
 368                         self.to_screen(u'[download] ' + reason)
 369                         return
 370
 371                 max_downloads = self.params.get('max_downloads')
 372                 if max_downloads is not None:
 373                         if self._num_downloads > int(max_downloads):
 374                                 raise MaxDownloadsReached()
 375
 376                 filename = self.prepare_filename(info_dict)
 377
 378                 # Forced printings
 379                 if self.params.get('forcetitle', False):
 380                         print(info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace'))
 381                 if self.params.get('forceurl', False):
 382                         print(info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace'))
 383                 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 384                         print(info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace'))
 385                 if self.params.get('forcedescription', False) and 'description' in info_dict:
 386                         print(info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace'))
 387                 if self.params.get('forcefilename', False) and filename is not None:
 388                         print(filename.encode(preferredencoding(), 'xmlcharrefreplace'))
 389                 if self.params.get('forceformat', False):
 390                         print(info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace'))
 391
 392                 # Do nothing else if in simulate mode
 393                 if self.params.get('simulate', False):
 394                         return
 395
 396                 if filename is None:
 397                         return
 398
 399                 try:
 400                         dn = os.path.dirname(encodeFilename(filename))
 401                         if dn != '' and not os.path.exists(dn): # dn is already encoded
 402                                 os.makedirs(dn)
 403                 except (OSError, IOError) as err:
 404                         self.trouble(u'ERROR: unable to create directory ' + compat_str(err))
 405                         return
 406
 407                 if self.params.get('writedescription', False):
 408                         try:
 409                                 descfn = filename + u'.description'
 410                                 self.report_writedescription(descfn)
 411                                 descfile = open(encodeFilename(descfn), 'wb')
 412                                 try:
 413                                         descfile.write(info_dict['description'].encode('utf-8'))
 414                                 finally:
 415                                         descfile.close()
 416                         except (OSError, IOError):
 417                                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
 418                                 return
 419
 420                 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
 421                         # subtitles download errors are already managed as troubles in relevant IE
 422                         # that way it will silently go on when used with unsupporting IE
 423                         try:
 424                                 srtfn = filename.rsplit('.', 1)[0] + u'.srt'
 425                                 self.report_writesubtitles(srtfn)
 426                                 srtfile = open(encodeFilename(srtfn), 'wb')
 427                                 try:
 428                                         srtfile.write(info_dict['subtitles'].encode('utf-8'))
 429                                 finally:
 430                                         srtfile.close()
 431                         except (OSError, IOError):
 432                                 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
 433                                 return
 434
 435                 if self.params.get('writeinfojson', False):
 436                         infofn = filename + u'.info.json'
 437                         self.report_writeinfojson(infofn)
 438                         try:
 439                                 json.dump
 440                         except (NameError,AttributeError):
 441                                 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
 442                                 return
 443                         try:
 444                                 infof = open(encodeFilename(infofn), 'wb')
 445                                 try:
 446                                         json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
 447                                         json.dump(json_info_dict, infof)
 448                                 finally:
 449                                         infof.close()
 450                         except (OSError, IOError):
 451                                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
 452                                 return
 453
 454                 if not self.params.get('skip_download', False):
 455                         if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
 456                                 success = True
 457                         else:
 458                                 try:
 459                                         success = self._do_download(filename, info_dict)
 460                                 except (OSError, IOError) as err:
 461                                         raise UnavailableVideoError
 462                                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 463                                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 464                                         return
 465                                 except (ContentTooShortError, ) as err:
 466                                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 467                                         return
 468
 469                         if success:
 470                                 try:
 471                                         self.post_process(filename, info_dict)
 472                                 except (PostProcessingError) as err:
 473                                         self.trouble(u'ERROR: postprocessing: %s' % str(err))
 474                                         return
 475
 476         def download(self, url_list):
 477                 """Download a given list of URLs."""
 478                 if len(url_list) > 1 and self.fixed_template():
 479                         raise SameFileError(self.params['outtmpl'])
 480
 481                 for url in url_list:
 482                         suitable_found = False
 483                         for ie in self._ies:
 484                                 # Go to next InfoExtractor if not suitable
 485                                 if not ie.suitable(url):
 486                                         continue
 487
 488                                 # Warn if the _WORKING attribute is False
 489                                 if not ie.working():
 490                                         self.trouble(u'WARNING: the program functionality for this site has been marked as broken, '
 491                                                          u'and will probably not work. If you want to go on, use the -i option.')
 492
 493                                 # Suitable InfoExtractor found
 494                                 suitable_found = True
 495
 496                                 # Extract information from URL and process it
 497                                 videos = ie.extract(url)
 498                                 for video in videos or []:
 499                                         video['extractor'] = ie.IE_NAME
 500                                         try:
 501                                                 self.increment_downloads()
 502                                                 self.process_info(video)
 503                                         except UnavailableVideoError:
 504                                                 self.trouble(u'\nERROR: unable to download video')
 505
 506                                 # Suitable InfoExtractor had been found; go to next URL
 507                                 break
 508
 509                         if not suitable_found:
 510                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 511
 512                 return self._download_retcode
 513
 514         def post_process(self, filename, ie_info):
 515                 """Run the postprocessing chain on the given file."""
 516                 info = dict(ie_info)
 517                 info['filepath'] = filename
 518                 for pp in self._pps:
 519                         info = pp.run(info)
 520                         if info is None:
 521                                 break
 522
 523         def _download_with_rtmpdump(self, filename, url, player_url):
 524                 self.report_destination(filename)
 525                 tmpfilename = self.temp_name(filename)
 526
 527                 # Check for rtmpdump first
 528                 try:
 529                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 530                 except (OSError, IOError):
 531                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 532                         return False
 533
 534                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 535                 # the connection was interrumpted and resuming appears to be
 536                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 537                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 538                 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
 539                 if self.params.get('verbose', False):
 540                         try:
 541                                 import pipes
 542                                 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
 543                         except ImportError:
 544                                 shell_quote = repr
 545                         self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
 546                 retval = subprocess.call(args)
 547                 while retval == 2 or retval == 1:
 548                         prevsize = os.path.getsize(encodeFilename(tmpfilename))
 549                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 550                         time.sleep(5.0) # This seems to be needed
 551                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 552                         cursize = os.path.getsize(encodeFilename(tmpfilename))
 553                         if prevsize == cursize and retval == 1:
 554                                 break
 555                          # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
 556                         if prevsize == cursize and retval == 2 and cursize > 1024:
 557                                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
 558                                 retval = 0
 559                                 break
 560                 if retval == 0:
 561                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename)))
 562                         self.try_rename(tmpfilename, filename)
 563                         return True
 564                 else:
 565                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 566                         return False
 567
 568         def _do_download(self, filename, info_dict):
 569                 url = info_dict['url']
 570                 player_url = info_dict.get('player_url', None)
 571
 572                 # Check file already present
 573                 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
 574                         self.report_file_already_downloaded(filename)
 575                         return True
 576
 577                 # Attempt to download using rtmpdump
 578                 if url.startswith('rtmp'):
 579                         return self._download_with_rtmpdump(filename, url, player_url)
 580
 581                 tmpfilename = self.temp_name(filename)
 582                 stream = None
 583
 584                 # Do not include the Accept-Encoding header
 585                 headers = {'Youtubedl-no-compression': 'True'}
 586                 basic_request = compat_urllib_request.Request(url, None, headers)
 587                 request = compat_urllib_request.Request(url, None, headers)
 588
 589                 # Establish possible resume length
 590                 if os.path.isfile(encodeFilename(tmpfilename)):
 591                         resume_len = os.path.getsize(encodeFilename(tmpfilename))
 592                 else:
 593                         resume_len = 0
 594
 595                 open_mode = 'wb'
 596                 if resume_len != 0:
 597                         if self.params.get('continuedl', False):
 598                                 self.report_resuming_byte(resume_len)
 599                                 request.add_header('Range','bytes=%d-' % resume_len)
 600                                 open_mode = 'ab'
 601                         else:
 602                                 resume_len = 0
 603
 604                 count = 0
 605                 retries = self.params.get('retries', 0)
 606                 while count <= retries:
 607                         # Establish connection
 608                         try:
 609                                 if count == 0 and 'urlhandle' in info_dict:
 610                                         data = info_dict['urlhandle']
 611                                 data = compat_urllib_request.urlopen(request)
 612                                 break
 613                         except (compat_urllib_error.HTTPError, ) as err:
 614                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 615                                         # Unexpected HTTP error
 616                                         raise
 617                                 elif err.code == 416:
 618                                         # Unable to resume (requested range not satisfiable)
 619                                         try:
 620                                                 # Open the connection again without the range header
 621                                                 data = compat_urllib_request.urlopen(basic_request)
 622                                                 content_length = data.info()['Content-Length']
 623                                         except (compat_urllib_error.HTTPError, ) as err:
 624                                                 if err.code < 500 or err.code >= 600:
 625                                                         raise
 626                                         else:
 627                                                 # Examine the reported length
 628                                                 if (content_length is not None and
 629                                                                 (resume_len - 100 < int(content_length) < resume_len + 100)):
 630                                                         # The file had already been fully downloaded.
 631                                                         # Explanation to the above condition: in issue #175 it was revealed that
 632                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 633                                                         # changing the file size slightly and causing problems for some users. So
 634                                                         # I decided to implement a suggested change and consider the file
 635                                                         # completely downloaded if the file size differs less than 100 bytes from
 636                                                         # the one in the hard drive.
 637                                                         self.report_file_already_downloaded(filename)
 638                                                         self.try_rename(tmpfilename, filename)
 639                                                         return True
 640                                                 else:
 641                                                         # The length does not match, we start the download over
 642                                                         self.report_unable_to_resume()
 643                                                         open_mode = 'wb'
 644                                                         break
 645                         # Retry
 646                         count += 1
 647                         if count <= retries:
 648                                 self.report_retry(count, retries)
 649
 650                 if count > retries:
 651                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 652                         return False
 653
 654                 data_len = data.info().get('Content-length', None)
 655                 if data_len is not None:
 656                         data_len = int(data_len) + resume_len
 657                 data_len_str = self.format_bytes(data_len)
 658                 byte_counter = 0 + resume_len
 659                 block_size = self.params.get('buffersize', 1024)
 660                 start = time.time()
 661                 while True:
 662                         # Download and write
 663                         before = time.time()
 664                         data_block = data.read(block_size)
 665                         after = time.time()
 666                         if len(data_block) == 0:
 667                                 break
 668                         byte_counter += len(data_block)
 669
 670                         # Open file just in time
 671                         if stream is None:
 672                                 try:
 673                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 674                                         assert stream is not None
 675                                         filename = self.undo_temp_name(tmpfilename)
 676                                         self.report_destination(filename)
 677                                 except (OSError, IOError) as err:
 678                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 679                                         return False
 680                         try:
 681                                 stream.write(data_block)
 682                         except (IOError, OSError) as err:
 683                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 684                                 return False
 685                         if not self.params.get('noresizebuffer', False):
 686                                 block_size = self.best_block_size(after - before, len(data_block))
 687
 688                         # Progress message
 689                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
 690                         if data_len is None:
 691                                 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
 692                         else:
 693                                 percent_str = self.calc_percent(byte_counter, data_len)
 694                                 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 695                                 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 696
 697                         # Apply rate limit
 698                         self.slow_down(start, byte_counter - resume_len)
 699
 700                 if stream is None:
 701                         self.trouble(u'\nERROR: Did not get any data blocks')
 702                         return False
 703                 stream.close()
 704                 self.report_finish()
 705                 if data_len is not None and byte_counter != data_len:
 706                         raise ContentTooShortError(byte_counter, int(data_len))
 707                 self.try_rename(tmpfilename, filename)
 708
 709                 # Update file modification time
 710                 if self.params.get('updatetime', True):
 711                         info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
 712
 713                 return True