_ Git - youtube-dl/blob - youtube_dl/FileDownloader.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import httplib
   5 import math
   6 import os
   7 import re
   8 import socket
   9 import subprocess
  10 import sys
  11 import time
  12 import urllib2
  13
  14 if os.name == 'nt':
  15         import ctypes
  16
  17 from utils import *
  18
  19
  20 class FileDownloader(object):
  21         """File Downloader class.
  22
  23         File downloader objects are the ones responsible of downloading the
  24         actual video file and writing it to disk if the user has requested
  25         it, among some other tasks. In most cases there should be one per
  26         program. As, given a video URL, the downloader doesn't know how to
  27         extract all the needed information, task that InfoExtractors do, it
  28         has to pass the URL to one of them.
  29
  30         For this, file downloader objects have a method that allows
  31         InfoExtractors to be registered in a given order. When it is passed
  32         a URL, the file downloader handles it to the first InfoExtractor it
  33         finds that reports being able to handle it. The InfoExtractor extracts
  34         all the information about the video or videos the URL refers to, and
  35         asks the FileDownloader to process the video information, possibly
  36         downloading the video.
  37
  38         File downloaders accept a lot of parameters. In order not to saturate
  39         the object constructor with arguments, it receives a dictionary of
  40         options instead. These options are available through the params
  41         attribute for the InfoExtractors to use. The FileDownloader also
  42         registers itself as the downloader in charge for the InfoExtractors
  43         that are added to it, so this is a "mutual registration".
  44
  45         Available options:
  46
  47         username:          Username for authentication purposes.
  48         password:          Password for authentication purposes.
  49         usenetrc:          Use netrc for authentication instead.
  50         quiet:             Do not print messages to stdout.
  51         forceurl:          Force printing final URL.
  52         forcetitle:        Force printing title.
  53         forcethumbnail:    Force printing thumbnail URL.
  54         forcedescription:  Force printing description.
  55         forcefilename:     Force printing final filename.
  56         simulate:          Do not download the video files.
  57         format:            Video format code.
  58         format_limit:      Highest quality format to try.
  59         outtmpl:           Template for output names.
  60         restrictfilenames: Do not allow "&" and spaces in file names
  61         ignoreerrors:      Do not stop on download errors.
  62         ratelimit:         Download speed limit, in bytes/sec.
  63         nooverwrites:      Prevent overwriting files.
  64         retries:           Number of times to retry for HTTP error 5xx
  65         continuedl:        Try to continue downloads if possible.
  66         noprogress:        Do not print the progress bar.
  67         playliststart:     Playlist item to start at.
  68         playlistend:       Playlist item to end at.
  69         matchtitle:        Download only matching titles.
  70         rejecttitle:       Reject downloads for matching titles.
  71         logtostderr:       Log messages to stderr instead of stdout.
  72         consoletitle:      Display progress in console window's titlebar.
  73         nopart:            Do not use temporary .part files.
  74         updatetime:        Use the Last-modified header to set output file timestamps.
  75         writedescription:  Write the video description to a .description file
  76         writeinfojson:     Write the video description to a .info.json file
  77         writesubtitles:    Write the video subtitles to a .srt file
  78         subtitleslang:     Language of the subtitles to download
  79         """
  80
  81         params = None
  82         _ies = []
  83         _pps = []
  84         _download_retcode = None
  85         _num_downloads = None
  86         _screen_file = None
  87
  88         def __init__(self, params):
  89                 """Create a FileDownloader object with the given options."""
  90                 self._ies = []
  91                 self._pps = []
  92                 self._download_retcode = 0
  93                 self._num_downloads = 0
  94                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
  95                 self.params = params
  96
  97                 if '%(stitle)s' in self.params['outtmpl']:
  98                         self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
  99
 100         @staticmethod
 101         def format_bytes(bytes):
 102                 if bytes is None:
 103                         return 'N/A'
 104                 if type(bytes) is str:
 105                         bytes = float(bytes)
 106                 if bytes == 0.0:
 107                         exponent = 0
 108                 else:
 109                         exponent = long(math.log(bytes, 1024.0))
 110                 suffix = 'bkMGTPEZY'[exponent]
 111                 converted = float(bytes) / float(1024 ** exponent)
 112                 return '%.2f%s' % (converted, suffix)
 113
 114         @staticmethod
 115         def calc_percent(byte_counter, data_len):
 116                 if data_len is None:
 117                         return '---.-%'
 118                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 119
 120         @staticmethod
 121         def calc_eta(start, now, total, current):
 122                 if total is None:
 123                         return '--:--'
 124                 dif = now - start
 125                 if current == 0 or dif < 0.001: # One millisecond
 126                         return '--:--'
 127                 rate = float(current) / dif
 128                 eta = long((float(total) - float(current)) / rate)
 129                 (eta_mins, eta_secs) = divmod(eta, 60)
 130                 if eta_mins > 99:
 131                         return '--:--'
 132                 return '%02d:%02d' % (eta_mins, eta_secs)
 133
 134         @staticmethod
 135         def calc_speed(start, now, bytes):
 136                 dif = now - start
 137                 if bytes == 0 or dif < 0.001: # One millisecond
 138                         return '%10s' % '---b/s'
 139                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 140
 141         @staticmethod
 142         def best_block_size(elapsed_time, bytes):
 143                 new_min = max(bytes / 2.0, 1.0)
 144                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 145                 if elapsed_time < 0.001:
 146                         return int(new_max)
 147                 rate = bytes / elapsed_time
 148                 if rate > new_max:
 149                         return int(new_max)
 150                 if rate < new_min:
 151                         return int(new_min)
 152                 return int(rate)
 153
 154         @staticmethod
 155         def parse_bytes(bytestr):
 156                 """Parse a string indicating a byte quantity into an integer."""
 157                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 158                 if matchobj is None:
 159                         return None
 160                 number = float(matchobj.group(1))
 161                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 162                 return int(round(number * multiplier))
 163
 164         def add_info_extractor(self, ie):
 165                 """Add an InfoExtractor object to the end of the list."""
 166                 self._ies.append(ie)
 167                 ie.set_downloader(self)
 168
 169         def add_post_processor(self, pp):
 170                 """Add a PostProcessor object to the end of the chain."""
 171                 self._pps.append(pp)
 172                 pp.set_downloader(self)
 173
 174         def to_screen(self, message, skip_eol=False):
 175                 """Print message to stdout if not in quiet mode."""
 176                 assert type(message) == type(u'')
 177                 if not self.params.get('quiet', False):
 178                         terminator = [u'\n', u''][skip_eol]
 179                         output = message + terminator
 180                         if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
 181                                 output = output.encode(preferredencoding(), 'ignore')
 182                         self._screen_file.write(output)
 183                         self._screen_file.flush()
 184
 185         def to_stderr(self, message):
 186                 """Print message to stderr."""
 187                 assert type(message) == type(u'')
 188                 sys.stderr.write((message + u'\n').encode(preferredencoding()))
 189
 190         def to_cons_title(self, message):
 191                 """Set console/terminal window title to message."""
 192                 if not self.params.get('consoletitle', False):
 193                         return
 194                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 195                         # c_wchar_p() might not be necessary if `message` is
 196                         # already of type unicode()
 197                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 198                 elif 'TERM' in os.environ:
 199                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
 200
 201         def fixed_template(self):
 202                 """Checks if the output template is fixed."""
 203                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 204
 205         def trouble(self, message=None):
 206                 """Determine action to take when a download problem appears.
 207
 208                 Depending on if the downloader has been configured to ignore
 209                 download errors or not, this method may throw an exception or
 210                 not when errors are found, after printing the message.
 211                 """
 212                 if message is not None:
 213                         self.to_stderr(message)
 214                 if not self.params.get('ignoreerrors', False):
 215                         raise DownloadError(message)
 216                 self._download_retcode = 1
 217
 218         def slow_down(self, start_time, byte_counter):
 219                 """Sleep if the download speed is over the rate limit."""
 220                 rate_limit = self.params.get('ratelimit', None)
 221                 if rate_limit is None or byte_counter == 0:
 222                         return
 223                 now = time.time()
 224                 elapsed = now - start_time
 225                 if elapsed <= 0.0:
 226                         return
 227                 speed = float(byte_counter) / elapsed
 228                 if speed > rate_limit:
 229                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 230
 231         def temp_name(self, filename):
 232                 """Returns a temporary filename for the given filename."""
 233                 if self.params.get('nopart', False) or filename == u'-' or \
 234                                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 235                         return filename
 236                 return filename + u'.part'
 237
 238         def undo_temp_name(self, filename):
 239                 if filename.endswith(u'.part'):
 240                         return filename[:-len(u'.part')]
 241                 return filename
 242
 243         def try_rename(self, old_filename, new_filename):
 244                 try:
 245                         if old_filename == new_filename:
 246                                 return
 247                         os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
 248                 except (IOError, OSError), err:
 249                         self.trouble(u'ERROR: unable to rename file')
 250
 251         def try_utime(self, filename, last_modified_hdr):
 252                 """Try to set the last-modified time of the given file."""
 253                 if last_modified_hdr is None:
 254                         return
 255                 if not os.path.isfile(encodeFilename(filename)):
 256                         return
 257                 timestr = last_modified_hdr
 258                 if timestr is None:
 259                         return
 260                 filetime = timeconvert(timestr)
 261                 if filetime is None:
 262                         return filetime
 263                 try:
 264                         os.utime(filename, (time.time(), filetime))
 265                 except:
 266                         pass
 267                 return filetime
 268
 269         def report_writedescription(self, descfn):
 270                 """ Report that the description file is being written """
 271                 self.to_screen(u'[info] Writing video description to: ' + descfn)
 272
 273         def report_writesubtitles(self, srtfn):
 274                 """ Report that the subtitles file is being written """
 275                 self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
 276
 277         def report_writeinfojson(self, infofn):
 278                 """ Report that the metadata file has been written """
 279                 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
 280
 281         def report_destination(self, filename):
 282                 """Report destination filename."""
 283                 self.to_screen(u'[download] Destination: ' + filename)
 284
 285         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 286                 """Report download progress."""
 287                 if self.params.get('noprogress', False):
 288                         return
 289                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 290                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 291                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
 292                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 293
 294         def report_resuming_byte(self, resume_len):
 295                 """Report attempt to resume at given byte."""
 296                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 297
 298         def report_retry(self, count, retries):
 299                 """Report retry in case of HTTP error 5xx"""
 300                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 301
 302         def report_file_already_downloaded(self, file_name):
 303                 """Report file has already been fully downloaded."""
 304                 try:
 305                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 306                 except (UnicodeEncodeError), err:
 307                         self.to_screen(u'[download] The file has already been downloaded')
 308
 309         def report_unable_to_resume(self):
 310                 """Report it was impossible to resume download."""
 311                 self.to_screen(u'[download] Unable to resume')
 312
 313         def report_finish(self):
 314                 """Report download finished."""
 315                 if self.params.get('noprogress', False):
 316                         self.to_screen(u'[download] Download completed')
 317                 else:
 318                         self.to_screen(u'')
 319
 320         def increment_downloads(self):
 321                 """Increment the ordinal that assigns a number to each file."""
 322                 self._num_downloads += 1
 323
 324         def prepare_filename(self, info_dict):
 325                 """Generate the output filename."""
 326                 try:
 327                         template_dict = dict(info_dict)
 328                         template_dict['epoch'] = unicode(int(time.time()))
 329                         template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
 330                         template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items())
 331                         filename = self.params['outtmpl'] % template_dict
 332                         return filename
 333                 except (ValueError, KeyError), err:
 334                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 335                         return None
 336
 337         def _match_entry(self, info_dict):
 338                 """ Returns None iff the file should be downloaded """
 339
 340                 title = info_dict['title']
 341                 matchtitle = self.params.get('matchtitle', False)
 342                 if matchtitle:
 343                         matchtitle = matchtitle.decode('utf8')
 344                         if not re.search(matchtitle, title, re.IGNORECASE):
 345                                 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
 346                 rejecttitle = self.params.get('rejecttitle', False)
 347                 if rejecttitle:
 348                         rejecttitle = rejecttitle.decode('utf8')
 349                         if re.search(rejecttitle, title, re.IGNORECASE):
 350                                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 351                 return None
 352
 353         def process_info(self, info_dict):
 354                 """Process a single dictionary returned by an InfoExtractor."""
 355
 356                 # Keep for backwards compatibility
 357                 info_dict['stitle'] = info_dict['title']
 358
 359                 if not 'format' in info_dict:
 360                         info_dict['format'] = info_dict['ext']
 361
 362                 reason = self._match_entry(info_dict)
 363                 if reason is not None:
 364                         self.to_screen(u'[download] ' + reason)
 365                         return
 366
 367                 max_downloads = self.params.get('max_downloads')
 368                 if max_downloads is not None:
 369                         if self._num_downloads > int(max_downloads):
 370                                 raise MaxDownloadsReached()
 371
 372                 filename = self.prepare_filename(info_dict)
 373                 filename = sanitize_filename(filename, self.params.get('restrictfilenames'))
 374
 375                 # Forced printings
 376                 if self.params.get('forcetitle', False):
 377                         print(info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace'))
 378                 if self.params.get('forceurl', False):
 379                         print(info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace'))
 380                 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 381                         print(info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace'))
 382                 if self.params.get('forcedescription', False) and 'description' in info_dict:
 383                         print(info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace'))
 384                 if self.params.get('forcefilename', False) and filename is not None:
 385                         print(filename.encode(preferredencoding(), 'xmlcharrefreplace'))
 386                 if self.params.get('forceformat', False):
 387                         print(info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace'))
 388
 389                 # Do nothing else if in simulate mode
 390                 if self.params.get('simulate', False):
 391                         return
 392
 393                 if filename is None:
 394                         return
 395
 396                 try:
 397                         dn = os.path.dirname(encodeFilename(filename))
 398                         if dn != '' and not os.path.exists(dn): # dn is already encoded
 399                                 os.makedirs(dn)
 400                 except (OSError, IOError), err:
 401                         self.trouble(u'ERROR: unable to create directory ' + unicode(err))
 402                         return
 403
 404                 if self.params.get('writedescription', False):
 405                         try:
 406                                 descfn = filename + u'.description'
 407                                 self.report_writedescription(descfn)
 408                                 descfile = open(encodeFilename(descfn), 'wb')
 409                                 try:
 410                                         descfile.write(info_dict['description'].encode('utf-8'))
 411                                 finally:
 412                                         descfile.close()
 413                         except (OSError, IOError):
 414                                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
 415                                 return
 416
 417                 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
 418                         # subtitles download errors are already managed as troubles in relevant IE
 419                         # that way it will silently go on when used with unsupporting IE
 420                         try:
 421                                 srtfn = filename.rsplit('.', 1)[0] + u'.srt'
 422                                 self.report_writesubtitles(srtfn)
 423                                 srtfile = open(encodeFilename(srtfn), 'wb')
 424                                 try:
 425                                         srtfile.write(info_dict['subtitles'].encode('utf-8'))
 426                                 finally:
 427                                         srtfile.close()
 428                         except (OSError, IOError):
 429                                 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
 430                                 return
 431
 432                 if self.params.get('writeinfojson', False):
 433                         infofn = filename + u'.info.json'
 434                         self.report_writeinfojson(infofn)
 435                         try:
 436                                 json.dump
 437                         except (NameError,AttributeError):
 438                                 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
 439                                 return
 440                         try:
 441                                 infof = open(encodeFilename(infofn), 'wb')
 442                                 try:
 443                                         json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
 444                                         json.dump(json_info_dict, infof)
 445                                 finally:
 446                                         infof.close()
 447                         except (OSError, IOError):
 448                                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
 449                                 return
 450
 451                 if not self.params.get('skip_download', False):
 452                         if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
 453                                 success = True
 454                         else:
 455                                 try:
 456                                         success = self._do_download(filename, info_dict)
 457                                 except (OSError, IOError), err:
 458                                         raise UnavailableVideoError
 459                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 460                                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 461                                         return
 462                                 except (ContentTooShortError, ), err:
 463                                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 464                                         return
 465
 466                         if success:
 467                                 try:
 468                                         self.post_process(filename, info_dict)
 469                                 except (PostProcessingError), err:
 470                                         self.trouble(u'ERROR: postprocessing: %s' % str(err))
 471                                         return
 472
 473         def download(self, url_list):
 474                 """Download a given list of URLs."""
 475                 if len(url_list) > 1 and self.fixed_template():
 476                         raise SameFileError(self.params['outtmpl'])
 477
 478                 for url in url_list:
 479                         suitable_found = False
 480                         for ie in self._ies:
 481                                 # Go to next InfoExtractor if not suitable
 482                                 if not ie.suitable(url):
 483                                         continue
 484
 485                                 # Warn if the _WORKING attribute is False
 486                                 if not ie.working():
 487                                         self.trouble(u'WARNING: the program functionality for this site has been marked as broken, '
 488                                                          u'and will probably not work. If you want to go on, use the -i option.')
 489
 490                                 # Suitable InfoExtractor found
 491                                 suitable_found = True
 492
 493                                 # Extract information from URL and process it
 494                                 videos = ie.extract(url)
 495                                 for video in videos or []:
 496                                         video['extractor'] = ie.IE_NAME
 497                                         try:
 498                                                 self.increment_downloads()
 499                                                 self.process_info(video)
 500                                         except UnavailableVideoError:
 501                                                 self.trouble(u'\nERROR: unable to download video')
 502
 503                                 # Suitable InfoExtractor had been found; go to next URL
 504                                 break
 505
 506                         if not suitable_found:
 507                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 508
 509                 return self._download_retcode
 510
 511         def post_process(self, filename, ie_info):
 512                 """Run the postprocessing chain on the given file."""
 513                 info = dict(ie_info)
 514                 info['filepath'] = filename
 515                 for pp in self._pps:
 516                         info = pp.run(info)
 517                         if info is None:
 518                                 break
 519
 520         def _download_with_rtmpdump(self, filename, url, player_url):
 521                 self.report_destination(filename)
 522                 tmpfilename = self.temp_name(filename)
 523
 524                 # Check for rtmpdump first
 525                 try:
 526                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 527                 except (OSError, IOError):
 528                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 529                         return False
 530
 531                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 532                 # the connection was interrumpted and resuming appears to be
 533                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 534                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 535                 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
 536                 if self.params.get('verbose', False):
 537                         try:
 538                                 import pipes
 539                                 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
 540                         except ImportError:
 541                                 shell_quote = repr
 542                         self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
 543                 retval = subprocess.call(args)
 544                 while retval == 2 or retval == 1:
 545                         prevsize = os.path.getsize(encodeFilename(tmpfilename))
 546                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 547                         time.sleep(5.0) # This seems to be needed
 548                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 549                         cursize = os.path.getsize(encodeFilename(tmpfilename))
 550                         if prevsize == cursize and retval == 1:
 551                                 break
 552                          # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
 553                         if prevsize == cursize and retval == 2 and cursize > 1024:
 554                                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
 555                                 retval = 0
 556                                 break
 557                 if retval == 0:
 558                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename)))
 559                         self.try_rename(tmpfilename, filename)
 560                         return True
 561                 else:
 562                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 563                         return False
 564
 565         def _do_download(self, filename, info_dict):
 566                 url = info_dict['url']
 567                 player_url = info_dict.get('player_url', None)
 568
 569                 # Check file already present
 570                 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
 571                         self.report_file_already_downloaded(filename)
 572                         return True
 573
 574                 # Attempt to download using rtmpdump
 575                 if url.startswith('rtmp'):
 576                         return self._download_with_rtmpdump(filename, url, player_url)
 577
 578                 tmpfilename = self.temp_name(filename)
 579                 stream = None
 580
 581                 # Do not include the Accept-Encoding header
 582                 headers = {'Youtubedl-no-compression': 'True'}
 583                 basic_request = urllib2.Request(url, None, headers)
 584                 request = urllib2.Request(url, None, headers)
 585
 586                 # Establish possible resume length
 587                 if os.path.isfile(encodeFilename(tmpfilename)):
 588                         resume_len = os.path.getsize(encodeFilename(tmpfilename))
 589                 else:
 590                         resume_len = 0
 591
 592                 open_mode = 'wb'
 593                 if resume_len != 0:
 594                         if self.params.get('continuedl', False):
 595                                 self.report_resuming_byte(resume_len)
 596                                 request.add_header('Range','bytes=%d-' % resume_len)
 597                                 open_mode = 'ab'
 598                         else:
 599                                 resume_len = 0
 600
 601                 count = 0
 602                 retries = self.params.get('retries', 0)
 603                 while count <= retries:
 604                         # Establish connection
 605                         try:
 606                                 if count == 0 and 'urlhandle' in info_dict:
 607                                         data = info_dict['urlhandle']
 608                                 data = urllib2.urlopen(request)
 609                                 break
 610                         except (urllib2.HTTPError, ), err:
 611                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 612                                         # Unexpected HTTP error
 613                                         raise
 614                                 elif err.code == 416:
 615                                         # Unable to resume (requested range not satisfiable)
 616                                         try:
 617                                                 # Open the connection again without the range header
 618                                                 data = urllib2.urlopen(basic_request)
 619                                                 content_length = data.info()['Content-Length']
 620                                         except (urllib2.HTTPError, ), err:
 621                                                 if err.code < 500 or err.code >= 600:
 622                                                         raise
 623                                         else:
 624                                                 # Examine the reported length
 625                                                 if (content_length is not None and
 626                                                                 (resume_len - 100 < long(content_length) < resume_len + 100)):
 627                                                         # The file had already been fully downloaded.
 628                                                         # Explanation to the above condition: in issue #175 it was revealed that
 629                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 630                                                         # changing the file size slightly and causing problems for some users. So
 631                                                         # I decided to implement a suggested change and consider the file
 632                                                         # completely downloaded if the file size differs less than 100 bytes from
 633                                                         # the one in the hard drive.
 634                                                         self.report_file_already_downloaded(filename)
 635                                                         self.try_rename(tmpfilename, filename)
 636                                                         return True
 637                                                 else:
 638                                                         # The length does not match, we start the download over
 639                                                         self.report_unable_to_resume()
 640                                                         open_mode = 'wb'
 641                                                         break
 642                         # Retry
 643                         count += 1
 644                         if count <= retries:
 645                                 self.report_retry(count, retries)
 646
 647                 if count > retries:
 648                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 649                         return False
 650
 651                 data_len = data.info().get('Content-length', None)
 652                 if data_len is not None:
 653                         data_len = long(data_len) + resume_len
 654                 data_len_str = self.format_bytes(data_len)
 655                 byte_counter = 0 + resume_len
 656                 block_size = 1024
 657                 start = time.time()
 658                 while True:
 659                         # Download and write
 660                         before = time.time()
 661                         data_block = data.read(block_size)
 662                         after = time.time()
 663                         if len(data_block) == 0:
 664                                 break
 665                         byte_counter += len(data_block)
 666
 667                         # Open file just in time
 668                         if stream is None:
 669                                 try:
 670                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 671                                         assert stream is not None
 672                                         filename = self.undo_temp_name(tmpfilename)
 673                                         self.report_destination(filename)
 674                                 except (OSError, IOError), err:
 675                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 676                                         return False
 677                         try:
 678                                 stream.write(data_block)
 679                         except (IOError, OSError), err:
 680                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 681                                 return False
 682                         block_size = self.best_block_size(after - before, len(data_block))
 683
 684                         # Progress message
 685                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
 686                         if data_len is None:
 687                                 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
 688                         else:
 689                                 percent_str = self.calc_percent(byte_counter, data_len)
 690                                 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 691                                 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 692
 693                         # Apply rate limit
 694                         self.slow_down(start, byte_counter - resume_len)
 695
 696                 if stream is None:
 697                         self.trouble(u'\nERROR: Did not get any data blocks')
 698                         return False
 699                 stream.close()
 700                 self.report_finish()
 701                 if data_len is not None and byte_counter != data_len:
 702                         raise ContentTooShortError(byte_counter, long(data_len))
 703                 self.try_rename(tmpfilename, filename)
 704
 705                 # Update file modification time
 706                 if self.params.get('updatetime', True):
 707                         info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
 708
 709                 return True