_ Git - youtube-dl/blob - youtube_dl/FileDownloader.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import httplib
   5 import math
   6 import os
   7 import re
   8 import socket
   9 import subprocess
  10 import sys
  11 import time
  12 import urllib2
  13
  14 if os.name == 'nt':
  15         import ctypes
  16
  17 from utils import *
  18
  19
  20 class FileDownloader(object):
  21         """File Downloader class.
  22
  23         File downloader objects are the ones responsible of downloading the
  24         actual video file and writing it to disk if the user has requested
  25         it, among some other tasks. In most cases there should be one per
  26         program. As, given a video URL, the downloader doesn't know how to
  27         extract all the needed information, task that InfoExtractors do, it
  28         has to pass the URL to one of them.
  29
  30         For this, file downloader objects have a method that allows
  31         InfoExtractors to be registered in a given order. When it is passed
  32         a URL, the file downloader handles it to the first InfoExtractor it
  33         finds that reports being able to handle it. The InfoExtractor extracts
  34         all the information about the video or videos the URL refers to, and
  35         asks the FileDownloader to process the video information, possibly
  36         downloading the video.
  37
  38         File downloaders accept a lot of parameters. In order not to saturate
  39         the object constructor with arguments, it receives a dictionary of
  40         options instead. These options are available through the params
  41         attribute for the InfoExtractors to use. The FileDownloader also
  42         registers itself as the downloader in charge for the InfoExtractors
  43         that are added to it, so this is a "mutual registration".
  44
  45         Available options:
  46
  47         username:          Username for authentication purposes.
  48         password:          Password for authentication purposes.
  49         usenetrc:          Use netrc for authentication instead.
  50         quiet:             Do not print messages to stdout.
  51         forceurl:          Force printing final URL.
  52         forcetitle:        Force printing title.
  53         forcethumbnail:    Force printing thumbnail URL.
  54         forcedescription:  Force printing description.
  55         forcefilename:     Force printing final filename.
  56         simulate:          Do not download the video files.
  57         format:            Video format code.
  58         format_limit:      Highest quality format to try.
  59         outtmpl:           Template for output names.
  60         restrictfilenames: Do not allow "&" and spaces in file names
  61         ignoreerrors:      Do not stop on download errors.
  62         ratelimit:         Download speed limit, in bytes/sec.
  63         nooverwrites:      Prevent overwriting files.
  64         retries:           Number of times to retry for HTTP error 5xx
  65         buffersize:        Size of download buffer in bytes.
  66         noresizebuffer:    Do not automatically resize the download buffer.
  67         continuedl:        Try to continue downloads if possible.
  68         noprogress:        Do not print the progress bar.
  69         playliststart:     Playlist item to start at.
  70         playlistend:       Playlist item to end at.
  71         matchtitle:        Download only matching titles.
  72         rejecttitle:       Reject downloads for matching titles.
  73         logtostderr:       Log messages to stderr instead of stdout.
  74         consoletitle:      Display progress in console window's titlebar.
  75         nopart:            Do not use temporary .part files.
  76         updatetime:        Use the Last-modified header to set output file timestamps.
  77         writedescription:  Write the video description to a .description file
  78         writeinfojson:     Write the video description to a .info.json file
  79         writesubtitles:    Write the video subtitles to a .srt file
  80         subtitleslang:     Language of the subtitles to download
  81         """
  82
  83         params = None
  84         _ies = []
  85         _pps = []
  86         _download_retcode = None
  87         _num_downloads = None
  88         _screen_file = None
  89
  90         def __init__(self, params):
  91                 """Create a FileDownloader object with the given options."""
  92                 self._ies = []
  93                 self._pps = []
  94                 self._download_retcode = 0
  95                 self._num_downloads = 0
  96                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
  97                 self.params = params
  98
  99                 if '%(stitle)s' in self.params['outtmpl']:
 100                         self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
 101
 102         @staticmethod
 103         def format_bytes(bytes):
 104                 if bytes is None:
 105                         return 'N/A'
 106                 if type(bytes) is str:
 107                         bytes = float(bytes)
 108                 if bytes == 0.0:
 109                         exponent = 0
 110                 else:
 111                         exponent = long(math.log(bytes, 1024.0))
 112                 suffix = 'bkMGTPEZY'[exponent]
 113                 converted = float(bytes) / float(1024 ** exponent)
 114                 return '%.2f%s' % (converted, suffix)
 115
 116         @staticmethod
 117         def calc_percent(byte_counter, data_len):
 118                 if data_len is None:
 119                         return '---.-%'
 120                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 121
 122         @staticmethod
 123         def calc_eta(start, now, total, current):
 124                 if total is None:
 125                         return '--:--'
 126                 dif = now - start
 127                 if current == 0 or dif < 0.001: # One millisecond
 128                         return '--:--'
 129                 rate = float(current) / dif
 130                 eta = long((float(total) - float(current)) / rate)
 131                 (eta_mins, eta_secs) = divmod(eta, 60)
 132                 if eta_mins > 99:
 133                         return '--:--'
 134                 return '%02d:%02d' % (eta_mins, eta_secs)
 135
 136         @staticmethod
 137         def calc_speed(start, now, bytes):
 138                 dif = now - start
 139                 if bytes == 0 or dif < 0.001: # One millisecond
 140                         return '%10s' % '---b/s'
 141                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 142
 143         @staticmethod
 144         def best_block_size(elapsed_time, bytes):
 145                 new_min = max(bytes / 2.0, 1.0)
 146                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 147                 if elapsed_time < 0.001:
 148                         return int(new_max)
 149                 rate = bytes / elapsed_time
 150                 if rate > new_max:
 151                         return int(new_max)
 152                 if rate < new_min:
 153                         return int(new_min)
 154                 return int(rate)
 155
 156         @staticmethod
 157         def parse_bytes(bytestr):
 158                 """Parse a string indicating a byte quantity into an integer."""
 159                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 160                 if matchobj is None:
 161                         return None
 162                 number = float(matchobj.group(1))
 163                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 164                 return int(round(number * multiplier))
 165
 166         def add_info_extractor(self, ie):
 167                 """Add an InfoExtractor object to the end of the list."""
 168                 self._ies.append(ie)
 169                 ie.set_downloader(self)
 170
 171         def add_post_processor(self, pp):
 172                 """Add a PostProcessor object to the end of the chain."""
 173                 self._pps.append(pp)
 174                 pp.set_downloader(self)
 175
 176         def to_screen(self, message, skip_eol=False):
 177                 """Print message to stdout if not in quiet mode."""
 178                 assert type(message) == type(u'')
 179                 if not self.params.get('quiet', False):
 180                         terminator = [u'\n', u''][skip_eol]
 181                         output = message + terminator
 182                         if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
 183                                 output = output.encode(preferredencoding(), 'ignore')
 184                         self._screen_file.write(output)
 185                         self._screen_file.flush()
 186
 187         def to_stderr(self, message):
 188                 """Print message to stderr."""
 189                 assert type(message) == type(u'')
 190                 sys.stderr.write((message + u'\n').encode(preferredencoding()))
 191
 192         def to_cons_title(self, message):
 193                 """Set console/terminal window title to message."""
 194                 if not self.params.get('consoletitle', False):
 195                         return
 196                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 197                         # c_wchar_p() might not be necessary if `message` is
 198                         # already of type unicode()
 199                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 200                 elif 'TERM' in os.environ:
 201                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
 202
 203         def fixed_template(self):
 204                 """Checks if the output template is fixed."""
 205                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 206
 207         def trouble(self, message=None):
 208                 """Determine action to take when a download problem appears.
 209
 210                 Depending on if the downloader has been configured to ignore
 211                 download errors or not, this method may throw an exception or
 212                 not when errors are found, after printing the message.
 213                 """
 214                 if message is not None:
 215                         self.to_stderr(message)
 216                 if not self.params.get('ignoreerrors', False):
 217                         raise DownloadError(message)
 218                 self._download_retcode = 1
 219
 220         def slow_down(self, start_time, byte_counter):
 221                 """Sleep if the download speed is over the rate limit."""
 222                 rate_limit = self.params.get('ratelimit', None)
 223                 if rate_limit is None or byte_counter == 0:
 224                         return
 225                 now = time.time()
 226                 elapsed = now - start_time
 227                 if elapsed <= 0.0:
 228                         return
 229                 speed = float(byte_counter) / elapsed
 230                 if speed > rate_limit:
 231                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 232
 233         def temp_name(self, filename):
 234                 """Returns a temporary filename for the given filename."""
 235                 if self.params.get('nopart', False) or filename == u'-' or \
 236                                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 237                         return filename
 238                 return filename + u'.part'
 239
 240         def undo_temp_name(self, filename):
 241                 if filename.endswith(u'.part'):
 242                         return filename[:-len(u'.part')]
 243                 return filename
 244
 245         def try_rename(self, old_filename, new_filename):
 246                 try:
 247                         if old_filename == new_filename:
 248                                 return
 249                         os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
 250                 except (IOError, OSError), err:
 251                         self.trouble(u'ERROR: unable to rename file')
 252
 253         def try_utime(self, filename, last_modified_hdr):
 254                 """Try to set the last-modified time of the given file."""
 255                 if last_modified_hdr is None:
 256                         return
 257                 if not os.path.isfile(encodeFilename(filename)):
 258                         return
 259                 timestr = last_modified_hdr
 260                 if timestr is None:
 261                         return
 262                 filetime = timeconvert(timestr)
 263                 if filetime is None:
 264                         return filetime
 265                 try:
 266                         os.utime(filename, (time.time(), filetime))
 267                 except:
 268                         pass
 269                 return filetime
 270
 271         def report_writedescription(self, descfn):
 272                 """ Report that the description file is being written """
 273                 self.to_screen(u'[info] Writing video description to: ' + descfn)
 274
 275         def report_writesubtitles(self, srtfn):
 276                 """ Report that the subtitles file is being written """
 277                 self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
 278
 279         def report_writeinfojson(self, infofn):
 280                 """ Report that the metadata file has been written """
 281                 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
 282
 283         def report_destination(self, filename):
 284                 """Report destination filename."""
 285                 self.to_screen(u'[download] Destination: ' + filename)
 286
 287         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 288                 """Report download progress."""
 289                 if self.params.get('noprogress', False):
 290                         return
 291                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 292                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 293                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
 294                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 295
 296         def report_resuming_byte(self, resume_len):
 297                 """Report attempt to resume at given byte."""
 298                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 299
 300         def report_retry(self, count, retries):
 301                 """Report retry in case of HTTP error 5xx"""
 302                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 303
 304         def report_file_already_downloaded(self, file_name):
 305                 """Report file has already been fully downloaded."""
 306                 try:
 307                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 308                 except (UnicodeEncodeError), err:
 309                         self.to_screen(u'[download] The file has already been downloaded')
 310
 311         def report_unable_to_resume(self):
 312                 """Report it was impossible to resume download."""
 313                 self.to_screen(u'[download] Unable to resume')
 314
 315         def report_finish(self):
 316                 """Report download finished."""
 317                 if self.params.get('noprogress', False):
 318                         self.to_screen(u'[download] Download completed')
 319                 else:
 320                         self.to_screen(u'')
 321
 322         def increment_downloads(self):
 323                 """Increment the ordinal that assigns a number to each file."""
 324                 self._num_downloads += 1
 325
 326         def prepare_filename(self, info_dict):
 327                 """Generate the output filename."""
 328                 try:
 329                         template_dict = dict(info_dict)
 330                         template_dict['epoch'] = int(time.time())
 331                         template_dict['autonumber'] = u'%05d' % self._num_downloads
 332
 333                         template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
 334                         filename = self.params['outtmpl'] % template_dict
 335                         return filename
 336                 except (ValueError, KeyError), err:
 337                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 338                         return None
 339
 340         def _match_entry(self, info_dict):
 341                 """ Returns None iff the file should be downloaded """
 342
 343                 title = info_dict['title']
 344                 matchtitle = self.params.get('matchtitle', False)
 345                 if matchtitle:
 346                         matchtitle = matchtitle.decode('utf8')
 347                         if not re.search(matchtitle, title, re.IGNORECASE):
 348                                 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
 349                 rejecttitle = self.params.get('rejecttitle', False)
 350                 if rejecttitle:
 351                         rejecttitle = rejecttitle.decode('utf8')
 352                         if re.search(rejecttitle, title, re.IGNORECASE):
 353                                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 354                 return None
 355
 356         def process_info(self, info_dict):
 357                 """Process a single dictionary returned by an InfoExtractor."""
 358
 359                 # Keep for backwards compatibility
 360                 info_dict['stitle'] = info_dict['title']
 361
 362                 reason = self._match_entry(info_dict)
 363                 if reason is not None:
 364                         self.to_screen(u'[download] ' + reason)
 365                         return
 366
 367                 max_downloads = self.params.get('max_downloads')
 368                 if max_downloads is not None:
 369                         if self._num_downloads > int(max_downloads):
 370                                 raise MaxDownloadsReached()
 371
 372                 filename = self.prepare_filename(info_dict)
 373
 374                 # Forced printings
 375                 if self.params.get('forcetitle', False):
 376                         print(info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace'))
 377                 if self.params.get('forceurl', False):
 378                         print(info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace'))
 379                 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 380                         print(info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace'))
 381                 if self.params.get('forcedescription', False) and 'description' in info_dict:
 382                         print(info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace'))
 383                 if self.params.get('forcefilename', False) and filename is not None:
 384                         print(filename.encode(preferredencoding(), 'xmlcharrefreplace'))
 385                 if self.params.get('forceformat', False):
 386                         print(info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace'))
 387
 388                 # Do nothing else if in simulate mode
 389                 if self.params.get('simulate', False):
 390                         return
 391
 392                 if filename is None:
 393                         return
 394
 395                 try:
 396                         dn = os.path.dirname(encodeFilename(filename))
 397                         if dn != '' and not os.path.exists(dn): # dn is already encoded
 398                                 os.makedirs(dn)
 399                 except (OSError, IOError), err:
 400                         self.trouble(u'ERROR: unable to create directory ' + unicode(err))
 401                         return
 402
 403                 if self.params.get('writedescription', False):
 404                         try:
 405                                 descfn = filename + u'.description'
 406                                 self.report_writedescription(descfn)
 407                                 descfile = open(encodeFilename(descfn), 'wb')
 408                                 try:
 409                                         descfile.write(info_dict['description'].encode('utf-8'))
 410                                 finally:
 411                                         descfile.close()
 412                         except (OSError, IOError):
 413                                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
 414                                 return
 415
 416                 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
 417                         # subtitles download errors are already managed as troubles in relevant IE
 418                         # that way it will silently go on when used with unsupporting IE
 419                         try:
 420                                 srtfn = filename.rsplit('.', 1)[0] + u'.srt'
 421                                 self.report_writesubtitles(srtfn)
 422                                 srtfile = open(encodeFilename(srtfn), 'wb')
 423                                 try:
 424                                         srtfile.write(info_dict['subtitles'].encode('utf-8'))
 425                                 finally:
 426                                         srtfile.close()
 427                         except (OSError, IOError):
 428                                 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
 429                                 return
 430
 431                 if self.params.get('writeinfojson', False):
 432                         infofn = filename + u'.info.json'
 433                         self.report_writeinfojson(infofn)
 434                         try:
 435                                 json.dump
 436                         except (NameError,AttributeError):
 437                                 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
 438                                 return
 439                         try:
 440                                 infof = open(encodeFilename(infofn), 'wb')
 441                                 try:
 442                                         json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
 443                                         json.dump(json_info_dict, infof)
 444                                 finally:
 445                                         infof.close()
 446                         except (OSError, IOError):
 447                                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
 448                                 return
 449
 450                 if not self.params.get('skip_download', False):
 451                         if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
 452                                 success = True
 453                         else:
 454                                 try:
 455                                         success = self._do_download(filename, info_dict)
 456                                 except (OSError, IOError), err:
 457                                         raise UnavailableVideoError
 458                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 459                                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 460                                         return
 461                                 except (ContentTooShortError, ), err:
 462                                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 463                                         return
 464
 465                         if success:
 466                                 try:
 467                                         self.post_process(filename, info_dict)
 468                                 except (PostProcessingError), err:
 469                                         self.trouble(u'ERROR: postprocessing: %s' % str(err))
 470                                         return
 471
 472         def download(self, url_list):
 473                 """Download a given list of URLs."""
 474                 if len(url_list) > 1 and self.fixed_template():
 475                         raise SameFileError(self.params['outtmpl'])
 476
 477                 for url in url_list:
 478                         suitable_found = False
 479                         for ie in self._ies:
 480                                 # Go to next InfoExtractor if not suitable
 481                                 if not ie.suitable(url):
 482                                         continue
 483
 484                                 # Suitable InfoExtractor found
 485                                 suitable_found = True
 486
 487                                 # Extract information from URL and process it
 488                                 videos = ie.extract(url)
 489                                 for video in videos or []:
 490                                         video['extractor'] = ie.IE_NAME
 491                                         try:
 492                                                 self.increment_downloads()
 493                                                 self.process_info(video)
 494                                         except UnavailableVideoError:
 495                                                 self.trouble(u'\nERROR: unable to download video')
 496
 497                                 # Suitable InfoExtractor had been found; go to next URL
 498                                 break
 499
 500                         if not suitable_found:
 501                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 502
 503                 return self._download_retcode
 504
 505         def post_process(self, filename, ie_info):
 506                 """Run the postprocessing chain on the given file."""
 507                 info = dict(ie_info)
 508                 info['filepath'] = filename
 509                 for pp in self._pps:
 510                         info = pp.run(info)
 511                         if info is None:
 512                                 break
 513
 514         def _download_with_rtmpdump(self, filename, url, player_url):
 515                 self.report_destination(filename)
 516                 tmpfilename = self.temp_name(filename)
 517
 518                 # Check for rtmpdump first
 519                 try:
 520                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 521                 except (OSError, IOError):
 522                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 523                         return False
 524
 525                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 526                 # the connection was interrumpted and resuming appears to be
 527                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 528                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 529                 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
 530                 if self.params.get('verbose', False):
 531                         try:
 532                                 import pipes
 533                                 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
 534                         except ImportError:
 535                                 shell_quote = repr
 536                         self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
 537                 retval = subprocess.call(args)
 538                 while retval == 2 or retval == 1:
 539                         prevsize = os.path.getsize(encodeFilename(tmpfilename))
 540                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 541                         time.sleep(5.0) # This seems to be needed
 542                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 543                         cursize = os.path.getsize(encodeFilename(tmpfilename))
 544                         if prevsize == cursize and retval == 1:
 545                                 break
 546                          # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
 547                         if prevsize == cursize and retval == 2 and cursize > 1024:
 548                                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
 549                                 retval = 0
 550                                 break
 551                 if retval == 0:
 552                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename)))
 553                         self.try_rename(tmpfilename, filename)
 554                         return True
 555                 else:
 556                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 557                         return False
 558
 559         def _do_download(self, filename, info_dict):
 560                 url = info_dict['url']
 561                 player_url = info_dict.get('player_url', None)
 562
 563                 # Check file already present
 564                 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
 565                         self.report_file_already_downloaded(filename)
 566                         return True
 567
 568                 # Attempt to download using rtmpdump
 569                 if url.startswith('rtmp'):
 570                         return self._download_with_rtmpdump(filename, url, player_url)
 571
 572                 tmpfilename = self.temp_name(filename)
 573                 stream = None
 574
 575                 # Do not include the Accept-Encoding header
 576                 headers = {'Youtubedl-no-compression': 'True'}
 577                 basic_request = urllib2.Request(url, None, headers)
 578                 request = urllib2.Request(url, None, headers)
 579
 580                 # Establish possible resume length
 581                 if os.path.isfile(encodeFilename(tmpfilename)):
 582                         resume_len = os.path.getsize(encodeFilename(tmpfilename))
 583                 else:
 584                         resume_len = 0
 585
 586                 open_mode = 'wb'
 587                 if resume_len != 0:
 588                         if self.params.get('continuedl', False):
 589                                 self.report_resuming_byte(resume_len)
 590                                 request.add_header('Range','bytes=%d-' % resume_len)
 591                                 open_mode = 'ab'
 592                         else:
 593                                 resume_len = 0
 594
 595                 count = 0
 596                 retries = self.params.get('retries', 0)
 597                 while count <= retries:
 598                         # Establish connection
 599                         try:
 600                                 if count == 0 and 'urlhandle' in info_dict:
 601                                         data = info_dict['urlhandle']
 602                                 data = urllib2.urlopen(request)
 603                                 break
 604                         except (urllib2.HTTPError, ), err:
 605                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 606                                         # Unexpected HTTP error
 607                                         raise
 608                                 elif err.code == 416:
 609                                         # Unable to resume (requested range not satisfiable)
 610                                         try:
 611                                                 # Open the connection again without the range header
 612                                                 data = urllib2.urlopen(basic_request)
 613                                                 content_length = data.info()['Content-Length']
 614                                         except (urllib2.HTTPError, ), err:
 615                                                 if err.code < 500 or err.code >= 600:
 616                                                         raise
 617                                         else:
 618                                                 # Examine the reported length
 619                                                 if (content_length is not None and
 620                                                                 (resume_len - 100 < long(content_length) < resume_len + 100)):
 621                                                         # The file had already been fully downloaded.
 622                                                         # Explanation to the above condition: in issue #175 it was revealed that
 623                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 624                                                         # changing the file size slightly and causing problems for some users. So
 625                                                         # I decided to implement a suggested change and consider the file
 626                                                         # completely downloaded if the file size differs less than 100 bytes from
 627                                                         # the one in the hard drive.
 628                                                         self.report_file_already_downloaded(filename)
 629                                                         self.try_rename(tmpfilename, filename)
 630                                                         return True
 631                                                 else:
 632                                                         # The length does not match, we start the download over
 633                                                         self.report_unable_to_resume()
 634                                                         open_mode = 'wb'
 635                                                         break
 636                         # Retry
 637                         count += 1
 638                         if count <= retries:
 639                                 self.report_retry(count, retries)
 640
 641                 if count > retries:
 642                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 643                         return False
 644
 645                 data_len = data.info().get('Content-length', None)
 646                 if data_len is not None:
 647                         data_len = long(data_len) + resume_len
 648                 data_len_str = self.format_bytes(data_len)
 649                 byte_counter = 0 + resume_len
 650                 block_size = self.params.get('buffersize', 1024)
 651                 start = time.time()
 652                 while True:
 653                         # Download and write
 654                         before = time.time()
 655                         data_block = data.read(block_size)
 656                         after = time.time()
 657                         if len(data_block) == 0:
 658                                 break
 659                         byte_counter += len(data_block)
 660
 661                         # Open file just in time
 662                         if stream is None:
 663                                 try:
 664                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 665                                         assert stream is not None
 666                                         filename = self.undo_temp_name(tmpfilename)
 667                                         self.report_destination(filename)
 668                                 except (OSError, IOError), err:
 669                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 670                                         return False
 671                         try:
 672                                 stream.write(data_block)
 673                         except (IOError, OSError), err:
 674                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 675                                 return False
 676                         if not self.params.get('noresizebuffer', False):
 677                                 block_size = self.best_block_size(after - before, len(data_block))
 678
 679                         # Progress message
 680                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
 681                         if data_len is None:
 682                                 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
 683                         else:
 684                                 percent_str = self.calc_percent(byte_counter, data_len)
 685                                 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 686                                 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 687
 688                         # Apply rate limit
 689                         self.slow_down(start, byte_counter - resume_len)
 690
 691                 if stream is None:
 692                         self.trouble(u'\nERROR: Did not get any data blocks')
 693                         return False
 694                 stream.close()
 695                 self.report_finish()
 696                 if data_len is not None and byte_counter != data_len:
 697                         raise ContentTooShortError(byte_counter, long(data_len))
 698                 self.try_rename(tmpfilename, filename)
 699
 700                 # Update file modification time
 701                 if self.params.get('updatetime', True):
 702                         info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
 703
 704                 return True