_ Git - youtube-dl/blob - youtube_dl/FileDownloader.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 import httplib
   5 import math
   6 import os
   7 import re
   8 import socket
   9 import subprocess
  10 import sys
  11 import time
  12 import urllib2
  13
  14 if os.name == 'nt':
  15         import ctypes
  16
  17 from utils import *
  18
  19
  20 class FileDownloader(object):
  21         """File Downloader class.
  22
  23         File downloader objects are the ones responsible of downloading the
  24         actual video file and writing it to disk if the user has requested
  25         it, among some other tasks. In most cases there should be one per
  26         program. As, given a video URL, the downloader doesn't know how to
  27         extract all the needed information, task that InfoExtractors do, it
  28         has to pass the URL to one of them.
  29
  30         For this, file downloader objects have a method that allows
  31         InfoExtractors to be registered in a given order. When it is passed
  32         a URL, the file downloader handles it to the first InfoExtractor it
  33         finds that reports being able to handle it. The InfoExtractor extracts
  34         all the information about the video or videos the URL refers to, and
  35         asks the FileDownloader to process the video information, possibly
  36         downloading the video.
  37
  38         File downloaders accept a lot of parameters. In order not to saturate
  39         the object constructor with arguments, it receives a dictionary of
  40         options instead. These options are available through the params
  41         attribute for the InfoExtractors to use. The FileDownloader also
  42         registers itself as the downloader in charge for the InfoExtractors
  43         that are added to it, so this is a "mutual registration".
  44
  45         Available options:
  46
  47         username:          Username for authentication purposes.
  48         password:          Password for authentication purposes.
  49         usenetrc:          Use netrc for authentication instead.
  50         quiet:             Do not print messages to stdout.
  51         forceurl:          Force printing final URL.
  52         forcetitle:        Force printing title.
  53         forcethumbnail:    Force printing thumbnail URL.
  54         forcedescription:  Force printing description.
  55         forcefilename:     Force printing final filename.
  56         simulate:          Do not download the video files.
  57         format:            Video format code.
  58         format_limit:      Highest quality format to try.
  59         outtmpl:           Template for output names.
  60         restrictfilenames: Do not allow "&" and spaces in file names
  61         ignoreerrors:      Do not stop on download errors.
  62         ratelimit:         Download speed limit, in bytes/sec.
  63         nooverwrites:      Prevent overwriting files.
  64         retries:           Number of times to retry for HTTP error 5xx
  65         buffersize:        Size of download buffer in bytes.
  66         noresizebuffer:    Do not automatically resize the download buffer.
  67         continuedl:        Try to continue downloads if possible.
  68         noprogress:        Do not print the progress bar.
  69         playliststart:     Playlist item to start at.
  70         playlistend:       Playlist item to end at.
  71         matchtitle:        Download only matching titles.
  72         rejecttitle:       Reject downloads for matching titles.
  73         logtostderr:       Log messages to stderr instead of stdout.
  74         consoletitle:      Display progress in console window's titlebar.
  75         nopart:            Do not use temporary .part files.
  76         updatetime:        Use the Last-modified header to set output file timestamps.
  77         writedescription:  Write the video description to a .description file
  78         writeinfojson:     Write the video description to a .info.json file
  79         writesubtitles:    Write the video subtitles to a .srt file
  80         subtitleslang:     Language of the subtitles to download
  81         """
  82
  83         params = None
  84         _ies = []
  85         _pps = []
  86         _download_retcode = None
  87         _num_downloads = None
  88         _screen_file = None
  89
  90         def __init__(self, params):
  91                 """Create a FileDownloader object with the given options."""
  92                 self._ies = []
  93                 self._pps = []
  94                 self._download_retcode = 0
  95                 self._num_downloads = 0
  96                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
  97                 self.params = params
  98
  99                 if '%(stitle)s' in self.params['outtmpl']:
 100                         self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
 101
 102         @staticmethod
 103         def format_bytes(bytes):
 104                 if bytes is None:
 105                         return 'N/A'
 106                 if type(bytes) is str:
 107                         bytes = float(bytes)
 108                 if bytes == 0.0:
 109                         exponent = 0
 110                 else:
 111                         exponent = long(math.log(bytes, 1024.0))
 112                 suffix = 'bkMGTPEZY'[exponent]
 113                 converted = float(bytes) / float(1024 ** exponent)
 114                 return '%.2f%s' % (converted, suffix)
 115
 116         @staticmethod
 117         def calc_percent(byte_counter, data_len):
 118                 if data_len is None:
 119                         return '---.-%'
 120                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 121
 122         @staticmethod
 123         def calc_eta(start, now, total, current):
 124                 if total is None:
 125                         return '--:--'
 126                 dif = now - start
 127                 if current == 0 or dif < 0.001: # One millisecond
 128                         return '--:--'
 129                 rate = float(current) / dif
 130                 eta = long((float(total) - float(current)) / rate)
 131                 (eta_mins, eta_secs) = divmod(eta, 60)
 132                 if eta_mins > 99:
 133                         return '--:--'
 134                 return '%02d:%02d' % (eta_mins, eta_secs)
 135
 136         @staticmethod
 137         def calc_speed(start, now, bytes):
 138                 dif = now - start
 139                 if bytes == 0 or dif < 0.001: # One millisecond
 140                         return '%10s' % '---b/s'
 141                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 142
 143         @staticmethod
 144         def best_block_size(elapsed_time, bytes):
 145                 new_min = max(bytes / 2.0, 1.0)
 146                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 147                 if elapsed_time < 0.001:
 148                         return int(new_max)
 149                 rate = bytes / elapsed_time
 150                 if rate > new_max:
 151                         return int(new_max)
 152                 if rate < new_min:
 153                         return int(new_min)
 154                 return int(rate)
 155
 156         @staticmethod
 157         def parse_bytes(bytestr):
 158                 """Parse a string indicating a byte quantity into an integer."""
 159                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 160                 if matchobj is None:
 161                         return None
 162                 number = float(matchobj.group(1))
 163                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 164                 return int(round(number * multiplier))
 165
 166         def add_info_extractor(self, ie):
 167                 """Add an InfoExtractor object to the end of the list."""
 168                 self._ies.append(ie)
 169                 ie.set_downloader(self)
 170
 171         def add_post_processor(self, pp):
 172                 """Add a PostProcessor object to the end of the chain."""
 173                 self._pps.append(pp)
 174                 pp.set_downloader(self)
 175
 176         def to_screen(self, message, skip_eol=False):
 177                 """Print message to stdout if not in quiet mode."""
 178                 assert type(message) == type(u'')
 179                 if not self.params.get('quiet', False):
 180                         terminator = [u'\n', u''][skip_eol]
 181                         output = message + terminator
 182                         if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
 183                                 output = output.encode(preferredencoding(), 'ignore')
 184                         self._screen_file.write(output)
 185                         self._screen_file.flush()
 186
 187         def to_stderr(self, message):
 188                 """Print message to stderr."""
 189                 assert type(message) == type(u'')
 190                 sys.stderr.write((message + u'\n').encode(preferredencoding()))
 191
 192         def to_cons_title(self, message):
 193                 """Set console/terminal window title to message."""
 194                 if not self.params.get('consoletitle', False):
 195                         return
 196                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 197                         # c_wchar_p() might not be necessary if `message` is
 198                         # already of type unicode()
 199                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 200                 elif 'TERM' in os.environ:
 201                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
 202
 203         def fixed_template(self):
 204                 """Checks if the output template is fixed."""
 205                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 206
 207         def trouble(self, message=None):
 208                 """Determine action to take when a download problem appears.
 209
 210                 Depending on if the downloader has been configured to ignore
 211                 download errors or not, this method may throw an exception or
 212                 not when errors are found, after printing the message.
 213                 """
 214                 if message is not None:
 215                         self.to_stderr(message)
 216                 if not self.params.get('ignoreerrors', False):
 217                         raise DownloadError(message)
 218                 self._download_retcode = 1
 219
 220         def slow_down(self, start_time, byte_counter):
 221                 """Sleep if the download speed is over the rate limit."""
 222                 rate_limit = self.params.get('ratelimit', None)
 223                 if rate_limit is None or byte_counter == 0:
 224                         return
 225                 now = time.time()
 226                 elapsed = now - start_time
 227                 if elapsed <= 0.0:
 228                         return
 229                 speed = float(byte_counter) / elapsed
 230                 if speed > rate_limit:
 231                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 232
 233         def temp_name(self, filename):
 234                 """Returns a temporary filename for the given filename."""
 235                 if self.params.get('nopart', False) or filename == u'-' or \
 236                                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
 237                         return filename
 238                 return filename + u'.part'
 239
 240         def undo_temp_name(self, filename):
 241                 if filename.endswith(u'.part'):
 242                         return filename[:-len(u'.part')]
 243                 return filename
 244
 245         def try_rename(self, old_filename, new_filename):
 246                 try:
 247                         if old_filename == new_filename:
 248                                 return
 249                         os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
 250                 except (IOError, OSError), err:
 251                         self.trouble(u'ERROR: unable to rename file')
 252
 253         def try_utime(self, filename, last_modified_hdr):
 254                 """Try to set the last-modified time of the given file."""
 255                 if last_modified_hdr is None:
 256                         return
 257                 if not os.path.isfile(encodeFilename(filename)):
 258                         return
 259                 timestr = last_modified_hdr
 260                 if timestr is None:
 261                         return
 262                 filetime = timeconvert(timestr)
 263                 if filetime is None:
 264                         return filetime
 265                 try:
 266                         os.utime(filename, (time.time(), filetime))
 267                 except:
 268                         pass
 269                 return filetime
 270
 271         def report_writedescription(self, descfn):
 272                 """ Report that the description file is being written """
 273                 self.to_screen(u'[info] Writing video description to: ' + descfn)
 274
 275         def report_writesubtitles(self, srtfn):
 276                 """ Report that the subtitles file is being written """
 277                 self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
 278
 279         def report_writeinfojson(self, infofn):
 280                 """ Report that the metadata file has been written """
 281                 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
 282
 283         def report_destination(self, filename):
 284                 """Report destination filename."""
 285                 self.to_screen(u'[download] Destination: ' + filename)
 286
 287         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 288                 """Report download progress."""
 289                 if self.params.get('noprogress', False):
 290                         return
 291                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
 292                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 293                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
 294                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 295
 296         def report_resuming_byte(self, resume_len):
 297                 """Report attempt to resume at given byte."""
 298                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
 299
 300         def report_retry(self, count, retries):
 301                 """Report retry in case of HTTP error 5xx"""
 302                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 303
 304         def report_file_already_downloaded(self, file_name):
 305                 """Report file has already been fully downloaded."""
 306                 try:
 307                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
 308                 except (UnicodeEncodeError), err:
 309                         self.to_screen(u'[download] The file has already been downloaded')
 310
 311         def report_unable_to_resume(self):
 312                 """Report it was impossible to resume download."""
 313                 self.to_screen(u'[download] Unable to resume')
 314
 315         def report_finish(self):
 316                 """Report download finished."""
 317                 if self.params.get('noprogress', False):
 318                         self.to_screen(u'[download] Download completed')
 319                 else:
 320                         self.to_screen(u'')
 321
 322         def increment_downloads(self):
 323                 """Increment the ordinal that assigns a number to each file."""
 324                 self._num_downloads += 1
 325
 326         def prepare_filename(self, info_dict):
 327                 """Generate the output filename."""
 328                 try:
 329                         template_dict = dict(info_dict)
 330                         template_dict['epoch'] = unicode(int(time.time()))
 331                         template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
 332                         filename = self.params['outtmpl'] % template_dict
 333                         return filename
 334                 except (ValueError, KeyError), err:
 335                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
 336                         return None
 337
 338         def _match_entry(self, info_dict):
 339                 """ Returns None iff the file should be downloaded """
 340
 341                 title = info_dict['title']
 342                 matchtitle = self.params.get('matchtitle', False)
 343                 if matchtitle:
 344                         matchtitle = matchtitle.decode('utf8')
 345                         if not re.search(matchtitle, title, re.IGNORECASE):
 346                                 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
 347                 rejecttitle = self.params.get('rejecttitle', False)
 348                 if rejecttitle:
 349                         rejecttitle = rejecttitle.decode('utf8')
 350                         if re.search(rejecttitle, title, re.IGNORECASE):
 351                                 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 352                 return None
 353
 354         def process_info(self, info_dict):
 355                 """Process a single dictionary returned by an InfoExtractor."""
 356
 357                 # Keep for backwards compatibility
 358                 info_dict['stitle'] = info_dict['title']
 359
 360                 reason = self._match_entry(info_dict)
 361                 if reason is not None:
 362                         self.to_screen(u'[download] ' + reason)
 363                         return
 364
 365                 max_downloads = self.params.get('max_downloads')
 366                 if max_downloads is not None:
 367                         if self._num_downloads > int(max_downloads):
 368                                 raise MaxDownloadsReached()
 369
 370                 filename = self.prepare_filename(info_dict)
 371                 filename = sanitize_filename(filename, self.params.get('restrictfilenames'))
 372
 373                 # Forced printings
 374                 if self.params.get('forcetitle', False):
 375                         print(info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace'))
 376                 if self.params.get('forceurl', False):
 377                         print(info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace'))
 378                 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
 379                         print(info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace'))
 380                 if self.params.get('forcedescription', False) and 'description' in info_dict:
 381                         print(info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace'))
 382                 if self.params.get('forcefilename', False) and filename is not None:
 383                         print(filename.encode(preferredencoding(), 'xmlcharrefreplace'))
 384                 if self.params.get('forceformat', False):
 385                         print(info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace'))
 386
 387                 # Do nothing else if in simulate mode
 388                 if self.params.get('simulate', False):
 389                         return
 390
 391                 if filename is None:
 392                         return
 393
 394                 try:
 395                         dn = os.path.dirname(encodeFilename(filename))
 396                         if dn != '' and not os.path.exists(dn): # dn is already encoded
 397                                 os.makedirs(dn)
 398                 except (OSError, IOError), err:
 399                         self.trouble(u'ERROR: unable to create directory ' + unicode(err))
 400                         return
 401
 402                 if self.params.get('writedescription', False):
 403                         try:
 404                                 descfn = filename + u'.description'
 405                                 self.report_writedescription(descfn)
 406                                 descfile = open(encodeFilename(descfn), 'wb')
 407                                 try:
 408                                         descfile.write(info_dict['description'].encode('utf-8'))
 409                                 finally:
 410                                         descfile.close()
 411                         except (OSError, IOError):
 412                                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
 413                                 return
 414
 415                 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
 416                         # subtitles download errors are already managed as troubles in relevant IE
 417                         # that way it will silently go on when used with unsupporting IE
 418                         try:
 419                                 srtfn = filename.rsplit('.', 1)[0] + u'.srt'
 420                                 self.report_writesubtitles(srtfn)
 421                                 srtfile = open(encodeFilename(srtfn), 'wb')
 422                                 try:
 423                                         srtfile.write(info_dict['subtitles'].encode('utf-8'))
 424                                 finally:
 425                                         srtfile.close()
 426                         except (OSError, IOError):
 427                                 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
 428                                 return
 429
 430                 if self.params.get('writeinfojson', False):
 431                         infofn = filename + u'.info.json'
 432                         self.report_writeinfojson(infofn)
 433                         try:
 434                                 json.dump
 435                         except (NameError,AttributeError):
 436                                 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
 437                                 return
 438                         try:
 439                                 infof = open(encodeFilename(infofn), 'wb')
 440                                 try:
 441                                         json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
 442                                         json.dump(json_info_dict, infof)
 443                                 finally:
 444                                         infof.close()
 445                         except (OSError, IOError):
 446                                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
 447                                 return
 448
 449                 if not self.params.get('skip_download', False):
 450                         if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
 451                                 success = True
 452                         else:
 453                                 try:
 454                                         success = self._do_download(filename, info_dict)
 455                                 except (OSError, IOError), err:
 456                                         raise UnavailableVideoError
 457                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 458                                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 459                                         return
 460                                 except (ContentTooShortError, ), err:
 461                                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 462                                         return
 463
 464                         if success:
 465                                 try:
 466                                         self.post_process(filename, info_dict)
 467                                 except (PostProcessingError), err:
 468                                         self.trouble(u'ERROR: postprocessing: %s' % str(err))
 469                                         return
 470
 471         def download(self, url_list):
 472                 """Download a given list of URLs."""
 473                 if len(url_list) > 1 and self.fixed_template():
 474                         raise SameFileError(self.params['outtmpl'])
 475
 476                 for url in url_list:
 477                         suitable_found = False
 478                         for ie in self._ies:
 479                                 # Go to next InfoExtractor if not suitable
 480                                 if not ie.suitable(url):
 481                                         continue
 482
 483                                 # Suitable InfoExtractor found
 484                                 suitable_found = True
 485
 486                                 # Extract information from URL and process it
 487                                 videos = ie.extract(url)
 488                                 for video in videos or []:
 489                                         video['extractor'] = ie.IE_NAME
 490                                         try:
 491                                                 self.increment_downloads()
 492                                                 self.process_info(video)
 493                                         except UnavailableVideoError:
 494                                                 self.trouble(u'\nERROR: unable to download video')
 495
 496                                 # Suitable InfoExtractor had been found; go to next URL
 497                                 break
 498
 499                         if not suitable_found:
 500                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 501
 502                 return self._download_retcode
 503
 504         def post_process(self, filename, ie_info):
 505                 """Run the postprocessing chain on the given file."""
 506                 info = dict(ie_info)
 507                 info['filepath'] = filename
 508                 for pp in self._pps:
 509                         info = pp.run(info)
 510                         if info is None:
 511                                 break
 512
 513         def _download_with_rtmpdump(self, filename, url, player_url):
 514                 self.report_destination(filename)
 515                 tmpfilename = self.temp_name(filename)
 516
 517                 # Check for rtmpdump first
 518                 try:
 519                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
 520                 except (OSError, IOError):
 521                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 522                         return False
 523
 524                 # Download using rtmpdump. rtmpdump returns exit code 2 when
 525                 # the connection was interrumpted and resuming appears to be
 526                 # possible. This is part of rtmpdump's normal usage, AFAIK.
 527                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
 528                 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
 529                 if self.params.get('verbose', False):
 530                         try:
 531                                 import pipes
 532                                 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
 533                         except ImportError:
 534                                 shell_quote = repr
 535                         self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
 536                 retval = subprocess.call(args)
 537                 while retval == 2 or retval == 1:
 538                         prevsize = os.path.getsize(encodeFilename(tmpfilename))
 539                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
 540                         time.sleep(5.0) # This seems to be needed
 541                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
 542                         cursize = os.path.getsize(encodeFilename(tmpfilename))
 543                         if prevsize == cursize and retval == 1:
 544                                 break
 545                          # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
 546                         if prevsize == cursize and retval == 2 and cursize > 1024:
 547                                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
 548                                 retval = 0
 549                                 break
 550                 if retval == 0:
 551                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename)))
 552                         self.try_rename(tmpfilename, filename)
 553                         return True
 554                 else:
 555                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 556                         return False
 557
 558         def _do_download(self, filename, info_dict):
 559                 url = info_dict['url']
 560                 player_url = info_dict.get('player_url', None)
 561
 562                 # Check file already present
 563                 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
 564                         self.report_file_already_downloaded(filename)
 565                         return True
 566
 567                 # Attempt to download using rtmpdump
 568                 if url.startswith('rtmp'):
 569                         return self._download_with_rtmpdump(filename, url, player_url)
 570
 571                 tmpfilename = self.temp_name(filename)
 572                 stream = None
 573
 574                 # Do not include the Accept-Encoding header
 575                 headers = {'Youtubedl-no-compression': 'True'}
 576                 basic_request = urllib2.Request(url, None, headers)
 577                 request = urllib2.Request(url, None, headers)
 578
 579                 # Establish possible resume length
 580                 if os.path.isfile(encodeFilename(tmpfilename)):
 581                         resume_len = os.path.getsize(encodeFilename(tmpfilename))
 582                 else:
 583                         resume_len = 0
 584
 585                 open_mode = 'wb'
 586                 if resume_len != 0:
 587                         if self.params.get('continuedl', False):
 588                                 self.report_resuming_byte(resume_len)
 589                                 request.add_header('Range','bytes=%d-' % resume_len)
 590                                 open_mode = 'ab'
 591                         else:
 592                                 resume_len = 0
 593
 594                 count = 0
 595                 retries = self.params.get('retries', 0)
 596                 while count <= retries:
 597                         # Establish connection
 598                         try:
 599                                 if count == 0 and 'urlhandle' in info_dict:
 600                                         data = info_dict['urlhandle']
 601                                 data = urllib2.urlopen(request)
 602                                 break
 603                         except (urllib2.HTTPError, ), err:
 604                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
 605                                         # Unexpected HTTP error
 606                                         raise
 607                                 elif err.code == 416:
 608                                         # Unable to resume (requested range not satisfiable)
 609                                         try:
 610                                                 # Open the connection again without the range header
 611                                                 data = urllib2.urlopen(basic_request)
 612                                                 content_length = data.info()['Content-Length']
 613                                         except (urllib2.HTTPError, ), err:
 614                                                 if err.code < 500 or err.code >= 600:
 615                                                         raise
 616                                         else:
 617                                                 # Examine the reported length
 618                                                 if (content_length is not None and
 619                                                                 (resume_len - 100 < long(content_length) < resume_len + 100)):
 620                                                         # The file had already been fully downloaded.
 621                                                         # Explanation to the above condition: in issue #175 it was revealed that
 622                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
 623                                                         # changing the file size slightly and causing problems for some users. So
 624                                                         # I decided to implement a suggested change and consider the file
 625                                                         # completely downloaded if the file size differs less than 100 bytes from
 626                                                         # the one in the hard drive.
 627                                                         self.report_file_already_downloaded(filename)
 628                                                         self.try_rename(tmpfilename, filename)
 629                                                         return True
 630                                                 else:
 631                                                         # The length does not match, we start the download over
 632                                                         self.report_unable_to_resume()
 633                                                         open_mode = 'wb'
 634                                                         break
 635                         # Retry
 636                         count += 1
 637                         if count <= retries:
 638                                 self.report_retry(count, retries)
 639
 640                 if count > retries:
 641                         self.trouble(u'ERROR: giving up after %s retries' % retries)
 642                         return False
 643
 644                 data_len = data.info().get('Content-length', None)
 645                 if data_len is not None:
 646                         data_len = long(data_len) + resume_len
 647                 data_len_str = self.format_bytes(data_len)
 648                 byte_counter = 0 + resume_len
 649                 block_size = self.params.get('buffersize', 1024)
 650                 start = time.time()
 651                 while True:
 652                         # Download and write
 653                         before = time.time()
 654                         data_block = data.read(block_size)
 655                         after = time.time()
 656                         if len(data_block) == 0:
 657                                 break
 658                         byte_counter += len(data_block)
 659
 660                         # Open file just in time
 661                         if stream is None:
 662                                 try:
 663                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
 664                                         assert stream is not None
 665                                         filename = self.undo_temp_name(tmpfilename)
 666                                         self.report_destination(filename)
 667                                 except (OSError, IOError), err:
 668                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 669                                         return False
 670                         try:
 671                                 stream.write(data_block)
 672                         except (IOError, OSError), err:
 673                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 674                                 return False
 675                         if not self.params.get('noresizebuffer', False):
 676                                 block_size = self.best_block_size(after - before, len(data_block))
 677
 678                         # Progress message
 679                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
 680                         if data_len is None:
 681                                 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
 682                         else:
 683                                 percent_str = self.calc_percent(byte_counter, data_len)
 684                                 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 685                                 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 686
 687                         # Apply rate limit
 688                         self.slow_down(start, byte_counter - resume_len)
 689
 690                 if stream is None:
 691                         self.trouble(u'\nERROR: Did not get any data blocks')
 692                         return False
 693                 stream.close()
 694                 self.report_finish()
 695                 if data_len is not None and byte_counter != data_len:
 696                         raise ContentTooShortError(byte_counter, long(data_len))
 697                 self.try_rename(tmpfilename, filename)
 698
 699                 # Update file modification time
 700                 if self.params.get('updatetime', True):
 701                         info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
 702
 703                 return True