Split code as a package, compiled into an executable zip
[youtube-dl] / youtube_dl / FileDownloader.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 import httplib
5 import math
6 import os
7 import re
8 import socket
9 import subprocess
10 import sys
11 import time
12 import urllib2
13
14 if os.name == 'nt':
15         import ctypes
16         
17 from Utils import *
18
19
20 class FileDownloader(object):
21         """File Downloader class.
22
23         File downloader objects are the ones responsible of downloading the
24         actual video file and writing it to disk if the user has requested
25         it, among some other tasks. In most cases there should be one per
26         program. As, given a video URL, the downloader doesn't know how to
27         extract all the needed information, task that InfoExtractors do, it
28         has to pass the URL to one of them.
29
30         For this, file downloader objects have a method that allows
31         InfoExtractors to be registered in a given order. When it is passed
32         a URL, the file downloader handles it to the first InfoExtractor it
33         finds that reports being able to handle it. The InfoExtractor extracts
34         all the information about the video or videos the URL refers to, and
35         asks the FileDownloader to process the video information, possibly
36         downloading the video.
37
38         File downloaders accept a lot of parameters. In order not to saturate
39         the object constructor with arguments, it receives a dictionary of
40         options instead. These options are available through the params
41         attribute for the InfoExtractors to use. The FileDownloader also
42         registers itself as the downloader in charge for the InfoExtractors
43         that are added to it, so this is a "mutual registration".
44
45         Available options:
46
47         username:         Username for authentication purposes.
48         password:         Password for authentication purposes.
49         usenetrc:         Use netrc for authentication instead.
50         quiet:            Do not print messages to stdout.
51         forceurl:         Force printing final URL.
52         forcetitle:       Force printing title.
53         forcethumbnail:   Force printing thumbnail URL.
54         forcedescription: Force printing description.
55         forcefilename:    Force printing final filename.
56         simulate:         Do not download the video files.
57         format:           Video format code.
58         format_limit:     Highest quality format to try.
59         outtmpl:          Template for output names.
60         ignoreerrors:     Do not stop on download errors.
61         ratelimit:        Download speed limit, in bytes/sec.
62         nooverwrites:     Prevent overwriting files.
63         retries:          Number of times to retry for HTTP error 5xx
64         continuedl:       Try to continue downloads if possible.
65         noprogress:       Do not print the progress bar.
66         playliststart:    Playlist item to start at.
67         playlistend:      Playlist item to end at.
68         matchtitle:       Download only matching titles.
69         rejecttitle:      Reject downloads for matching titles.
70         logtostderr:      Log messages to stderr instead of stdout.
71         consoletitle:     Display progress in console window's titlebar.
72         nopart:           Do not use temporary .part files.
73         updatetime:       Use the Last-modified header to set output file timestamps.
74         writedescription: Write the video description to a .description file
75         writeinfojson:    Write the video description to a .info.json file
76         writesubtitles:   Write the video subtitles to a .srt file
77         subtitleslang:    Language of the subtitles to download
78         """
79
80         params = None
81         _ies = []
82         _pps = []
83         _download_retcode = None
84         _num_downloads = None
85         _screen_file = None
86
87         def __init__(self, params):
88                 """Create a FileDownloader object with the given options."""
89                 self._ies = []
90                 self._pps = []
91                 self._download_retcode = 0
92                 self._num_downloads = 0
93                 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
94                 self.params = params
95
96         @staticmethod
97         def format_bytes(bytes):
98                 if bytes is None:
99                         return 'N/A'
100                 if type(bytes) is str:
101                         bytes = float(bytes)
102                 if bytes == 0.0:
103                         exponent = 0
104                 else:
105                         exponent = long(math.log(bytes, 1024.0))
106                 suffix = 'bkMGTPEZY'[exponent]
107                 converted = float(bytes) / float(1024 ** exponent)
108                 return '%.2f%s' % (converted, suffix)
109
110         @staticmethod
111         def calc_percent(byte_counter, data_len):
112                 if data_len is None:
113                         return '---.-%'
114                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
115
116         @staticmethod
117         def calc_eta(start, now, total, current):
118                 if total is None:
119                         return '--:--'
120                 dif = now - start
121                 if current == 0 or dif < 0.001: # One millisecond
122                         return '--:--'
123                 rate = float(current) / dif
124                 eta = long((float(total) - float(current)) / rate)
125                 (eta_mins, eta_secs) = divmod(eta, 60)
126                 if eta_mins > 99:
127                         return '--:--'
128                 return '%02d:%02d' % (eta_mins, eta_secs)
129
130         @staticmethod
131         def calc_speed(start, now, bytes):
132                 dif = now - start
133                 if bytes == 0 or dif < 0.001: # One millisecond
134                         return '%10s' % '---b/s'
135                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
136
137         @staticmethod
138         def best_block_size(elapsed_time, bytes):
139                 new_min = max(bytes / 2.0, 1.0)
140                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
141                 if elapsed_time < 0.001:
142                         return long(new_max)
143                 rate = bytes / elapsed_time
144                 if rate > new_max:
145                         return long(new_max)
146                 if rate < new_min:
147                         return long(new_min)
148                 return long(rate)
149
150         @staticmethod
151         def parse_bytes(bytestr):
152                 """Parse a string indicating a byte quantity into a long integer."""
153                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
154                 if matchobj is None:
155                         return None
156                 number = float(matchobj.group(1))
157                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
158                 return long(round(number * multiplier))
159
160         def add_info_extractor(self, ie):
161                 """Add an InfoExtractor object to the end of the list."""
162                 self._ies.append(ie)
163                 ie.set_downloader(self)
164
165         def add_post_processor(self, pp):
166                 """Add a PostProcessor object to the end of the chain."""
167                 self._pps.append(pp)
168                 pp.set_downloader(self)
169
170         def to_screen(self, message, skip_eol=False):
171                 """Print message to stdout if not in quiet mode."""
172                 assert type(message) == type(u'')
173                 if not self.params.get('quiet', False):
174                         terminator = [u'\n', u''][skip_eol]
175                         output = message + terminator
176
177                         if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
178                                 output = output.encode(preferredencoding(), 'ignore')
179                         self._screen_file.write(output)
180                         self._screen_file.flush()
181
182         def to_stderr(self, message):
183                 """Print message to stderr."""
184                 print >>sys.stderr, message.encode(preferredencoding())
185
186         def to_cons_title(self, message):
187                 """Set console/terminal window title to message."""
188                 if not self.params.get('consoletitle', False):
189                         return
190                 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
191                         # c_wchar_p() might not be necessary if `message` is
192                         # already of type unicode()
193                         ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
194                 elif 'TERM' in os.environ:
195                         sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
196
197         def fixed_template(self):
198                 """Checks if the output template is fixed."""
199                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
200
201         def trouble(self, message=None):
202                 """Determine action to take when a download problem appears.
203
204                 Depending on if the downloader has been configured to ignore
205                 download errors or not, this method may throw an exception or
206                 not when errors are found, after printing the message.
207                 """
208                 if message is not None:
209                         self.to_stderr(message)
210                 if not self.params.get('ignoreerrors', False):
211                         raise DownloadError(message)
212                 self._download_retcode = 1
213
214         def slow_down(self, start_time, byte_counter):
215                 """Sleep if the download speed is over the rate limit."""
216                 rate_limit = self.params.get('ratelimit', None)
217                 if rate_limit is None or byte_counter == 0:
218                         return
219                 now = time.time()
220                 elapsed = now - start_time
221                 if elapsed <= 0.0:
222                         return
223                 speed = float(byte_counter) / elapsed
224                 if speed > rate_limit:
225                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
226
227         def temp_name(self, filename):
228                 """Returns a temporary filename for the given filename."""
229                 if self.params.get('nopart', False) or filename == u'-' or \
230                                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
231                         return filename
232                 return filename + u'.part'
233
234         def undo_temp_name(self, filename):
235                 if filename.endswith(u'.part'):
236                         return filename[:-len(u'.part')]
237                 return filename
238
239         def try_rename(self, old_filename, new_filename):
240                 try:
241                         if old_filename == new_filename:
242                                 return
243                         os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
244                 except (IOError, OSError), err:
245                         self.trouble(u'ERROR: unable to rename file')
246
247         def try_utime(self, filename, last_modified_hdr):
248                 """Try to set the last-modified time of the given file."""
249                 if last_modified_hdr is None:
250                         return
251                 if not os.path.isfile(encodeFilename(filename)):
252                         return
253                 timestr = last_modified_hdr
254                 if timestr is None:
255                         return
256                 filetime = timeconvert(timestr)
257                 if filetime is None:
258                         return filetime
259                 try:
260                         os.utime(filename, (time.time(), filetime))
261                 except:
262                         pass
263                 return filetime
264
265         def report_writedescription(self, descfn):
266                 """ Report that the description file is being written """
267                 self.to_screen(u'[info] Writing video description to: ' + descfn)
268
269         def report_writesubtitles(self, srtfn):
270                 """ Report that the subtitles file is being written """
271                 self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
272
273         def report_writeinfojson(self, infofn):
274                 """ Report that the metadata file has been written """
275                 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
276
277         def report_destination(self, filename):
278                 """Report destination filename."""
279                 self.to_screen(u'[download] Destination: ' + filename)
280
281         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
282                 """Report download progress."""
283                 if self.params.get('noprogress', False):
284                         return
285                 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
286                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
287                 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
288                                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
289
290         def report_resuming_byte(self, resume_len):
291                 """Report attempt to resume at given byte."""
292                 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
293
294         def report_retry(self, count, retries):
295                 """Report retry in case of HTTP error 5xx"""
296                 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
297
298         def report_file_already_downloaded(self, file_name):
299                 """Report file has already been fully downloaded."""
300                 try:
301                         self.to_screen(u'[download] %s has already been downloaded' % file_name)
302                 except (UnicodeEncodeError), err:
303                         self.to_screen(u'[download] The file has already been downloaded')
304
305         def report_unable_to_resume(self):
306                 """Report it was impossible to resume download."""
307                 self.to_screen(u'[download] Unable to resume')
308
309         def report_finish(self):
310                 """Report download finished."""
311                 if self.params.get('noprogress', False):
312                         self.to_screen(u'[download] Download completed')
313                 else:
314                         self.to_screen(u'')
315
316         def increment_downloads(self):
317                 """Increment the ordinal that assigns a number to each file."""
318                 self._num_downloads += 1
319
320         def prepare_filename(self, info_dict):
321                 """Generate the output filename."""
322                 try:
323                         template_dict = dict(info_dict)
324                         template_dict['epoch'] = unicode(long(time.time()))
325                         template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
326                         filename = self.params['outtmpl'] % template_dict
327                         return filename
328                 except (ValueError, KeyError), err:
329                         self.trouble(u'ERROR: invalid system charset or erroneous output template')
330                         return None
331
332         def _match_entry(self, info_dict):
333                 """ Returns None iff the file should be downloaded """
334
335                 title = info_dict['title']
336                 matchtitle = self.params.get('matchtitle', False)
337                 if matchtitle and not re.search(matchtitle, title, re.IGNORECASE):
338                         return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
339                 rejecttitle = self.params.get('rejecttitle', False)
340                 if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE):
341                         return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
342                 return None
343
344         def process_info(self, info_dict):
345                 """Process a single dictionary returned by an InfoExtractor."""
346
347                 reason = self._match_entry(info_dict)
348                 if reason is not None:
349                         self.to_screen(u'[download] ' + reason)
350                         return
351
352                 max_downloads = self.params.get('max_downloads')
353                 if max_downloads is not None:
354                         if self._num_downloads > int(max_downloads):
355                                 raise MaxDownloadsReached()
356
357                 filename = self.prepare_filename(info_dict)
358                 
359                 # Forced printings
360                 if self.params.get('forcetitle', False):
361                         print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
362                 if self.params.get('forceurl', False):
363                         print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
364                 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
365                         print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
366                 if self.params.get('forcedescription', False) and 'description' in info_dict:
367                         print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
368                 if self.params.get('forcefilename', False) and filename is not None:
369                         print filename.encode(preferredencoding(), 'xmlcharrefreplace')
370                 if self.params.get('forceformat', False):
371                         print info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace')
372
373                 # Do nothing else if in simulate mode
374                 if self.params.get('simulate', False):
375                         return
376
377                 if filename is None:
378                         return
379
380                 try:
381                         dn = os.path.dirname(encodeFilename(filename))
382                         if dn != '' and not os.path.exists(dn): # dn is already encoded
383                                 os.makedirs(dn)
384                 except (OSError, IOError), err:
385                         self.trouble(u'ERROR: unable to create directory ' + unicode(err))
386                         return
387
388                 if self.params.get('writedescription', False):
389                         try:
390                                 descfn = filename + u'.description'
391                                 self.report_writedescription(descfn)
392                                 descfile = open(encodeFilename(descfn), 'wb')
393                                 try:
394                                         descfile.write(info_dict['description'].encode('utf-8'))
395                                 finally:
396                                         descfile.close()
397                         except (OSError, IOError):
398                                 self.trouble(u'ERROR: Cannot write description file ' + descfn)
399                                 return
400                                 
401                 if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
402                         # subtitles download errors are already managed as troubles in relevant IE
403                         # that way it will silently go on when used with unsupporting IE 
404                         try:
405                                 srtfn = filename.rsplit('.', 1)[0] + u'.srt'
406                                 self.report_writesubtitles(srtfn)
407                                 srtfile = open(encodeFilename(srtfn), 'wb')
408                                 try:
409                                         srtfile.write(info_dict['subtitles'].encode('utf-8'))
410                                 finally:
411                                         srtfile.close()
412                         except (OSError, IOError):
413                                 self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
414                                 return
415
416                 if self.params.get('writeinfojson', False):
417                         infofn = filename + u'.info.json'
418                         self.report_writeinfojson(infofn)
419                         try:
420                                 json.dump
421                         except (NameError,AttributeError):
422                                 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
423                                 return
424                         try:
425                                 infof = open(encodeFilename(infofn), 'wb')
426                                 try:
427                                         json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
428                                         json.dump(json_info_dict, infof)
429                                 finally:
430                                         infof.close()
431                         except (OSError, IOError):
432                                 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
433                                 return
434
435                 if not self.params.get('skip_download', False):
436                         if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
437                                 success = True
438                         else:
439                                 try:
440                                         success = self._do_download(filename, info_dict)
441                                 except (OSError, IOError), err:
442                                         raise UnavailableVideoError
443                                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
444                                         self.trouble(u'ERROR: unable to download video data: %s' % str(err))
445                                         return
446                                 except (ContentTooShortError, ), err:
447                                         self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
448                                         return
449         
450                         if success:
451                                 try:
452                                         self.post_process(filename, info_dict)
453                                 except (PostProcessingError), err:
454                                         self.trouble(u'ERROR: postprocessing: %s' % str(err))
455                                         return
456
457         def download(self, url_list):
458                 """Download a given list of URLs."""
459                 if len(url_list) > 1 and self.fixed_template():
460                         raise SameFileError(self.params['outtmpl'])
461
462                 for url in url_list:
463                         suitable_found = False
464                         for ie in self._ies:
465                                 # Go to next InfoExtractor if not suitable
466                                 if not ie.suitable(url):
467                                         continue
468
469                                 # Suitable InfoExtractor found
470                                 suitable_found = True
471
472                                 # Extract information from URL and process it
473                                 ie.extract(url)
474
475                                 # Suitable InfoExtractor had been found; go to next URL
476                                 break
477
478                         if not suitable_found:
479                                 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
480
481                 return self._download_retcode
482
483         def post_process(self, filename, ie_info):
484                 """Run the postprocessing chain on the given file."""
485                 info = dict(ie_info)
486                 info['filepath'] = filename
487                 for pp in self._pps:
488                         info = pp.run(info)
489                         if info is None:
490                                 break
491
492         def _download_with_rtmpdump(self, filename, url, player_url):
493                 self.report_destination(filename)
494                 tmpfilename = self.temp_name(filename)
495
496                 # Check for rtmpdump first
497                 try:
498                         subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
499                 except (OSError, IOError):
500                         self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
501                         return False
502
503                 # Download using rtmpdump. rtmpdump returns exit code 2 when
504                 # the connection was interrumpted and resuming appears to be
505                 # possible. This is part of rtmpdump's normal usage, AFAIK.
506                 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
507                 args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
508                 if self.params.get('verbose', False):
509                         try:
510                                 import pipes
511                                 shell_quote = lambda args: ' '.join(map(pipes.quote, args))
512                         except ImportError:
513                                 shell_quote = repr
514                         self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args))
515                 retval = subprocess.call(args)
516                 while retval == 2 or retval == 1:
517                         prevsize = os.path.getsize(encodeFilename(tmpfilename))
518                         self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
519                         time.sleep(5.0) # This seems to be needed
520                         retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
521                         cursize = os.path.getsize(encodeFilename(tmpfilename))
522                         if prevsize == cursize and retval == 1:
523                                 break
524                          # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
525                         if prevsize == cursize and retval == 2 and cursize > 1024:
526                                 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
527                                 retval = 0
528                                 break
529                 if retval == 0:
530                         self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename)))
531                         self.try_rename(tmpfilename, filename)
532                         return True
533                 else:
534                         self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
535                         return False
536
537         def _do_download(self, filename, info_dict):
538                 url = info_dict['url']
539                 player_url = info_dict.get('player_url', None)
540
541                 # Check file already present
542                 if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
543                         self.report_file_already_downloaded(filename)
544                         return True
545
546                 # Attempt to download using rtmpdump
547                 if url.startswith('rtmp'):
548                         return self._download_with_rtmpdump(filename, url, player_url)
549
550                 tmpfilename = self.temp_name(filename)
551                 stream = None
552
553                 # Do not include the Accept-Encoding header
554                 headers = {'Youtubedl-no-compression': 'True'}
555                 basic_request = urllib2.Request(url, None, headers)
556                 request = urllib2.Request(url, None, headers)
557
558                 # Establish possible resume length
559                 if os.path.isfile(encodeFilename(tmpfilename)):
560                         resume_len = os.path.getsize(encodeFilename(tmpfilename))
561                 else:
562                         resume_len = 0
563
564                 open_mode = 'wb'
565                 if resume_len != 0:
566                         if self.params.get('continuedl', False):
567                                 self.report_resuming_byte(resume_len)
568                                 request.add_header('Range','bytes=%d-' % resume_len)
569                                 open_mode = 'ab'
570                         else:
571                                 resume_len = 0
572
573                 count = 0
574                 retries = self.params.get('retries', 0)
575                 while count <= retries:
576                         # Establish connection
577                         try:
578                                 if count == 0 and 'urlhandle' in info_dict:
579                                         data = info_dict['urlhandle']
580                                 data = urllib2.urlopen(request)
581                                 break
582                         except (urllib2.HTTPError, ), err:
583                                 if (err.code < 500 or err.code >= 600) and err.code != 416:
584                                         # Unexpected HTTP error
585                                         raise
586                                 elif err.code == 416:
587                                         # Unable to resume (requested range not satisfiable)
588                                         try:
589                                                 # Open the connection again without the range header
590                                                 data = urllib2.urlopen(basic_request)
591                                                 content_length = data.info()['Content-Length']
592                                         except (urllib2.HTTPError, ), err:
593                                                 if err.code < 500 or err.code >= 600:
594                                                         raise
595                                         else:
596                                                 # Examine the reported length
597                                                 if (content_length is not None and
598                                                                 (resume_len - 100 < long(content_length) < resume_len + 100)):
599                                                         # The file had already been fully downloaded.
600                                                         # Explanation to the above condition: in issue #175 it was revealed that
601                                                         # YouTube sometimes adds or removes a few bytes from the end of the file,
602                                                         # changing the file size slightly and causing problems for some users. So
603                                                         # I decided to implement a suggested change and consider the file
604                                                         # completely downloaded if the file size differs less than 100 bytes from
605                                                         # the one in the hard drive.
606                                                         self.report_file_already_downloaded(filename)
607                                                         self.try_rename(tmpfilename, filename)
608                                                         return True
609                                                 else:
610                                                         # The length does not match, we start the download over
611                                                         self.report_unable_to_resume()
612                                                         open_mode = 'wb'
613                                                         break
614                         # Retry
615                         count += 1
616                         if count <= retries:
617                                 self.report_retry(count, retries)
618
619                 if count > retries:
620                         self.trouble(u'ERROR: giving up after %s retries' % retries)
621                         return False
622
623                 data_len = data.info().get('Content-length', None)
624                 if data_len is not None:
625                         data_len = long(data_len) + resume_len
626                 data_len_str = self.format_bytes(data_len)
627                 byte_counter = 0 + resume_len
628                 block_size = 1024
629                 start = time.time()
630                 while True:
631                         # Download and write
632                         before = time.time()
633                         data_block = data.read(block_size)
634                         after = time.time()
635                         if len(data_block) == 0:
636                                 break
637                         byte_counter += len(data_block)
638
639                         # Open file just in time
640                         if stream is None:
641                                 try:
642                                         (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
643                                         assert stream is not None
644                                         filename = self.undo_temp_name(tmpfilename)
645                                         self.report_destination(filename)
646                                 except (OSError, IOError), err:
647                                         self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
648                                         return False
649                         try:
650                                 stream.write(data_block)
651                         except (IOError, OSError), err:
652                                 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
653                                 return False
654                         block_size = self.best_block_size(after - before, len(data_block))
655
656                         # Progress message
657                         speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
658                         if data_len is None:
659                                 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
660                         else:
661                                 percent_str = self.calc_percent(byte_counter, data_len)
662                                 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
663                                 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
664
665                         # Apply rate limit
666                         self.slow_down(start, byte_counter - resume_len)
667
668                 if stream is None:
669                         self.trouble(u'\nERROR: Did not get any data blocks')
670                         return False
671                 stream.close()
672                 self.report_finish()
673                 if data_len is not None and byte_counter != data_len:
674                         raise ContentTooShortError(byte_counter, long(data_len))
675                 self.try_rename(tmpfilename, filename)
676
677                 # Update file modification time
678                 if self.params.get('updatetime', True):
679                         info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
680
681                 return True