Merge remote-tracking branch 'Asido/master'
[youtube-dl] / youtube_dl / FileDownloader.py
index 4449fe711f28fbcfc427b5b23719d3e5cbdf045c..d7d5b1521ec72ea46782064f3c6fadd3d5032cb7 100644 (file)
@@ -44,37 +44,40 @@ class FileDownloader(object):
 
        Available options:
 
-       username:         Username for authentication purposes.
-       password:         Password for authentication purposes.
-       usenetrc:         Use netrc for authentication instead.
-       quiet:            Do not print messages to stdout.
-       forceurl:         Force printing final URL.
-       forcetitle:       Force printing title.
-       forcethumbnail:   Force printing thumbnail URL.
-       forcedescription: Force printing description.
-       forcefilename:    Force printing final filename.
-       simulate:         Do not download the video files.
-       format:           Video format code.
-       format_limit:     Highest quality format to try.
-       outtmpl:          Template for output names.
-       ignoreerrors:     Do not stop on download errors.
-       ratelimit:        Download speed limit, in bytes/sec.
-       nooverwrites:     Prevent overwriting files.
-       retries:          Number of times to retry for HTTP error 5xx
-       continuedl:       Try to continue downloads if possible.
-       noprogress:       Do not print the progress bar.
-       playliststart:    Playlist item to start at.
-       playlistend:      Playlist item to end at.
-       matchtitle:       Download only matching titles.
-       rejecttitle:      Reject downloads for matching titles.
-       logtostderr:      Log messages to stderr instead of stdout.
-       consoletitle:     Display progress in console window's titlebar.
-       nopart:           Do not use temporary .part files.
-       updatetime:       Use the Last-modified header to set output file timestamps.
-       writedescription: Write the video description to a .description file
-       writeinfojson:    Write the video description to a .info.json file
-       writesubtitles:   Write the video subtitles to a .srt file
-       subtitleslang:    Language of the subtitles to download
+       username:          Username for authentication purposes.
+       password:          Password for authentication purposes.
+       usenetrc:          Use netrc for authentication instead.
+       quiet:             Do not print messages to stdout.
+       forceurl:          Force printing final URL.
+       forcetitle:        Force printing title.
+       forcethumbnail:    Force printing thumbnail URL.
+       forcedescription:  Force printing description.
+       forcefilename:     Force printing final filename.
+       simulate:          Do not download the video files.
+       format:            Video format code.
+       format_limit:      Highest quality format to try.
+       outtmpl:           Template for output names.
+       restrictfilenames: Do not allow "&" and spaces in file names
+       ignoreerrors:      Do not stop on download errors.
+       ratelimit:         Download speed limit, in bytes/sec.
+       nooverwrites:      Prevent overwriting files.
+       retries:           Number of times to retry for HTTP error 5xx
+       buffersize:        Size of download buffer in bytes.
+       noresizebuffer:    Do not automatically resize the download buffer.
+       continuedl:        Try to continue downloads if possible.
+       noprogress:        Do not print the progress bar.
+       playliststart:     Playlist item to start at.
+       playlistend:       Playlist item to end at.
+       matchtitle:        Download only matching titles.
+       rejecttitle:       Reject downloads for matching titles.
+       logtostderr:       Log messages to stderr instead of stdout.
+       consoletitle:      Display progress in console window's titlebar.
+       nopart:            Do not use temporary .part files.
+       updatetime:        Use the Last-modified header to set output file timestamps.
+       writedescription:  Write the video description to a .description file
+       writeinfojson:     Write the video description to a .info.json file
+       writesubtitles:    Write the video subtitles to a .srt file
+       subtitleslang:     Language of the subtitles to download
        """
 
        params = None
@@ -93,6 +96,9 @@ class FileDownloader(object):
                self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
                self.params = params
 
+               if '%(stitle)s' in self.params['outtmpl']:
+                       self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
+
        @staticmethod
        def format_bytes(bytes):
                if bytes is None:
@@ -139,23 +145,23 @@ class FileDownloader(object):
                new_min = max(bytes / 2.0, 1.0)
                new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
                if elapsed_time < 0.001:
-                       return long(new_max)
+                       return int(new_max)
                rate = bytes / elapsed_time
                if rate > new_max:
-                       return long(new_max)
+                       return int(new_max)
                if rate < new_min:
-                       return long(new_min)
-               return long(rate)
+                       return int(new_min)
+               return int(rate)
 
        @staticmethod
        def parse_bytes(bytestr):
-               """Parse a string indicating a byte quantity into a long integer."""
+               """Parse a string indicating a byte quantity into an integer."""
                matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
                if matchobj is None:
                        return None
                number = float(matchobj.group(1))
                multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
-               return long(round(number * multiplier))
+               return int(round(number * multiplier))
 
        def add_info_extractor(self, ie):
                """Add an InfoExtractor object to the end of the list."""
@@ -180,7 +186,8 @@ class FileDownloader(object):
 
        def to_stderr(self, message):
                """Print message to stderr."""
-               print >>sys.stderr, message.encode(preferredencoding())
+               assert type(message) == type(u'')
+               sys.stderr.write((message + u'\n').encode(preferredencoding()))
 
        def to_cons_title(self, message):
                """Set console/terminal window title to message."""
@@ -320,8 +327,10 @@ class FileDownloader(object):
                """Generate the output filename."""
                try:
                        template_dict = dict(info_dict)
-                       template_dict['epoch'] = unicode(long(time.time()))
-                       template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
+                       template_dict['epoch'] = int(time.time())
+                       template_dict['autonumber'] = u'%05d' % self._num_downloads
+
+                       template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
                        filename = self.params['outtmpl'] % template_dict
                        return filename
                except (ValueError, KeyError), err:
@@ -347,7 +356,8 @@ class FileDownloader(object):
        def process_info(self, info_dict):
                """Process a single dictionary returned by an InfoExtractor."""
 
-               info_dict['stitle'] = sanitize_filename(info_dict['title'])
+               # Keep for backwards compatibility
+               info_dict['stitle'] = info_dict['title']
 
                reason = self._match_entry(info_dict)
                if reason is not None:
@@ -363,17 +373,17 @@ class FileDownloader(object):
 
                # Forced printings
                if self.params.get('forcetitle', False):
-                       print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
+                       print(info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace'))
                if self.params.get('forceurl', False):
-                       print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
+                       print(info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace'))
                if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
-                       print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
+                       print(info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace'))
                if self.params.get('forcedescription', False) and 'description' in info_dict:
-                       print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
+                       print(info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace'))
                if self.params.get('forcefilename', False) and filename is not None:
-                       print filename.encode(preferredencoding(), 'xmlcharrefreplace')
+                       print(filename.encode(preferredencoding(), 'xmlcharrefreplace'))
                if self.params.get('forceformat', False):
-                       print info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace')
+                       print(info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace'))
 
                # Do nothing else if in simulate mode
                if self.params.get('simulate', False):
@@ -637,7 +647,7 @@ class FileDownloader(object):
                        data_len = long(data_len) + resume_len
                data_len_str = self.format_bytes(data_len)
                byte_counter = 0 + resume_len
-               block_size = 1024
+               block_size = self.params.get('buffersize', 1024)
                start = time.time()
                while True:
                        # Download and write
@@ -663,7 +673,8 @@ class FileDownloader(object):
                        except (IOError, OSError), err:
                                self.trouble(u'\nERROR: unable to write data: %s' % str(err))
                                return False
-                       block_size = self.best_block_size(after - before, len(data_block))
+                       if not self.params.get('noresizebuffer', False):
+                               block_size = self.best_block_size(after - before, len(data_block))
 
                        # Progress message
                        speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)