Merge pull request #736 from rg3/retry
[youtube-dl] / youtube_dl / FileDownloader.py
index 164d25e54d44b261c16c3bf25e7336c8cb4048e6..d82aa2d83beb049bbcb783f11b5d3101d5b74b5b 100644 (file)
@@ -108,7 +108,7 @@ class FileDownloader(object):
         self.params = params
 
         if '%(stitle)s' in self.params['outtmpl']:
-            self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
+            self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
 
     @staticmethod
     def format_bytes(bytes):
@@ -231,13 +231,47 @@ class FileDownloader(object):
             self.to_stderr(message)
         if self.params.get('verbose'):
             if tb is None:
-                tb_data = traceback.format_list(traceback.extract_stack())
-                tb = u''.join(tb_data)
+                if sys.exc_info()[0]:  # if .trouble has been called from an except block
+                    tb = u''
+                    if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
+                        tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
+                    tb += compat_str(traceback.format_exc())
+                else:
+                    tb_data = traceback.format_list(traceback.extract_stack())
+                    tb = u''.join(tb_data)
             self.to_stderr(tb)
         if not self.params.get('ignoreerrors', False):
-            raise DownloadError(message)
+            if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
+                exc_info = sys.exc_info()[1].exc_info
+            else:
+                exc_info = sys.exc_info()
+            raise DownloadError(message, exc_info)
         self._download_retcode = 1
 
+    def report_warning(self, message):
+        '''
+        Print the message to stderr, it will be prefixed with 'WARNING:'
+        If stderr is a tty file the 'WARNING:' will be colored
+        '''
+        if sys.stderr.isatty():
+            _msg_header=u'\033[0;33mWARNING:\033[0m'
+        else:
+            _msg_header=u'WARNING:'
+        warning_message=u'%s %s' % (_msg_header,message)
+        self.to_stderr(warning_message)
+
+    def report_error(self, message, tb=None):
+        '''
+        Do the same as trouble, but prefixes the message with 'ERROR:', colored
+        in red if stderr is a tty file.
+        '''
+        if sys.stderr.isatty():
+            _msg_header = u'\033[0;31mERROR:\033[0m'
+        else:
+            _msg_header = u'ERROR:'
+        error_message = u'%s %s' % (_msg_header, message)
+        self.trouble(error_message, tb)
+
     def slow_down(self, start_time, byte_counter):
         """Sleep if the download speed is over the rate limit."""
         rate_limit = self.params.get('ratelimit', None)
@@ -269,7 +303,7 @@ class FileDownloader(object):
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
-            self.trouble(u'ERROR: unable to rename file')
+            self.report_error(u'unable to rename file')
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
@@ -364,8 +398,11 @@ class FileDownloader(object):
 
             filename = self.params['outtmpl'] % template_dict
             return filename
-        except (ValueError, KeyError) as err:
-            self.trouble(u'ERROR: invalid system charset or erroneous output template')
+        except KeyError as err:
+            self.trouble(u'ERROR: Erroneous output template')
+            return None
+        except ValueError as err:
+            self.trouble(u'ERROR: Insufficient system charset ' + repr(preferredencoding()))
             return None
 
     def _match_entry(self, info_dict):
@@ -429,7 +466,7 @@ class FileDownloader(object):
             if dn != '' and not os.path.exists(dn): # dn is already encoded
                 os.makedirs(dn)
         except (OSError, IOError) as err:
-            self.trouble(u'ERROR: unable to create directory ' + compat_str(err))
+            self.report_error(u'unable to create directory ' + compat_str(err))
             return
 
         if self.params.get('writedescription', False):
@@ -439,7 +476,7 @@ class FileDownloader(object):
                 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
                     descfile.write(info_dict['description'])
             except (OSError, IOError):
-                self.trouble(u'ERROR: Cannot write description file ' + descfn)
+                self.report_error(u'Cannot write description file ' + descfn)
                 return
 
         if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
@@ -454,7 +491,7 @@ class FileDownloader(object):
                 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
                     subfile.write(sub)
             except (OSError, IOError):
-                self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
+                self.report_error(u'Cannot write subtitles file ' + descfn)
                 return
             if self.params.get('onlysubtitles', False):
                 return 
@@ -482,7 +519,7 @@ class FileDownloader(object):
                 json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
                 write_json_file(json_info_dict, encodeFilename(infofn))
             except (OSError, IOError):
-                self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
+                self.report_error(u'Cannot write metadata to JSON file ' + infofn)
                 return
 
         if not self.params.get('skip_download', False):
@@ -494,17 +531,17 @@ class FileDownloader(object):
                 except (OSError, IOError) as err:
                     raise UnavailableVideoError()
                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                    self.trouble(u'ERROR: unable to download video data: %s' % str(err))
+                    self.report_error(u'unable to download video data: %s' % str(err))
                     return
                 except (ContentTooShortError, ) as err:
-                    self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+                    self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
                     return
 
             if success:
                 try:
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
-                    self.trouble(u'ERROR: postprocessing: %s' % str(err))
+                    self.report_error(u'postprocessing: %s' % str(err))
                     return
 
     def download(self, url_list):
@@ -521,8 +558,8 @@ class FileDownloader(object):
 
                 # Warn if the _WORKING attribute is False
                 if not ie.working():
-                    self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, '
-                                   u'and will probably not work. If you want to go on, use the -i option.')
+                    self.report_warning(u'the program functionality for this site has been marked as broken, '
+                                        u'and will probably not work. If you want to go on, use the -i option.')
 
                 # Suitable InfoExtractor found
                 suitable_found = True
@@ -533,9 +570,12 @@ class FileDownloader(object):
                 except ExtractorError as de: # An error we somewhat expected
                     self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback())
                     break
+                except MaxDownloadsReached:
+                    self.to_screen(u'[info] Maximum number of downloaded files reached.')
+                    raise
                 except Exception as e:
                     if self.params.get('ignoreerrors', False):
-                        self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc()))
+                        self.report_error(u'' + compat_str(e), tb=compat_str(traceback.format_exc()))
                         break
                     else:
                         raise
@@ -549,13 +589,14 @@ class FileDownloader(object):
                         self.increment_downloads()
                         self.process_info(video)
                     except UnavailableVideoError:
-                        self.trouble(u'\nERROR: unable to download video')
+                        self.to_stderr(u"\n")
+                        self.report_error(u'unable to download video')
 
                 # Suitable InfoExtractor had been found; go to next URL
                 break
 
             if not suitable_found:
-                self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
+                self.report_error(u'no suitable InfoExtractor: %s' % url)
 
         return self._download_retcode
 
@@ -577,10 +618,10 @@ class FileDownloader(object):
                 self.to_stderr(u'ERROR: ' + e.msg)
         if keep_video is False and not self.params.get('keepvideo', False):
             try:
-                self.to_stderr(u'Deleting original file %s (pass -k to keep)' % filename)
+                self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename)
                 os.remove(encodeFilename(filename))
             except (IOError, OSError):
-                self.to_stderr(u'WARNING: Unable to remove downloaded video file')
+                self.report_warning(u'Unable to remove downloaded video file')
 
     def _download_with_rtmpdump(self, filename, url, player_url, page_url):
         self.report_destination(filename)
@@ -588,9 +629,9 @@ class FileDownloader(object):
 
         # Check for rtmpdump first
         try:
-            subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+            subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
         except (OSError, IOError):
-            self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
+            self.report_error(u'RTMP download detected but "rtmpdump" could not be run')
             return False
 
         # Download using rtmpdump. rtmpdump returns exit code 2 when
@@ -635,7 +676,8 @@ class FileDownloader(object):
             })
             return True
         else:
-            self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
+            self.to_stderr(u"\n")
+            self.report_error(u'rtmpdump exited with code %d' % retval)
             return False
 
     def _do_download(self, filename, info_dict):
@@ -735,7 +777,7 @@ class FileDownloader(object):
                 self.report_retry(count, retries)
 
         if count > retries:
-            self.trouble(u'ERROR: giving up after %s retries' % retries)
+            self.report_error(u'giving up after %s retries' % retries)
             return False
 
         data_len = data.info().get('Content-length', None)
@@ -771,12 +813,13 @@ class FileDownloader(object):
                     filename = self.undo_temp_name(tmpfilename)
                     self.report_destination(filename)
                 except (OSError, IOError) as err:
-                    self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
+                    self.report_error(u'unable to open for writing: %s' % str(err))
                     return False
             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:
-                self.trouble(u'\nERROR: unable to write data: %s' % str(err))
+                self.to_stderr(u"\n")
+                self.report_error(u'unable to write data: %s' % str(err))
                 return False
             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))
@@ -802,7 +845,8 @@ class FileDownloader(object):
             self.slow_down(start, byte_counter - resume_len)
 
         if stream is None:
-            self.trouble(u'\nERROR: Did not get any data blocks')
+            self.to_stderr(u"\n")
+            self.report_error(u'Did not get any data blocks')
             return False
         stream.close()
         self.report_finish()