Merge pull request #736 from rg3/retry
authorFilippo Valsorda <filippo.valsorda@gmail.com>
Fri, 29 Mar 2013 16:01:27 +0000 (09:01 -0700)
committerFilippo Valsorda <filippo.valsorda@gmail.com>
Fri, 29 Mar 2013 16:01:27 +0000 (09:01 -0700)
Exception stacking and test retry

1  2 
youtube_dl/FileDownloader.py
youtube_dl/utils.py

index 5087b4cc824db2d7857407f3215cd0baaede9838,a13a5f9d79cab1a0cf7da5c5ac6a640ad26016cb..d82aa2d83beb049bbcb783f11b5d3101d5b74b5b
@@@ -78,11 -78,7 +78,11 @@@ class FileDownloader(object)
      updatetime:        Use the Last-modified header to set output file timestamps.
      writedescription:  Write the video description to a .description file
      writeinfojson:     Write the video description to a .info.json file
 -    writesubtitles:    Write the video subtitles to a .srt file
 +    writesubtitles:    Write the video subtitles to a file
 +    onlysubtitles:     Downloads only the subtitles of the video
 +    allsubtitles:      Downloads all the subtitles of the video
 +    listsubtitles:     Lists all available subtitles for the video
 +    subtitlesformat:   Subtitle format [sbv/srt] (default=srt)
      subtitleslang:     Language of the subtitles to download
      test:              Download only first bytes to test the downloader.
      keepvideo:         Keep the video file after post-processing
              self.to_stderr(message)
          if self.params.get('verbose'):
              if tb is None:
-                 tb_data = traceback.format_list(traceback.extract_stack())
-                 tb = u''.join(tb_data)
+                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
+                     tb = u''
+                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
+                         tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
+                     tb += compat_str(traceback.format_exc())
+                 else:
+                     tb_data = traceback.format_list(traceback.extract_stack())
+                     tb = u''.join(tb_data)
              self.to_stderr(tb)
          if not self.params.get('ignoreerrors', False):
-             raise DownloadError(message)
+             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
+                 exc_info = sys.exc_info()[1].exc_info
+             else:
+                 exc_info = sys.exc_info()
+             raise DownloadError(message, exc_info)
          self._download_retcode = 1
  
      def report_warning(self, message):
          warning_message=u'%s %s' % (_msg_header,message)
          self.to_stderr(warning_message)
  
 +    def report_error(self, message, tb=None):
 +        '''
 +        Do the same as trouble, but prefixes the message with 'ERROR:', colored
 +        in red if stderr is a tty file.
 +        '''
 +        if sys.stderr.isatty():
 +            _msg_header = u'\033[0;31mERROR:\033[0m'
 +        else:
 +            _msg_header = u'ERROR:'
 +        error_message = u'%s %s' % (_msg_header, message)
 +        self.trouble(error_message, tb)
 +
      def slow_down(self, start_time, byte_counter):
          """Sleep if the download speed is over the rate limit."""
          rate_limit = self.params.get('ratelimit', None)
                  return
              os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
          except (IOError, OSError) as err:
 -            self.trouble(u'ERROR: unable to rename file')
 +            self.report_error(u'unable to rename file')
  
      def try_utime(self, filename, last_modified_hdr):
          """Try to set the last-modified time of the given file."""
          """ Report that the description file is being written """
          self.to_screen(u'[info] Writing video description to: ' + descfn)
  
 -    def report_writesubtitles(self, srtfn):
 +    def report_writesubtitles(self, sub_filename):
          """ Report that the subtitles file is being written """
 -        self.to_screen(u'[info] Writing video subtitles to: ' + srtfn)
 +        self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename)
  
      def report_writeinfojson(self, infofn):
          """ Report that the metadata file has been written """
              if dn != '' and not os.path.exists(dn): # dn is already encoded
                  os.makedirs(dn)
          except (OSError, IOError) as err:
 -            self.trouble(u'ERROR: unable to create directory ' + compat_str(err))
 +            self.report_error(u'unable to create directory ' + compat_str(err))
              return
  
          if self.params.get('writedescription', False):
                  with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
                      descfile.write(info_dict['description'])
              except (OSError, IOError):
 -                self.trouble(u'ERROR: Cannot write description file ' + descfn)
 +                self.report_error(u'Cannot write description file ' + descfn)
                  return
  
          if self.params.get('writesubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
              # subtitles download errors are already managed as troubles in relevant IE
              # that way it will silently go on when used with unsupporting IE
 +            subtitle = info_dict['subtitles'][0]
 +            (sub_error, sub_lang, sub) = subtitle
 +            sub_format = self.params.get('subtitlesformat')
              try:
 -                srtfn = filename.rsplit('.', 1)[0] + u'.srt'
 -                self.report_writesubtitles(srtfn)
 -                with io.open(encodeFilename(srtfn), 'w', encoding='utf-8') as srtfile:
 -                    srtfile.write(info_dict['subtitles'])
 +                sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
 +                self.report_writesubtitles(sub_filename)
 +                with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
 +                    subfile.write(sub)
              except (OSError, IOError):
 -                self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
 +                self.report_error(u'Cannot write subtitles file ' + descfn)
                  return
 +            if self.params.get('onlysubtitles', False):
 +                return 
 +
 +        if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
 +            subtitles = info_dict['subtitles']
 +            sub_format = self.params.get('subtitlesformat')
 +            for subtitle in subtitles:
 +                (sub_error, sub_lang, sub) = subtitle
 +                try:
 +                    sub_filename = filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
 +                    self.report_writesubtitles(sub_filename)
 +                    with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
 +                            subfile.write(sub)
 +                except (OSError, IOError):
 +                    self.trouble(u'ERROR: Cannot write subtitles file ' + descfn)
 +                    return
 +            if self.params.get('onlysubtitles', False):
 +                return 
  
          if self.params.get('writeinfojson', False):
              infofn = filename + u'.info.json'
                  json_info_dict = dict((k, v) for k,v in info_dict.items() if not k in ['urlhandle'])
                  write_json_file(json_info_dict, encodeFilename(infofn))
              except (OSError, IOError):
 -                self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
 +                self.report_error(u'Cannot write metadata to JSON file ' + infofn)
                  return
  
          if not self.params.get('skip_download', False):
                  except (OSError, IOError) as err:
                      raise UnavailableVideoError()
                  except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 -                    self.trouble(u'ERROR: unable to download video data: %s' % str(err))
 +                    self.report_error(u'unable to download video data: %s' % str(err))
                      return
                  except (ContentTooShortError, ) as err:
 -                    self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 +                    self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
                      return
  
              if success:
                  try:
                      self.post_process(filename, info_dict)
                  except (PostProcessingError) as err:
 -                    self.trouble(u'ERROR: postprocessing: %s' % str(err))
 +                    self.report_error(u'postprocessing: %s' % str(err))
                      return
  
      def download(self, url_list):
                  except ExtractorError as de: # An error we somewhat expected
                      self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback())
                      break
 +                except MaxDownloadsReached:
 +                    self.to_screen(u'[info] Maximum number of downloaded files reached.')
 +                    raise
                  except Exception as e:
                      if self.params.get('ignoreerrors', False):
 -                        self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc()))
 +                        self.report_error(u'' + compat_str(e), tb=compat_str(traceback.format_exc()))
                          break
                      else:
                          raise
                          self.increment_downloads()
                          self.process_info(video)
                      except UnavailableVideoError:
 -                        self.trouble(u'\nERROR: unable to download video')
 +                        self.to_stderr(u"\n")
 +                        self.report_error(u'unable to download video')
  
                  # Suitable InfoExtractor had been found; go to next URL
                  break
  
              if not suitable_found:
 -                self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
 +                self.report_error(u'no suitable InfoExtractor: %s' % url)
  
          return self._download_retcode
  
          try:
              subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
          except (OSError, IOError):
 -            self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
 +            self.report_error(u'RTMP download detected but "rtmpdump" could not be run')
              return False
  
          # Download using rtmpdump. rtmpdump returns exit code 2 when
              })
              return True
          else:
 -            self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
 +            self.to_stderr(u"\n")
 +            self.report_error(u'rtmpdump exited with code %d' % retval)
              return False
  
      def _do_download(self, filename, info_dict):
                  self.report_retry(count, retries)
  
          if count > retries:
 -            self.trouble(u'ERROR: giving up after %s retries' % retries)
 +            self.report_error(u'giving up after %s retries' % retries)
              return False
  
          data_len = data.info().get('Content-length', None)
                      filename = self.undo_temp_name(tmpfilename)
                      self.report_destination(filename)
                  except (OSError, IOError) as err:
 -                    self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
 +                    self.report_error(u'unable to open for writing: %s' % str(err))
                      return False
              try:
                  stream.write(data_block)
              except (IOError, OSError) as err:
 -                self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 +                self.to_stderr(u"\n")
 +                self.report_error(u'unable to write data: %s' % str(err))
                  return False
              if not self.params.get('noresizebuffer', False):
                  block_size = self.best_block_size(after - before, len(data_block))
              self.slow_down(start, byte_counter - resume_len)
  
          if stream is None:
 -            self.trouble(u'\nERROR: Did not get any data blocks')
 +            self.to_stderr(u"\n")
 +            self.report_error(u'Did not get any data blocks')
              return False
          stream.close()
          self.report_finish()
diff --combined youtube_dl/utils.py
index d366c417360a70678dac87a0567a3d96f3a97c74,88d4ece13a740cdd7239020af872018189e79f6b..017f06c42e9a019e18e25480c5e5d8d3aaaef335
@@@ -311,7 -311,7 +311,7 @@@ def clean_html(html)
      html = re.sub('<.*?>', '', html)
      # Replace html entities
      html = unescapeHTML(html)
 -    return html
 +    return html.strip()
  
  
  def sanitize_open(filename, open_mode):
              if sys.platform == 'win32':
                  import msvcrt
                  msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
 -            return (sys.stdout, filename)
 +            return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
          stream = open(encodeFilename(filename), open_mode)
          return (stream, filename)
      except (IOError, OSError) as err:
@@@ -435,6 -435,7 +435,7 @@@ class ExtractorError(Exception)
          """ tb, if given, is the original traceback (so that it can be printed out). """
          super(ExtractorError, self).__init__(msg)
          self.traceback = tb
+         self.exc_info = sys.exc_info()  # preserve original exception
  
      def format_traceback(self):
          if self.traceback is None:
@@@ -449,7 -450,10 +450,10 @@@ class DownloadError(Exception)
      configured to continue on errors. They will contain the appropriate
      error message.
      """
-     pass
+     def __init__(self, msg, exc_info=None):
+         """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
+         super(DownloadError, self).__init__(msg)
+         self.exc_info = exc_info
  
  
  class SameFileError(Exception):