Handle "content too short" errors properly
[youtube-dl] / youtube-dl
index d3acd9d0a86205776b8bc2bf9cce87ca8c32718e..f6e472445d8e0d9512805487a04afc88bcc68662 100755 (executable)
@@ -58,6 +58,22 @@ class UnavailableFormatError(Exception):
        This exception will be thrown when a video is requested
        in a format that is not available for that video.
        """
+       pass
+
+class ContentTooShortError(Exception):
+       """Content Too Short exception.
+
+       This exception may be raised by FileDownloader objects when a file they
+       download is too small for what the server announced first, indicating
+       the connection was probably interrupted.
+       """
+       # Both in bytes
+       downloaded = None
+       expected = None
+
+       def __init__(self, downloaded, expected):
+               self.downloaded = downloaded
+               self.expected = expected
 
 class FileDownloader(object):
        """File Downloader class.
@@ -260,7 +276,9 @@ class FileDownloader(object):
                        return
 
                try:
-                       filename = self.params['outtmpl'] % info_dict
+                       template_dict = dict(info_dict)
+                       template_dict['epoch'] = unicode(long(time.time()))
+                       filename = self.params['outtmpl'] % template_dict
                        self.report_destination(filename)
                except (ValueError, KeyError), err:
                        self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
@@ -284,14 +302,15 @@ class FileDownloader(object):
                        self._do_download(outstream, info_dict['url'])
                        outstream.close()
                except (OSError, IOError), err:
-                       if info_dict['best_quality']:
-                               raise UnavailableFormatError
-                       else:
-                               self.trouble('ERROR: unable to write video data: %s' % str(err))
-                               return
+                       outstream.close()
+                       os.remove(filename)
+                       raise UnavailableFormatError
                except (urllib2.URLError, httplib.HTTPException, socket.error), err:
                        self.trouble('ERROR: unable to download video data: %s' % str(err))
                        return
+               except (ContentTooShortError, ), err:
+                       self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+                       return
 
                try:
                        self.post_process(filename, info_dict)
@@ -365,7 +384,7 @@ class FileDownloader(object):
 
                self.report_finish()
                if data_len is not None and str(byte_counter) != data_len:
-                       raise ValueError('Content too short: %s/%s bytes' % (byte_counter, data_len))
+                       raise ContentTooShortError(byte_counter, long(data_len))
 
 class InfoExtractor(object):
        """Information Extractor class.
@@ -436,7 +455,7 @@ class YoutubeIE(InfoExtractor):
        _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
        _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
        _NETRC_MACHINE = 'youtube'
-       _available_formats = ['22', '18', '17', '13'] # listed in order of priority for -b flag
+       _available_formats = ['22', '35', '18', '17', '13'] # listed in order of priority for -b flag
        _video_extensions = {
                '13': '3gp',
                '17': 'mp4',
@@ -643,21 +662,25 @@ class YoutubeIE(InfoExtractor):
                                        'title':        video_title,
                                        'stitle':       simple_title,
                                        'ext':          video_extension.decode('utf-8'),
-                                       'best_quality': best_quality,
                                })
 
                                return
 
-                       except UnavailableFormatError:
-                               if quality_index == len(self._available_formats) - 1:
-                                       # I don't ever expect this to happen
-                                       self._downloader.trouble(u'ERROR: no known formats available for video')
+                       except UnavailableFormatError, err:
+                               if best_quality:
+                                       if quality_index == len(self._available_formats) - 1:
+                                               # I don't ever expect this to happen
+                                               self._downloader.trouble(u'ERROR: no known formats available for video')
+                                               return
+                                       else:
+                                               self.report_unavailable_format(video_id, format_param)
+                                               quality_index += 1
+                                               format_param = self._available_formats[quality_index]
+                                               continue
+                               else: 
+                                       self._downloader.trouble('ERROR: format not available for video')
                                        return
-                               else:
-                                       self.report_unavailable_format(video_id, format_param)
-                                       quality_index += 1
-                                       format_param = self._available_formats[quality_index]
-                                       continue
+
 
 class MetacafeIE(InfoExtractor):
        """Information Extractor for metacafe.com."""
@@ -769,16 +792,18 @@ class MetacafeIE(InfoExtractor):
                        return
                video_uploader = mobj.group(1)
 
-               # Process video information
-               self._downloader.process_info({
-                       'id':           video_id.decode('utf-8'),
-                       'url':          video_url.decode('utf-8'),
-                       'uploader':     video_uploader.decode('utf-8'),
-                       'title':        video_title,
-                       'stitle':       simple_title,
-                       'ext':          video_extension.decode('utf-8'),
-                       'best_quality': False, # TODO
-               })
+               try:
+                       # Process video information
+                       self._downloader.process_info({
+                               'id':           video_id.decode('utf-8'),
+                               'url':          video_url.decode('utf-8'),
+                               'uploader':     video_uploader.decode('utf-8'),
+                               'title':        video_title,
+                               'stitle':       simple_title,
+                               'ext':          video_extension.decode('utf-8'),
+                       })
+               except UnavailableFormatError:
+                       self._downloader.trouble(u'ERROR: format not available for video')
 
 
 class YoutubeSearchIE(InfoExtractor):
@@ -1015,7 +1040,7 @@ if __name__ == '__main__':
 
                video_format = optparse.OptionGroup(parser, 'Video Format Options')
                video_format.add_option('-f', '--format',
-                               dest='format', metavar='FMT', help='video format code')
+                               action='append', dest='format', metavar='FMT', help='video format code')
                video_format.add_option('-b', '--best-quality',
                                action='store_const', dest='format', help='download the best quality video possible', const='0')
                video_format.add_option('-m', '--mobile-version',
@@ -1033,13 +1058,13 @@ if __name__ == '__main__':
                                action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
                verbosity.add_option('-e', '--get-title',
                                action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
-               verbosity.add_option('-t', '--title',
-                               action='store_true', dest='usetitle', help='use title in file name', default=False)
-               verbosity.add_option('-l', '--literal',
-                               action='store_true', dest='useliteral', help='use literal title in file name', default=False)
                parser.add_option_group(verbosity)
 
                filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
+               filesystem.add_option('-t', '--title',
+                               action='store_true', dest='usetitle', help='use title in file name', default=False)
+               filesystem.add_option('-l', '--literal',
+                               action='store_true', dest='useliteral', help='use literal title in file name', default=False)
                filesystem.add_option('-o', '--output',
                                dest='outtmpl', metavar='TPL', help='output filename template')
                filesystem.add_option('-a', '--batch-file',
@@ -1063,22 +1088,29 @@ if __name__ == '__main__':
 
                # Conflicting, missing and erroneous options
                if len(all_urls) < 1:
-                       sys.exit(u'ERROR: you must provide at least one URL')
+                       parser.error(u'you must provide at least one URL')
                if opts.usenetrc and (opts.username is not None or opts.password is not None):
-                       sys.exit(u'ERROR: using .netrc conflicts with giving username/password')
+                       parser.error(u'using .netrc conflicts with giving username/password')
                if opts.password is not None and opts.username is None:
-                       sys.exit(u'ERROR: account username missing')
+                       parser.error(u'account username missing')
                if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
-                       sys.exit(u'ERROR: using output template conflicts with using title or literal title')
+                       parser.error(u'using output template conflicts with using title or literal title')
                if opts.usetitle and opts.useliteral:
-                       sys.exit(u'ERROR: using title conflicts with using literal title')
+                       parser.error(u'using title conflicts with using literal title')
                if opts.username is not None and opts.password is None:
                        opts.password = getpass.getpass(u'Type account password and press return:')
                if opts.ratelimit is not None:
                        numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
                        if numeric_limit is None:
-                               sys.exit(u'ERROR: invalid rate limit specified')
+                               parser.error(u'invalid rate limit specified')
                        opts.ratelimit = numeric_limit
+               if opts.format is not None and len(opts.format) > 1:
+                       parser.error(u'pass at most one of the video format option flags (-f, -b, -m, -d)')
+               if opts.format is None:
+                       real_format = None
+               else:
+                       real_format = opts.format[0]
+
 
                # Information extractors
                youtube_ie = YoutubeIE()
@@ -1095,7 +1127,7 @@ if __name__ == '__main__':
                        'forceurl': opts.geturl,
                        'forcetitle': opts.gettitle,
                        'simulate': (opts.simulate or opts.geturl or opts.gettitle),
-                       'format': opts.format,
+                       'format': real_format,
                        'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(locale.getpreferredencoding()))
                                or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
                                or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')