[YoutubeDL] Check if merger can actually merge

[youtube-dl] / youtube_dl / YoutubeDL.py

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 088b111eb835f684bf934b4ba9a2334a9822f43a..00f86b342dfd6bbf14c609353d3c4493719db738 100755 (executable)
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -4,8 +4,10 @@
 from __future__ import absolute_import, unicode_literals
 
 import collections
 from __future__ import absolute_import, unicode_literals
 
 import collections

+import contextlib

 import datetime
 import errno
 import datetime
 import errno

+import fileinput

 import io
 import itertools
 import json
 import io
 import itertools
 import json


@@ -28,6 +30,7 @@ from .compat import (
     compat_basestring,
     compat_cookiejar,
     compat_expanduser,
     compat_basestring,
     compat_cookiejar,
     compat_expanduser,

+    compat_get_terminal_size,

     compat_http_client,
     compat_kwargs,
     compat_str,
     compat_http_client,
     compat_kwargs,
     compat_str,


@@ -46,21 +49,21 @@ from .utils import (
     ExtractorError,
     format_bytes,
     formatSeconds,
     ExtractorError,
     format_bytes,
     formatSeconds,

-    get_term_width,

     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,
     PagedList,
     parse_filesize,
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,
     PagedList,
     parse_filesize,

+    PerRequestProxyHandler,

     PostProcessingError,
     platform_name,
     preferredencoding,
     render_table,
     SameFileError,
     sanitize_filename,
     PostProcessingError,
     platform_name,
     preferredencoding,
     render_table,
     SameFileError,
     sanitize_filename,

+    sanitize_path,

     std_headers,
     subtitles_filename,
     std_headers,
     subtitles_filename,

-    takewhile_inclusive,

     UnavailableVideoError,
     url_basename,
     version_tuple,
     UnavailableVideoError,
     url_basename,
     version_tuple,


@@ -68,6 +71,7 @@ from .utils import (
     write_string,
     YoutubeDLHandler,
     prepend_extension,
     write_string,
     YoutubeDLHandler,
     prepend_extension,

+    replace_extension,

     args_to_str,
     age_restricted,
 )
     args_to_str,
     age_restricted,
 )


@@ -131,7 +135,6 @@ class YoutubeDL(object):
                        (or video) as a single JSON line.
     simulate:          Do not download the video files.
     format:            Video format code. See options.py for more information.
                        (or video) as a single JSON line.
     simulate:          Do not download the video files.
     format:            Video format code. See options.py for more information.

-    format_limit:      Highest quality format to try.

     outtmpl:           Template for output names.
     restrictfilenames: Do not allow "&" and spaces in file names
     ignoreerrors:      Do not stop on download errors.
     outtmpl:           Template for output names.
     restrictfilenames: Do not allow "&" and spaces in file names
     ignoreerrors:      Do not stop on download errors.


@@ -181,6 +184,8 @@ class YoutubeDL(object):
     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
                        At the moment, this is only supported by YouTube.
     proxy:             URL of the proxy server to use
     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
                        At the moment, this is only supported by YouTube.
     proxy:             URL of the proxy server to use

+    cn_verification_proxy:  URL of the proxy to use for IP address verification
+                       on Chinese sites. (Experimental)

     socket_timeout:    Time to wait for unresponsive hosts, in seconds
     bidi_workaround:   Work around buggy terminals without bidirectional text
                        support, using fridibi
     socket_timeout:    Time to wait for unresponsive hosts, in seconds
     bidi_workaround:   Work around buggy terminals without bidirectional text
                        support, using fridibi


@@ -199,18 +204,25 @@ class YoutubeDL(object):
                        postprocessor.
     progress_hooks:    A list of functions that get called on download
                        progress, with a dictionary with the entries
                        postprocessor.
     progress_hooks:    A list of functions that get called on download
                        progress, with a dictionary with the entries

-                       * status: One of "downloading" and "finished".
+                       * status: One of "downloading", "error", or "finished".

                                  Check this first and ignore unknown values.
 
                                  Check this first and ignore unknown values.
 

-                       If status is one of "downloading" or "finished", the
+                       If status is one of "downloading", or "finished", the

                        following properties may also be present:
                        * filename: The final filename (always present)
                        following properties may also be present:
                        * filename: The final filename (always present)

+                       * tmpfilename: The filename we're currently writing to

                        * downloaded_bytes: Bytes on disk
                        * total_bytes: Size of the whole file, None if unknown
                        * downloaded_bytes: Bytes on disk
                        * total_bytes: Size of the whole file, None if unknown

-                       * tmpfilename: The filename we're currently writing to
+                       * total_bytes_estimate: Guess of the eventual file size,
+                                               None if unavailable.
+                       * elapsed: The number of seconds since download started.

                        * eta: The estimated time in seconds, None if unknown
                        * speed: The download speed in bytes/second, None if
                                 unknown
                        * eta: The estimated time in seconds, None if unknown
                        * speed: The download speed in bytes/second, None if
                                 unknown

+                       * fragment_index: The counter of the currently
+                                         downloaded video fragment.
+                       * fragment_count: The number of fragments (= individual
+                                         files that will be merged)

 
                        Progress hooks are guaranteed to be called at least once
                        (with status "finished") if the download is successful.
 
                        Progress hooks are guaranteed to be called at least once
                        (with status "finished") if the download is successful.


@@ -225,7 +237,6 @@ class YoutubeDL(object):
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.

-    external_downloader:  Executable of the external downloader to call.

     listformats:       Print an overview of available video formats and exit.
     list_thumbnails:   Print a table of all thumbnails and exit.
     match_filter:      A function that gets called with the info_dict of
     listformats:       Print an overview of available video formats and exit.
     list_thumbnails:   Print a table of all thumbnails and exit.
     match_filter:      A function that gets called with the info_dict of


@@ -235,17 +246,20 @@ class YoutubeDL(object):
                        match_filter_func in utils.py is one example for this.
     no_color:          Do not emit color codes in output.
 
                        match_filter_func in utils.py is one example for this.
     no_color:          Do not emit color codes in output.
 

+    The following options determine which downloader is picked:
+    external_downloader: Executable of the external downloader to call.
+                       None or unset for standard (built-in) downloader.
+    hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.

 
     The following parameters are not used by YoutubeDL itself, they are used by
 
     The following parameters are not used by YoutubeDL itself, they are used by

-    the FileDownloader:
+    the downloader (see youtube_dl/downloader/common.py):

     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
     noresizebuffer, retries, continuedl, noprogress, consoletitle,
     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
     noresizebuffer, retries, continuedl, noprogress, consoletitle,

-    xattr_set_filesize.
+    xattr_set_filesize, external_downloader_args.

 
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.
 
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
                        otherwise prefer avconv.

-    exec_cmd:          Arbitrary command to run after downloading

     """
 
     params = None
     """
 
     params = None


@@ -274,7 +288,7 @@ class YoutubeDL(object):
             try:
                 import pty
                 master, slave = pty.openpty()
             try:
                 import pty
                 master, slave = pty.openpty()

-                width = get_term_width()
+                width = compat_get_terminal_size().columns

                 if width is None:
                     width_args = []
                 else:
                 if width is None:
                     width_args = []
                 else:


@@ -298,8 +312,8 @@ class YoutubeDL(object):
                     raise
 
         if (sys.version_info >= (3,) and sys.platform != 'win32' and
                     raise
 
         if (sys.version_info >= (3,) and sys.platform != 'win32' and

-                sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
-                and not params.get('restrictfilenames', False)):
+                sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
+                not params.get('restrictfilenames', False)):

             # On Python 3, the Unicode filesystem API will throw errors (#1474)
             self.report_warning(
                 'Assuming --restrict-filenames since file system encoding '
             # On Python 3, the Unicode filesystem API will throw errors (#1474)
             self.report_warning(
                 'Assuming --restrict-filenames since file system encoding '


@@ -307,8 +321,10 @@ class YoutubeDL(object):
                 'Set the LC_ALL environment variable to fix this.')
             self.params['restrictfilenames'] = True
 
                 'Set the LC_ALL environment variable to fix this.')
             self.params['restrictfilenames'] = True
 

-        if '%(stitle)s' in self.params.get('outtmpl', ''):
-            self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
+        if isinstance(params.get('outtmpl'), bytes):
+            self.report_warning(
+                'Parameter outtmpl is bytes, but should be a unicode string. '
+                'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')

 
         self._setup_opener()
 
 
         self._setup_opener()
 


@@ -547,7 +563,7 @@ class YoutubeDL(object):
                                  if v is not None)
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 
                                  if v is not None)
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 

-            outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
+            outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))

             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
             # Temporary fix for #4787
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
             # Temporary fix for #4787


@@ -614,7 +630,7 @@ class YoutubeDL(object):
         Returns a list with a dictionary for each video we find.
         If 'download', also downloads the videos.
         extra_info is a dict containing the extra values to add to each result
         Returns a list with a dictionary for each video we find.
         If 'download', also downloads the videos.
         extra_info is a dict containing the extra values to add to each result

-         '''
+        '''

 
         if ie_key:
             ies = [self.get_info_extractor(ie_key)]
 
         if ie_key:
             ies = [self.get_info_extractor(ie_key)]


@@ -898,10 +914,16 @@ class YoutubeDL(object):
         if not available_formats:
             return None
 
         if not available_formats:
             return None
 

-        if format_spec == 'best' or format_spec is None:
-            return available_formats[-1]
-        elif format_spec == 'worst':
-            return available_formats[0]
+        if format_spec in ['best', 'worst', None]:
+            format_idx = 0 if format_spec == 'worst' else -1
+            audiovideo_formats = [
+                f for f in available_formats
+                if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
+            if audiovideo_formats:
+                return audiovideo_formats[format_idx]
+            # for audio only urls, select the best/worst audio format
+            elif all(f.get('acodec') != 'none' for f in available_formats):
+                return available_formats[format_idx]

         elif format_spec == 'bestaudio':
             audio_formats = [
                 f for f in available_formats
         elif format_spec == 'bestaudio':
             audio_formats = [
                 f for f in available_formats


@@ -951,30 +973,9 @@ class YoutubeDL(object):
         return res
 
     def _calc_cookies(self, info_dict):
         return res
 
     def _calc_cookies(self, info_dict):

-        class _PseudoRequest(object):
-            def __init__(self, url):
-                self.url = url
-                self.headers = {}
-                self.unverifiable = False
-
-            def add_unredirected_header(self, k, v):
-                self.headers[k] = v
-
-            def get_full_url(self):
-                return self.url
-
-            def is_unverifiable(self):
-                return self.unverifiable
-
-            def has_header(self, h):
-                return h in self.headers
-
-            def get_header(self, h, default=None):
-                return self.headers.get(h, default)
-
-        pr = _PseudoRequest(info_dict['url'])
+        pr = compat_urllib_request.Request(info_dict['url'])

         self.cookiejar.add_cookie_header(pr)
         self.cookiejar.add_cookie_header(pr)

-        return pr.headers.get('Cookie')
+        return pr.get_header('Cookie')

 
     def process_video_result(self, info_dict, download=True):
         assert info_dict.get('_type', 'video') == 'video'
 
     def process_video_result(self, info_dict, download=True):
         assert info_dict.get('_type', 'video') == 'video'


@@ -1066,12 +1067,6 @@ class YoutubeDL(object):
             full_format_info.update(format)
             format['http_headers'] = self._calc_headers(full_format_info)
 
             full_format_info.update(format)
             format['http_headers'] = self._calc_headers(full_format_info)
 

-        format_limit = self.params.get('format_limit', None)
-        if format_limit:
-            formats = list(takewhile_inclusive(
-                lambda f: f['format_id'] != format_limit, formats
-            ))
-

         # TODO Central sorting goes here
 
         if formats[0] is not info_dict:
         # TODO Central sorting goes here
 
         if formats[0] is not info_dict:


@@ -1089,10 +1084,16 @@ class YoutubeDL(object):
 
         req_format = self.params.get('format')
         if req_format is None:
 
         req_format = self.params.get('format')
         if req_format is None:

-            req_format = 'best'
+            req_format_list = []
+            if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-'
+                    and info_dict['extractor'] in ['youtube', 'ted']):
+                merger = FFmpegMergerPP(self)
+                if merger.available and merger.can_merge():
+                    req_format_list.append('bestvideo+bestaudio')
+            req_format_list.append('best')
+            req_format = '/'.join(req_format_list)

         formats_to_download = []
         formats_to_download = []

-        # The -1 is for supporting YoutubeIE
-        if req_format in ('-1', 'all'):
+        if req_format == 'all':

             formats_to_download = formats
         else:
             for rfstr in req_format.split(','):
             formats_to_download = formats
         else:
             for rfstr in req_format.split(','):


@@ -1156,11 +1157,13 @@ class YoutubeDL(object):
         info_dict.update(formats_to_download[-1])
         return info_dict
 
         info_dict.update(formats_to_download[-1])
         return info_dict
 

-    def process_subtitles(self, video_id, available_subs, available_autocaps):
+    def process_subtitles(self, video_id, normal_subtitles, automatic_captions):

         """Select the requested subtitles and their format"""
         """Select the requested subtitles and their format"""

-        if available_autocaps and self.params.get('writeautomaticsub'):
-            available_subs = available_subs.copy()
-            for lang, cap_info in available_autocaps.items():
+        available_subs = {}
+        if normal_subtitles and self.params.get('writesubtitles'):
+            available_subs.update(normal_subtitles)
+        if automatic_captions and self.params.get('writeautomaticsub'):
+            for lang, cap_info in automatic_captions.items():

                 if lang not in available_subs:
                     available_subs[lang] = cap_info
 
                 if lang not in available_subs:
                     available_subs[lang] = cap_info
 


@@ -1187,14 +1190,6 @@ class YoutubeDL(object):
             if formats is None:
                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
                 continue
             if formats is None:
                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
                 continue

-            if isinstance(formats, compat_str):
-                # TODO: convert all IE with subtitles support to the new format
-                # and remove this
-                subs[lang] = {
-                    'ext': formats_preference[0],
-                    'data': formats,
-                }
-                continue

             for ext in formats_preference:
                 if ext == 'best':
                     f = formats[-1]
             for ext in formats_preference:
                 if ext == 'best':
                     f = formats[-1]


@@ -1225,9 +1220,6 @@ class YoutubeDL(object):
         if len(info_dict['title']) > 200:
             info_dict['title'] = info_dict['title'][:197] + '...'
 
         if len(info_dict['title']) > 200:
             info_dict['title'] = info_dict['title'][:197] + '...'
 

-        # Keep for backwards compatibility
-        info_dict['stitle'] = info_dict['title']
-

         if 'format' not in info_dict:
             info_dict['format'] = info_dict['ext']
 
         if 'format' not in info_dict:
             info_dict['format'] = info_dict['ext']
 


@@ -1273,7 +1265,7 @@ class YoutubeDL(object):
             return
 
         try:
             return
 
         try:

-            dn = os.path.dirname(encodeFilename(filename))
+            dn = os.path.dirname(sanitize_path(encodeFilename(filename)))

             if dn and not os.path.exists(dn):
                 os.makedirs(dn)
         except (OSError, IOError) as err:
             if dn and not os.path.exists(dn):
                 os.makedirs(dn)
         except (OSError, IOError) as err:


@@ -1281,7 +1273,7 @@ class YoutubeDL(object):
             return
 
         if self.params.get('writedescription', False):
             return
 
         if self.params.get('writedescription', False):

-            descfn = filename + '.description'
+            descfn = replace_extension(filename, 'description', info_dict.get('ext'))

             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
                 self.to_screen('[info] Video description is already present')
             elif info_dict.get('description') is None:
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
                 self.to_screen('[info] Video description is already present')
             elif info_dict.get('description') is None:


@@ -1296,7 +1288,7 @@ class YoutubeDL(object):
                     return
 
         if self.params.get('writeannotations', False):
                     return
 
         if self.params.get('writeannotations', False):

-            annofn = filename + '.annotations.xml'
+            annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))

             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
                 self.to_screen('[info] Video annotations are already present')
             else:
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
                 self.to_screen('[info] Video annotations are already present')
             else:


@@ -1317,17 +1309,18 @@ class YoutubeDL(object):
             # subtitles download errors are already managed as troubles in relevant IE
             # that way it will silently go on when used with unsupporting IE
             subtitles = info_dict['requested_subtitles']
             # subtitles download errors are already managed as troubles in relevant IE
             # that way it will silently go on when used with unsupporting IE
             subtitles = info_dict['requested_subtitles']

+            ie = self.get_info_extractor(info_dict['extractor_key'])

             for sub_lang, sub_info in subtitles.items():
                 sub_format = sub_info['ext']
                 if sub_info.get('data') is not None:
                     sub_data = sub_info['data']
                 else:
                     try:
             for sub_lang, sub_info in subtitles.items():
                 sub_format = sub_info['ext']
                 if sub_info.get('data') is not None:
                     sub_data = sub_info['data']
                 else:
                     try:

-                        uf = self.urlopen(sub_info['url'])
-                        sub_data = uf.read().decode('utf-8')
-                    except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+                        sub_data = ie._download_webpage(
+                            sub_info['url'], info_dict['id'], note=False)
+                    except ExtractorError as err:

                         self.report_warning('Unable to download subtitle for "%s": %s' %
                         self.report_warning('Unable to download subtitle for "%s": %s' %

-                                            (sub_lang, compat_str(err)))
+                                            (sub_lang, compat_str(err.cause)))

                         continue
                 try:
                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
                         continue
                 try:
                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)


@@ -1342,13 +1335,13 @@ class YoutubeDL(object):
                     return
 
         if self.params.get('writeinfojson', False):
                     return
 
         if self.params.get('writeinfojson', False):

-            infofn = os.path.splitext(filename)[0] + '.info.json'
+            infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))

             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
                 self.to_screen('[info] Video description metadata is already present')
             else:
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
                 try:
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
                 self.to_screen('[info] Video description metadata is already present')
             else:
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
                 try:

-                    write_json_file(info_dict, infofn)
+                    write_json_file(self.filter_requested_info(info_dict), infofn)

                 except (OSError, IOError):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return
                 except (OSError, IOError):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return


@@ -1368,24 +1361,57 @@ class YoutubeDL(object):
                 if info_dict.get('requested_formats') is not None:
                     downloaded = []
                     success = True
                 if info_dict.get('requested_formats') is not None:
                     downloaded = []
                     success = True

-                    merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
-                    if not merger._executable:
+                    merger = FFmpegMergerPP(self)
+                    if not merger.available:

                         postprocessors = []
                         self.report_warning('You have requested multiple '
                                             'formats but ffmpeg or avconv are not installed.'
                                             ' The formats won\'t be merged')
                     else:
                         postprocessors = [merger]
                         postprocessors = []
                         self.report_warning('You have requested multiple '
                                             'formats but ffmpeg or avconv are not installed.'
                                             ' The formats won\'t be merged')
                     else:
                         postprocessors = [merger]

-                    for f in info_dict['requested_formats']:
-                        new_info = dict(info_dict)
-                        new_info.update(f)
-                        fname = self.prepare_filename(new_info)
-                        fname = prepend_extension(fname, 'f%s' % f['format_id'])
-                        downloaded.append(fname)
-                        partial_success = dl(fname, new_info)
-                        success = success and partial_success
-                    info_dict['__postprocessors'] = postprocessors
-                    info_dict['__files_to_merge'] = downloaded
+
+                    def compatible_formats(formats):
+                        video, audio = formats
+                        # Check extension
+                        video_ext, audio_ext = audio.get('ext'), video.get('ext')
+                        if video_ext and audio_ext:
+                            COMPATIBLE_EXTS = (
+                                ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
+                                ('webm')
+                            )
+                            for exts in COMPATIBLE_EXTS:
+                                if video_ext in exts and audio_ext in exts:
+                                    return True
+                        # TODO: Check acodec/vcodec
+                        return False
+
+                    filename_real_ext = os.path.splitext(filename)[1][1:]
+                    filename_wo_ext = (
+                        os.path.splitext(filename)[0]
+                        if filename_real_ext == info_dict['ext']
+                        else filename)
+                    requested_formats = info_dict['requested_formats']
+                    if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
+                        info_dict['ext'] = 'mkv'
+                        self.report_warning('You have requested formats incompatible for merge. '
+                                            'The formats will be merged into mkv')
+                    # Ensure filename always has a correct extension for successful merge
+                    filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
+                    if os.path.exists(encodeFilename(filename)):
+                        self.to_screen(
+                            '[download] %s has already been downloaded and '
+                            'merged' % filename)
+                    else:
+                        for f in requested_formats:
+                            new_info = dict(info_dict)
+                            new_info.update(f)
+                            fname = self.prepare_filename(new_info)
+                            fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
+                            downloaded.append(fname)
+                            partial_success = dl(fname, new_info)
+                            success = success and partial_success
+                        info_dict['__postprocessors'] = postprocessors
+                        info_dict['__files_to_merge'] = downloaded

                 else:
                     # Just a single file
                     success = dl(filename, info_dict)
                 else:
                     # Just a single file
                     success = dl(filename, info_dict)


@@ -1448,8 +1474,8 @@ class YoutubeDL(object):
         """Download a given list of URLs."""
         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
         if (len(url_list) > 1 and
         """Download a given list of URLs."""
         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
         if (len(url_list) > 1 and

-                '%' not in outtmpl
-                and self.params.get('max_downloads') != 1):
+                '%' not in outtmpl and
+                self.params.get('max_downloads') != 1):

             raise SameFileError(outtmpl)
 
         for url in url_list:
             raise SameFileError(outtmpl)
 
         for url in url_list:


@@ -1468,8 +1494,11 @@ class YoutubeDL(object):
         return self._download_retcode
 
     def download_with_info_file(self, info_filename):
         return self._download_retcode
 
     def download_with_info_file(self, info_filename):

-        with io.open(info_filename, 'r', encoding='utf-8') as f:
-            info = json.load(f)
+        with contextlib.closing(fileinput.FileInput(
+                [info_filename], mode='r',
+                openhook=fileinput.hook_encoded('utf-8'))) as f:
+            # FileInput doesn't have a read method, we can't call json.load
+            info = self.filter_requested_info(json.loads('\n'.join(f)))

         try:
             self.process_ie_result(info, download=True)
         except DownloadError:
         try:
             self.process_ie_result(info, download=True)
         except DownloadError:


@@ -1481,6 +1510,12 @@ class YoutubeDL(object):
                 raise
         return self._download_retcode
 
                 raise
         return self._download_retcode
 

+    @staticmethod
+    def filter_requested_info(info_dict):
+        return dict(
+            (k, v) for k, v in info_dict.items()
+            if k not in ['requested_formats', 'requested_subtitles'])
+

     def post_process(self, filename, ie_info):
         """Run all the postprocessors on the given file."""
         info = dict(ie_info)
     def post_process(self, filename, ie_info):
         """Run all the postprocessors on the given file."""
         info = dict(ie_info)


@@ -1490,24 +1525,17 @@ class YoutubeDL(object):
             pps_chain.extend(ie_info['__postprocessors'])
         pps_chain.extend(self._pps)
         for pp in pps_chain:
             pps_chain.extend(ie_info['__postprocessors'])
         pps_chain.extend(self._pps)
         for pp in pps_chain:

-            keep_video = None
-            old_filename = info['filepath']

             try:
             try:

-                keep_video_wish, info = pp.run(info)
-                if keep_video_wish is not None:
-                    if keep_video_wish:
-                        keep_video = keep_video_wish
-                    elif keep_video is None:
-                        # No clear decision yet, let IE decide
-                        keep_video = keep_video_wish
+                files_to_delete, info = pp.run(info)

             except PostProcessingError as e:
                 self.report_error(e.msg)
             except PostProcessingError as e:
                 self.report_error(e.msg)

-            if keep_video is False and not self.params.get('keepvideo', False):
-                try:
+            if files_to_delete and not self.params.get('keepvideo', False):
+                for old_filename in files_to_delete:

                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)

-                    os.remove(encodeFilename(old_filename))
-                except (IOError, OSError):
-                    self.report_warning('Unable to remove downloaded video file')
+                    try:
+                        os.remove(encodeFilename(old_filename))
+                    except (IOError, OSError):
+                        self.report_warning('Unable to remove downloaded original file')

 
     def _make_archive_id(self, info_dict):
         # Future-proof against any change in case
 
     def _make_archive_id(self, info_dict):
         # Future-proof against any change in case


@@ -1616,29 +1644,18 @@ class YoutubeDL(object):
         return res
 
     def list_formats(self, info_dict):
         return res
 
     def list_formats(self, info_dict):

-        def line(format, idlen=20):
-            return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
-                format['format_id'],
-                format['ext'],
-                self.format_resolution(format),
-                self._format_note(format),
-            ))
-

         formats = info_dict.get('formats', [info_dict])
         formats = info_dict.get('formats', [info_dict])

-        idlen = max(len('format code'),
-                    max(len(f['format_id']) for f in formats))
-        formats_s = [
-            line(f, idlen) for f in formats
+        table = [
+            [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
+            for f in formats

             if f.get('preference') is None or f['preference'] >= -1000]
         if len(formats) > 1:
             if f.get('preference') is None or f['preference'] >= -1000]
         if len(formats) > 1:

-            formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
+            table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'

 
 

-        header_line = line({
-            'format_id': 'format code', 'ext': 'extension',
-            'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
+        header_line = ['format code', 'extension', 'resolution', 'note']

         self.to_screen(
         self.to_screen(

-            '[info] Available formats for %s:\n%s\n%s' %
-            (info_dict['id'], header_line, '\n'.join(formats_s)))
+            '[info] Available formats for %s:\n%s' %
+            (info_dict['id'], render_table(header_line, table)))

 
     def list_thumbnails(self, info_dict):
         thumbnails = info_dict.get('thumbnails')
 
     def list_thumbnails(self, info_dict):
         thumbnails = info_dict.get('thumbnails')


@@ -1721,15 +1738,15 @@ class YoutubeDL(object):
             out = out.decode().strip()
             if re.match('[0-9a-f]+', out):
                 self._write_string('[debug] Git HEAD: ' + out + '\n')
             out = out.decode().strip()
             if re.match('[0-9a-f]+', out):
                 self._write_string('[debug] Git HEAD: ' + out + '\n')

-        except:
+        except Exception:

             try:
                 sys.exc_clear()
             try:
                 sys.exc_clear()

-            except:
+            except Exception:

                 pass
         self._write_string('[debug] Python version %s - %s\n' % (
             platform.python_version(), platform_name()))
 
                 pass
         self._write_string('[debug] Python version %s - %s\n' % (
             platform.python_version(), platform_name()))
 

-        exe_versions = FFmpegPostProcessor.get_versions()
+        exe_versions = FFmpegPostProcessor.get_versions(self)

         exe_versions['rtmpdump'] = rtmpdump_version()
         exe_str = ', '.join(
             '%s %s' % (exe, v)
         exe_versions['rtmpdump'] = rtmpdump_version()
         exe_str = ', '.join(
             '%s %s' % (exe, v)


@@ -1784,13 +1801,14 @@ class YoutubeDL(object):
             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
             if 'http' in proxies and 'https' not in proxies:
                 proxies['https'] = proxies['http']
             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
             if 'http' in proxies and 'https' not in proxies:
                 proxies['https'] = proxies['http']

-        proxy_handler = compat_urllib_request.ProxyHandler(proxies)
+        proxy_handler = PerRequestProxyHandler(proxies)

 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         opener = compat_urllib_request.build_opener(
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         opener = compat_urllib_request.build_opener(

-            https_handler, proxy_handler, cookie_processor, ydlh)
+            proxy_handler, https_handler, cookie_processor, ydlh)
+

         # Delete the default user-agent header, which would otherwise apply in
         # cases where our custom HTTP handler doesn't come into play
         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
         # Delete the default user-agent header, which would otherwise apply in
         # cases where our custom HTTP handler doesn't come into play
         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)