_ Git - youtube-dl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import contextlib
   8 import datetime
   9 import errno
  10 import fileinput
  11 import io
  12 import itertools
  13 import json
  14 import locale
  15 import operator
  16 import os
  17 import platform
  18 import re
  19 import shutil
  20 import subprocess
  21 import socket
  22 import sys
  23 import time
  24 import traceback
  25
  26 if os.name == 'nt':
  27     import ctypes
  28
  29 from .compat import (
  30     compat_basestring,
  31     compat_cookiejar,
  32     compat_expanduser,
  33     compat_get_terminal_size,
  34     compat_http_client,
  35     compat_kwargs,
  36     compat_str,
  37     compat_urllib_error,
  38     compat_urllib_request,
  39 )
  40 from .utils import (
  41     escape_url,
  42     ContentTooShortError,
  43     date_from_str,
  44     DateRange,
  45     DEFAULT_OUTTMPL,
  46     determine_ext,
  47     DownloadError,
  48     encodeFilename,
  49     ExtractorError,
  50     format_bytes,
  51     formatSeconds,
  52     HEADRequest,
  53     locked_file,
  54     make_HTTPS_handler,
  55     MaxDownloadsReached,
  56     PagedList,
  57     parse_filesize,
  58     PerRequestProxyHandler,
  59     PostProcessingError,
  60     platform_name,
  61     preferredencoding,
  62     render_table,
  63     SameFileError,
  64     sanitize_filename,
  65     sanitize_path,
  66     std_headers,
  67     subtitles_filename,
  68     UnavailableVideoError,
  69     url_basename,
  70     version_tuple,
  71     write_json_file,
  72     write_string,
  73     YoutubeDLHandler,
  74     prepend_extension,
  75     replace_extension,
  76     args_to_str,
  77     age_restricted,
  78 )
  79 from .cache import Cache
  80 from .extractor import get_info_extractor, gen_extractors
  81 from .downloader import get_suitable_downloader
  82 from .downloader.rtmp import rtmpdump_version
  83 from .postprocessor import (
  84     FFmpegFixupM4aPP,
  85     FFmpegFixupStretchedPP,
  86     FFmpegMergerPP,
  87     FFmpegPostProcessor,
  88     get_postprocessor,
  89 )
  90 from .version import __version__
  91
  92
  93 class YoutubeDL(object):
  94     """YoutubeDL class.
  95
  96     YoutubeDL objects are the ones responsible of downloading the
  97     actual video file and writing it to disk if the user has requested
  98     it, among some other tasks. In most cases there should be one per
  99     program. As, given a video URL, the downloader doesn't know how to
 100     extract all the needed information, task that InfoExtractors do, it
 101     has to pass the URL to one of them.
 102
 103     For this, YoutubeDL objects have a method that allows
 104     InfoExtractors to be registered in a given order. When it is passed
 105     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 106     finds that reports being able to handle it. The InfoExtractor extracts
 107     all the information about the video or videos the URL refers to, and
 108     YoutubeDL process the extracted information, possibly using a File
 109     Downloader to download the video.
 110
 111     YoutubeDL objects accept a lot of parameters. In order not to saturate
 112     the object constructor with arguments, it receives a dictionary of
 113     options instead. These options are available through the params
 114     attribute for the InfoExtractors to use. The YoutubeDL also
 115     registers itself as the downloader in charge for the InfoExtractors
 116     that are added to it, so this is a "mutual registration".
 117
 118     Available options:
 119
 120     username:          Username for authentication purposes.
 121     password:          Password for authentication purposes.
 122     videopassword:     Password for acces a video.
 123     usenetrc:          Use netrc for authentication instead.
 124     verbose:           Print additional info to stdout.
 125     quiet:             Do not print messages to stdout.
 126     no_warnings:       Do not print out anything for warnings.
 127     forceurl:          Force printing final URL.
 128     forcetitle:        Force printing title.
 129     forceid:           Force printing ID.
 130     forcethumbnail:    Force printing thumbnail URL.
 131     forcedescription:  Force printing description.
 132     forcefilename:     Force printing final filename.
 133     forceduration:     Force printing duration.
 134     forcejson:         Force printing info_dict as JSON.
 135     dump_single_json:  Force printing the info_dict of the whole playlist
 136                        (or video) as a single JSON line.
 137     simulate:          Do not download the video files.
 138     format:            Video format code. See options.py for more information.
 139     outtmpl:           Template for output names.
 140     restrictfilenames: Do not allow "&" and spaces in file names
 141     ignoreerrors:      Do not stop on download errors.
 142     force_generic_extractor: Force downloader to use the generic extractor
 143     nooverwrites:      Prevent overwriting files.
 144     playliststart:     Playlist item to start at.
 145     playlistend:       Playlist item to end at.
 146     playlist_items:    Specific indices of playlist to download.
 147     playlistreverse:   Download playlist items in reverse order.
 148     matchtitle:        Download only matching titles.
 149     rejecttitle:       Reject downloads for matching titles.
 150     logger:            Log messages to a logging.Logger instance.
 151     logtostderr:       Log messages to stderr instead of stdout.
 152     writedescription:  Write the video description to a .description file
 153     writeinfojson:     Write the video description to a .info.json file
 154     writeannotations:  Write the video annotations to a .annotations.xml file
 155     writethumbnail:    Write the thumbnail image to a file
 156     write_all_thumbnails:  Write all thumbnail formats to files
 157     writesubtitles:    Write the video subtitles to a file
 158     writeautomaticsub: Write the automatic subtitles to a file
 159     allsubtitles:      Downloads all the subtitles of the video
 160                        (requires writesubtitles or writeautomaticsub)
 161     listsubtitles:     Lists all available subtitles for the video
 162     subtitlesformat:   The format code for subtitles
 163     subtitleslangs:    List of languages of the subtitles to download
 164     keepvideo:         Keep the video file after post-processing
 165     daterange:         A DateRange object, download only if the upload_date is in the range.
 166     skip_download:     Skip the actual download of the video file
 167     cachedir:          Location of the cache files in the filesystem.
 168                        False to disable filesystem cache.
 169     noplaylist:        Download single video instead of a playlist if in doubt.
 170     age_limit:         An integer representing the user's age in years.
 171                        Unsuitable videos for the given age are skipped.
 172     min_views:         An integer representing the minimum view count the video
 173                        must have in order to not be skipped.
 174                        Videos without view count information are always
 175                        downloaded. None for no limit.
 176     max_views:         An integer representing the maximum view count.
 177                        Videos that are more popular than that are not
 178                        downloaded.
 179                        Videos without view count information are always
 180                        downloaded. None for no limit.
 181     download_archive:  File name of a file where all downloads are recorded.
 182                        Videos already present in the file are not downloaded
 183                        again.
 184     cookiefile:        File name where cookies should be read from and dumped to.
 185     nocheckcertificate:Do not verify SSL certificates
 186     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 187                        At the moment, this is only supported by YouTube.
 188     proxy:             URL of the proxy server to use
 189     cn_verification_proxy:  URL of the proxy to use for IP address verification
 190                        on Chinese sites. (Experimental)
 191     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 192     bidi_workaround:   Work around buggy terminals without bidirectional text
 193                        support, using fridibi
 194     debug_printtraffic:Print out sent and received HTTP traffic
 195     include_ads:       Download ads as well
 196     default_search:    Prepend this string if an input url is not valid.
 197                        'auto' for elaborate guessing
 198     encoding:          Use this encoding instead of the system-specified.
 199     extract_flat:      Do not resolve URLs, return the immediate result.
 200                        Pass in 'in_playlist' to only show this behavior for
 201                        playlist items.
 202     postprocessors:    A list of dictionaries, each with an entry
 203                        * key:  The name of the postprocessor. See
 204                                youtube_dl/postprocessor/__init__.py for a list.
 205                        as well as any further keyword arguments for the
 206                        postprocessor.
 207     progress_hooks:    A list of functions that get called on download
 208                        progress, with a dictionary with the entries
 209                        * status: One of "downloading", "error", or "finished".
 210                                  Check this first and ignore unknown values.
 211
 212                        If status is one of "downloading", or "finished", the
 213                        following properties may also be present:
 214                        * filename: The final filename (always present)
 215                        * tmpfilename: The filename we're currently writing to
 216                        * downloaded_bytes: Bytes on disk
 217                        * total_bytes: Size of the whole file, None if unknown
 218                        * total_bytes_estimate: Guess of the eventual file size,
 219                                                None if unavailable.
 220                        * elapsed: The number of seconds since download started.
 221                        * eta: The estimated time in seconds, None if unknown
 222                        * speed: The download speed in bytes/second, None if
 223                                 unknown
 224                        * fragment_index: The counter of the currently
 225                                          downloaded video fragment.
 226                        * fragment_count: The number of fragments (= individual
 227                                          files that will be merged)
 228
 229                        Progress hooks are guaranteed to be called at least once
 230                        (with status "finished") if the download is successful.
 231     merge_output_format: Extension to use when merging formats.
 232     fixup:             Automatically correct known faults of the file.
 233                        One of:
 234                        - "never": do nothing
 235                        - "warn": only emit a warning
 236                        - "detect_or_warn": check whether we can do anything
 237                                            about it, warn otherwise (default)
 238     source_address:    (Experimental) Client-side IP address to bind to.
 239     call_home:         Boolean, true iff we are allowed to contact the
 240                        youtube-dl servers for debugging.
 241     sleep_interval:    Number of seconds to sleep before each download.
 242     listformats:       Print an overview of available video formats and exit.
 243     list_thumbnails:   Print a table of all thumbnails and exit.
 244     match_filter:      A function that gets called with the info_dict of
 245                        every video.
 246                        If it returns a message, the video is ignored.
 247                        If it returns None, the video is downloaded.
 248                        match_filter_func in utils.py is one example for this.
 249     no_color:          Do not emit color codes in output.
 250
 251     The following options determine which downloader is picked:
 252     external_downloader: Executable of the external downloader to call.
 253                        None or unset for standard (built-in) downloader.
 254     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
 255
 256     The following parameters are not used by YoutubeDL itself, they are used by
 257     the downloader (see youtube_dl/downloader/common.py):
 258     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 259     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 260     xattr_set_filesize, external_downloader_args.
 261
 262     The following options are used by the post processors:
 263     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 264                        otherwise prefer avconv.
 265     """
 266
 267     params = None
 268     _ies = []
 269     _pps = []
 270     _download_retcode = None
 271     _num_downloads = None
 272     _screen_file = None
 273
 274     def __init__(self, params=None, auto_init=True):
 275         """Create a FileDownloader object with the given options."""
 276         if params is None:
 277             params = {}
 278         self._ies = []
 279         self._ies_instances = {}
 280         self._pps = []
 281         self._progress_hooks = []
 282         self._download_retcode = 0
 283         self._num_downloads = 0
 284         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 285         self._err_file = sys.stderr
 286         self.params = params
 287         self.cache = Cache(self)
 288
 289         if params.get('bidi_workaround', False):
 290             try:
 291                 import pty
 292                 master, slave = pty.openpty()
 293                 width = compat_get_terminal_size().columns
 294                 if width is None:
 295                     width_args = []
 296                 else:
 297                     width_args = ['-w', str(width)]
 298                 sp_kwargs = dict(
 299                     stdin=subprocess.PIPE,
 300                     stdout=slave,
 301                     stderr=self._err_file)
 302                 try:
 303                     self._output_process = subprocess.Popen(
 304                         ['bidiv'] + width_args, **sp_kwargs
 305                     )
 306                 except OSError:
 307                     self._output_process = subprocess.Popen(
 308                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 309                 self._output_channel = os.fdopen(master, 'rb')
 310             except OSError as ose:
 311                 if ose.errno == 2:
 312                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 313                 else:
 314                     raise
 315
 316         if (sys.version_info >= (3,) and sys.platform != 'win32' and
 317                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
 318                 not params.get('restrictfilenames', False)):
 319             # On Python 3, the Unicode filesystem API will throw errors (#1474)
 320             self.report_warning(
 321                 'Assuming --restrict-filenames since file system encoding '
 322                 'cannot encode all characters. '
 323                 'Set the LC_ALL environment variable to fix this.')
 324             self.params['restrictfilenames'] = True
 325
 326         if isinstance(params.get('outtmpl'), bytes):
 327             self.report_warning(
 328                 'Parameter outtmpl is bytes, but should be a unicode string. '
 329                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 330
 331         self._setup_opener()
 332
 333         if auto_init:
 334             self.print_debug_header()
 335             self.add_default_info_extractors()
 336
 337         for pp_def_raw in self.params.get('postprocessors', []):
 338             pp_class = get_postprocessor(pp_def_raw['key'])
 339             pp_def = dict(pp_def_raw)
 340             del pp_def['key']
 341             pp = pp_class(self, **compat_kwargs(pp_def))
 342             self.add_post_processor(pp)
 343
 344         for ph in self.params.get('progress_hooks', []):
 345             self.add_progress_hook(ph)
 346
 347     def warn_if_short_id(self, argv):
 348         # short YouTube ID starting with dash?
 349         idxs = [
 350             i for i, a in enumerate(argv)
 351             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 352         if idxs:
 353             correct_argv = (
 354                 ['youtube-dl'] +
 355                 [a for i, a in enumerate(argv) if i not in idxs] +
 356                 ['--'] + [argv[i] for i in idxs]
 357             )
 358             self.report_warning(
 359                 'Long argument string detected. '
 360                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 361                 args_to_str(correct_argv))
 362
 363     def add_info_extractor(self, ie):
 364         """Add an InfoExtractor object to the end of the list."""
 365         self._ies.append(ie)
 366         self._ies_instances[ie.ie_key()] = ie
 367         ie.set_downloader(self)
 368
 369     def get_info_extractor(self, ie_key):
 370         """
 371         Get an instance of an IE with name ie_key, it will try to get one from
 372         the _ies list, if there's no instance it will create a new one and add
 373         it to the extractor list.
 374         """
 375         ie = self._ies_instances.get(ie_key)
 376         if ie is None:
 377             ie = get_info_extractor(ie_key)()
 378             self.add_info_extractor(ie)
 379         return ie
 380
 381     def add_default_info_extractors(self):
 382         """
 383         Add the InfoExtractors returned by gen_extractors to the end of the list
 384         """
 385         for ie in gen_extractors():
 386             self.add_info_extractor(ie)
 387
 388     def add_post_processor(self, pp):
 389         """Add a PostProcessor object to the end of the chain."""
 390         self._pps.append(pp)
 391         pp.set_downloader(self)
 392
 393     def add_progress_hook(self, ph):
 394         """Add the progress hook (currently only for the file downloader)"""
 395         self._progress_hooks.append(ph)
 396
 397     def _bidi_workaround(self, message):
 398         if not hasattr(self, '_output_channel'):
 399             return message
 400
 401         assert hasattr(self, '_output_process')
 402         assert isinstance(message, compat_str)
 403         line_count = message.count('\n') + 1
 404         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 405         self._output_process.stdin.flush()
 406         res = ''.join(self._output_channel.readline().decode('utf-8')
 407                       for _ in range(line_count))
 408         return res[:-len('\n')]
 409
 410     def to_screen(self, message, skip_eol=False):
 411         """Print message to stdout if not in quiet mode."""
 412         return self.to_stdout(message, skip_eol, check_quiet=True)
 413
 414     def _write_string(self, s, out=None):
 415         write_string(s, out=out, encoding=self.params.get('encoding'))
 416
 417     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 418         """Print message to stdout if not in quiet mode."""
 419         if self.params.get('logger'):
 420             self.params['logger'].debug(message)
 421         elif not check_quiet or not self.params.get('quiet', False):
 422             message = self._bidi_workaround(message)
 423             terminator = ['\n', ''][skip_eol]
 424             output = message + terminator
 425
 426             self._write_string(output, self._screen_file)
 427
 428     def to_stderr(self, message):
 429         """Print message to stderr."""
 430         assert isinstance(message, compat_str)
 431         if self.params.get('logger'):
 432             self.params['logger'].error(message)
 433         else:
 434             message = self._bidi_workaround(message)
 435             output = message + '\n'
 436             self._write_string(output, self._err_file)
 437
 438     def to_console_title(self, message):
 439         if not self.params.get('consoletitle', False):
 440             return
 441         if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 442             # c_wchar_p() might not be necessary if `message` is
 443             # already of type unicode()
 444             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 445         elif 'TERM' in os.environ:
 446             self._write_string('\033]0;%s\007' % message, self._screen_file)
 447
 448     def save_console_title(self):
 449         if not self.params.get('consoletitle', False):
 450             return
 451         if 'TERM' in os.environ:
 452             # Save the title on stack
 453             self._write_string('\033[22;0t', self._screen_file)
 454
 455     def restore_console_title(self):
 456         if not self.params.get('consoletitle', False):
 457             return
 458         if 'TERM' in os.environ:
 459             # Restore the title from stack
 460             self._write_string('\033[23;0t', self._screen_file)
 461
 462     def __enter__(self):
 463         self.save_console_title()
 464         return self
 465
 466     def __exit__(self, *args):
 467         self.restore_console_title()
 468
 469         if self.params.get('cookiefile') is not None:
 470             self.cookiejar.save()
 471
 472     def trouble(self, message=None, tb=None):
 473         """Determine action to take when a download problem appears.
 474
 475         Depending on if the downloader has been configured to ignore
 476         download errors or not, this method may throw an exception or
 477         not when errors are found, after printing the message.
 478
 479         tb, if given, is additional traceback information.
 480         """
 481         if message is not None:
 482             self.to_stderr(message)
 483         if self.params.get('verbose'):
 484             if tb is None:
 485                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 486                     tb = ''
 487                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 488                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 489                     tb += compat_str(traceback.format_exc())
 490                 else:
 491                     tb_data = traceback.format_list(traceback.extract_stack())
 492                     tb = ''.join(tb_data)
 493             self.to_stderr(tb)
 494         if not self.params.get('ignoreerrors', False):
 495             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 496                 exc_info = sys.exc_info()[1].exc_info
 497             else:
 498                 exc_info = sys.exc_info()
 499             raise DownloadError(message, exc_info)
 500         self._download_retcode = 1
 501
 502     def report_warning(self, message):
 503         '''
 504         Print the message to stderr, it will be prefixed with 'WARNING:'
 505         If stderr is a tty file the 'WARNING:' will be colored
 506         '''
 507         if self.params.get('logger') is not None:
 508             self.params['logger'].warning(message)
 509         else:
 510             if self.params.get('no_warnings'):
 511                 return
 512             if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
 513                 _msg_header = '\033[0;33mWARNING:\033[0m'
 514             else:
 515                 _msg_header = 'WARNING:'
 516             warning_message = '%s %s' % (_msg_header, message)
 517             self.to_stderr(warning_message)
 518
 519     def report_error(self, message, tb=None):
 520         '''
 521         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 522         in red if stderr is a tty file.
 523         '''
 524         if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
 525             _msg_header = '\033[0;31mERROR:\033[0m'
 526         else:
 527             _msg_header = 'ERROR:'
 528         error_message = '%s %s' % (_msg_header, message)
 529         self.trouble(error_message, tb)
 530
 531     def report_file_already_downloaded(self, file_name):
 532         """Report file has already been fully downloaded."""
 533         try:
 534             self.to_screen('[download] %s has already been downloaded' % file_name)
 535         except UnicodeEncodeError:
 536             self.to_screen('[download] The file has already been downloaded')
 537
 538     def prepare_filename(self, info_dict):
 539         """Generate the output filename."""
 540         try:
 541             template_dict = dict(info_dict)
 542
 543             template_dict['epoch'] = int(time.time())
 544             autonumber_size = self.params.get('autonumber_size')
 545             if autonumber_size is None:
 546                 autonumber_size = 5
 547             autonumber_templ = '%0' + str(autonumber_size) + 'd'
 548             template_dict['autonumber'] = autonumber_templ % self._num_downloads
 549             if template_dict.get('playlist_index') is not None:
 550                 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
 551             if template_dict.get('resolution') is None:
 552                 if template_dict.get('width') and template_dict.get('height'):
 553                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 554                 elif template_dict.get('height'):
 555                     template_dict['resolution'] = '%sp' % template_dict['height']
 556                 elif template_dict.get('width'):
 557                     template_dict['resolution'] = '?x%d' % template_dict['width']
 558
 559             sanitize = lambda k, v: sanitize_filename(
 560                 compat_str(v),
 561                 restricted=self.params.get('restrictfilenames'),
 562                 is_id=(k == 'id'))
 563             template_dict = dict((k, sanitize(k, v))
 564                                  for k, v in template_dict.items()
 565                                  if v is not None)
 566             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 567
 568             outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
 569             tmpl = compat_expanduser(outtmpl)
 570             filename = tmpl % template_dict
 571             # Temporary fix for #4787
 572             # 'Treat' all problem characters by passing filename through preferredencoding
 573             # to workaround encoding issues with subprocess on python2 @ Windows
 574             if sys.version_info < (3, 0) and sys.platform == 'win32':
 575                 filename = encodeFilename(filename, True).decode(preferredencoding())
 576             return filename
 577         except ValueError as err:
 578             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 579             return None
 580
 581     def _match_entry(self, info_dict, incomplete):
 582         """ Returns None iff the file should be downloaded """
 583
 584         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 585         if 'title' in info_dict:
 586             # This can happen when we're just evaluating the playlist
 587             title = info_dict['title']
 588             matchtitle = self.params.get('matchtitle', False)
 589             if matchtitle:
 590                 if not re.search(matchtitle, title, re.IGNORECASE):
 591                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 592             rejecttitle = self.params.get('rejecttitle', False)
 593             if rejecttitle:
 594                 if re.search(rejecttitle, title, re.IGNORECASE):
 595                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 596         date = info_dict.get('upload_date', None)
 597         if date is not None:
 598             dateRange = self.params.get('daterange', DateRange())
 599             if date not in dateRange:
 600                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 601         view_count = info_dict.get('view_count', None)
 602         if view_count is not None:
 603             min_views = self.params.get('min_views')
 604             if min_views is not None and view_count < min_views:
 605                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 606             max_views = self.params.get('max_views')
 607             if max_views is not None and view_count > max_views:
 608                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 609         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 610             return 'Skipping "%s" because it is age restricted' % video_title
 611         if self.in_download_archive(info_dict):
 612             return '%s has already been recorded in archive' % video_title
 613
 614         if not incomplete:
 615             match_filter = self.params.get('match_filter')
 616             if match_filter is not None:
 617                 ret = match_filter(info_dict)
 618                 if ret is not None:
 619                     return ret
 620
 621         return None
 622
 623     @staticmethod
 624     def add_extra_info(info_dict, extra_info):
 625         '''Set the keys from extra_info in info dict if they are missing'''
 626         for key, value in extra_info.items():
 627             info_dict.setdefault(key, value)
 628
 629     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 630                      process=True):
 631         '''
 632         Returns a list with a dictionary for each video we find.
 633         If 'download', also downloads the videos.
 634         extra_info is a dict containing the extra values to add to each result
 635         '''
 636
 637         if not ie_key and self._force_generic_extractor_required:
 638             self._force_generic_extractor_required = False
 639             ie_key = 'Generic'
 640
 641         if ie_key:
 642             ies = [self.get_info_extractor(ie_key)]
 643         else:
 644             ies = self._ies
 645
 646         for ie in ies:
 647             if not ie.suitable(url):
 648                 continue
 649
 650             if not ie.working():
 651                 self.report_warning('The program functionality for this site has been marked as broken, '
 652                                     'and will probably not work.')
 653
 654             try:
 655                 ie_result = ie.extract(url)
 656                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 657                     break
 658                 if isinstance(ie_result, list):
 659                     # Backwards compatibility: old IE result format
 660                     ie_result = {
 661                         '_type': 'compat_list',
 662                         'entries': ie_result,
 663                     }
 664                 self.add_default_extra_info(ie_result, ie, url)
 665                 if process:
 666                     return self.process_ie_result(ie_result, download, extra_info)
 667                 else:
 668                     return ie_result
 669             except ExtractorError as de:  # An error we somewhat expected
 670                 self.report_error(compat_str(de), de.format_traceback())
 671                 break
 672             except MaxDownloadsReached:
 673                 raise
 674             except Exception as e:
 675                 if self.params.get('ignoreerrors', False):
 676                     self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
 677                     break
 678                 else:
 679                     raise
 680         else:
 681             self.report_error('no suitable InfoExtractor for URL %s' % url)
 682
 683     def add_default_extra_info(self, ie_result, ie, url):
 684         self.add_extra_info(ie_result, {
 685             'extractor': ie.IE_NAME,
 686             'webpage_url': url,
 687             'webpage_url_basename': url_basename(url),
 688             'extractor_key': ie.ie_key(),
 689         })
 690
 691     def process_ie_result(self, ie_result, download=True, extra_info={}):
 692         """
 693         Take the result of the ie(may be modified) and resolve all unresolved
 694         references (URLs, playlist items).
 695
 696         It will also download the videos if 'download'.
 697         Returns the resolved ie_result.
 698         """
 699
 700         result_type = ie_result.get('_type', 'video')
 701
 702         if result_type in ('url', 'url_transparent'):
 703             extract_flat = self.params.get('extract_flat', False)
 704             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 705                     extract_flat is True):
 706                 if self.params.get('forcejson', False):
 707                     self.to_stdout(json.dumps(ie_result))
 708                 return ie_result
 709
 710         if result_type == 'video':
 711             self.add_extra_info(ie_result, extra_info)
 712             return self.process_video_result(ie_result, download=download)
 713         elif result_type == 'url':
 714             # We have to add extra_info to the results because it may be
 715             # contained in a playlist
 716             return self.extract_info(ie_result['url'],
 717                                      download,
 718                                      ie_key=ie_result.get('ie_key'),
 719                                      extra_info=extra_info)
 720         elif result_type == 'url_transparent':
 721             # Use the information from the embedding page
 722             info = self.extract_info(
 723                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 724                 extra_info=extra_info, download=False, process=False)
 725
 726             force_properties = dict(
 727                 (k, v) for k, v in ie_result.items() if v is not None)
 728             for f in ('_type', 'url'):
 729                 if f in force_properties:
 730                     del force_properties[f]
 731             new_result = info.copy()
 732             new_result.update(force_properties)
 733
 734             assert new_result.get('_type') != 'url_transparent'
 735
 736             return self.process_ie_result(
 737                 new_result, download=download, extra_info=extra_info)
 738         elif result_type == 'playlist' or result_type == 'multi_video':
 739             # We process each entry in the playlist
 740             playlist = ie_result.get('title', None) or ie_result.get('id', None)
 741             self.to_screen('[download] Downloading playlist: %s' % playlist)
 742
 743             playlist_results = []
 744
 745             playliststart = self.params.get('playliststart', 1) - 1
 746             playlistend = self.params.get('playlistend', None)
 747             # For backwards compatibility, interpret -1 as whole list
 748             if playlistend == -1:
 749                 playlistend = None
 750
 751             playlistitems_str = self.params.get('playlist_items', None)
 752             playlistitems = None
 753             if playlistitems_str is not None:
 754                 def iter_playlistitems(format):
 755                     for string_segment in format.split(','):
 756                         if '-' in string_segment:
 757                             start, end = string_segment.split('-')
 758                             for item in range(int(start), int(end) + 1):
 759                                 yield int(item)
 760                         else:
 761                             yield int(string_segment)
 762                 playlistitems = iter_playlistitems(playlistitems_str)
 763
 764             ie_entries = ie_result['entries']
 765             if isinstance(ie_entries, list):
 766                 n_all_entries = len(ie_entries)
 767                 if playlistitems:
 768                     entries = [
 769                         ie_entries[i - 1] for i in playlistitems
 770                         if -n_all_entries <= i - 1 < n_all_entries]
 771                 else:
 772                     entries = ie_entries[playliststart:playlistend]
 773                 n_entries = len(entries)
 774                 self.to_screen(
 775                     "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
 776                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 777             elif isinstance(ie_entries, PagedList):
 778                 if playlistitems:
 779                     entries = []
 780                     for item in playlistitems:
 781                         entries.extend(ie_entries.getslice(
 782                             item - 1, item
 783                         ))
 784                 else:
 785                     entries = ie_entries.getslice(
 786                         playliststart, playlistend)
 787                 n_entries = len(entries)
 788                 self.to_screen(
 789                     "[%s] playlist %s: Downloading %d videos" %
 790                     (ie_result['extractor'], playlist, n_entries))
 791             else:  # iterable
 792                 if playlistitems:
 793                     entry_list = list(ie_entries)
 794                     entries = [entry_list[i - 1] for i in playlistitems]
 795                 else:
 796                     entries = list(itertools.islice(
 797                         ie_entries, playliststart, playlistend))
 798                 n_entries = len(entries)
 799                 self.to_screen(
 800                     "[%s] playlist %s: Downloading %d videos" %
 801                     (ie_result['extractor'], playlist, n_entries))
 802
 803             if self.params.get('playlistreverse', False):
 804                 entries = entries[::-1]
 805
 806             for i, entry in enumerate(entries, 1):
 807                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 808                 extra = {
 809                     'n_entries': n_entries,
 810                     'playlist': playlist,
 811                     'playlist_id': ie_result.get('id'),
 812                     'playlist_title': ie_result.get('title'),
 813                     'playlist_index': i + playliststart,
 814                     'extractor': ie_result['extractor'],
 815                     'webpage_url': ie_result['webpage_url'],
 816                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 817                     'extractor_key': ie_result['extractor_key'],
 818                 }
 819
 820                 reason = self._match_entry(entry, incomplete=True)
 821                 if reason is not None:
 822                     self.to_screen('[download] ' + reason)
 823                     continue
 824
 825                 entry_result = self.process_ie_result(entry,
 826                                                       download=download,
 827                                                       extra_info=extra)
 828                 playlist_results.append(entry_result)
 829             ie_result['entries'] = playlist_results
 830             return ie_result
 831         elif result_type == 'compat_list':
 832             self.report_warning(
 833                 'Extractor %s returned a compat_list result. '
 834                 'It needs to be updated.' % ie_result.get('extractor'))
 835
 836             def _fixup(r):
 837                 self.add_extra_info(
 838                     r,
 839                     {
 840                         'extractor': ie_result['extractor'],
 841                         'webpage_url': ie_result['webpage_url'],
 842                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 843                         'extractor_key': ie_result['extractor_key'],
 844                     }
 845                 )
 846                 return r
 847             ie_result['entries'] = [
 848                 self.process_ie_result(_fixup(r), download, extra_info)
 849                 for r in ie_result['entries']
 850             ]
 851             return ie_result
 852         else:
 853             raise Exception('Invalid result type: %s' % result_type)
 854
 855     def _apply_format_filter(self, format_spec, available_formats):
 856         " Returns a tuple of the remaining format_spec and filtered formats "
 857
 858         OPERATORS = {
 859             '<': operator.lt,
 860             '<=': operator.le,
 861             '>': operator.gt,
 862             '>=': operator.ge,
 863             '=': operator.eq,
 864             '!=': operator.ne,
 865         }
 866         operator_rex = re.compile(r'''(?x)\s*\[
 867             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
 868             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
 869             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
 870             \]$
 871             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
 872         m = operator_rex.search(format_spec)
 873         if m:
 874             try:
 875                 comparison_value = int(m.group('value'))
 876             except ValueError:
 877                 comparison_value = parse_filesize(m.group('value'))
 878                 if comparison_value is None:
 879                     comparison_value = parse_filesize(m.group('value') + 'B')
 880                 if comparison_value is None:
 881                     raise ValueError(
 882                         'Invalid value %r in format specification %r' % (
 883                             m.group('value'), format_spec))
 884             op = OPERATORS[m.group('op')]
 885
 886         if not m:
 887             STR_OPERATORS = {
 888                 '=': operator.eq,
 889                 '!=': operator.ne,
 890             }
 891             str_operator_rex = re.compile(r'''(?x)\s*\[
 892                 \s*(?P<key>ext|acodec|vcodec|container|protocol)
 893                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
 894                 \s*(?P<value>[a-zA-Z0-9_-]+)
 895                 \s*\]$
 896                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
 897             m = str_operator_rex.search(format_spec)
 898             if m:
 899                 comparison_value = m.group('value')
 900                 op = STR_OPERATORS[m.group('op')]
 901
 902         if not m:
 903             raise ValueError('Invalid format specification %r' % format_spec)
 904
 905         def _filter(f):
 906             actual_value = f.get(m.group('key'))
 907             if actual_value is None:
 908                 return m.group('none_inclusive')
 909             return op(actual_value, comparison_value)
 910         new_formats = [f for f in available_formats if _filter(f)]
 911
 912         new_format_spec = format_spec[:-len(m.group(0))]
 913         if not new_format_spec:
 914             new_format_spec = 'best'
 915
 916         return (new_format_spec, new_formats)
 917
 918     def select_format(self, format_spec, available_formats):
 919         while format_spec.endswith(']'):
 920             format_spec, available_formats = self._apply_format_filter(
 921                 format_spec, available_formats)
 922         if not available_formats:
 923             return None
 924
 925         if format_spec in ['best', 'worst', None]:
 926             format_idx = 0 if format_spec == 'worst' else -1
 927             audiovideo_formats = [
 928                 f for f in available_formats
 929                 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
 930             if audiovideo_formats:
 931                 return audiovideo_formats[format_idx]
 932             # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
 933             elif (all(f.get('acodec') != 'none' for f in available_formats) or
 934                   all(f.get('vcodec') != 'none' for f in available_formats)):
 935                 return available_formats[format_idx]
 936         elif format_spec == 'bestaudio':
 937             audio_formats = [
 938                 f for f in available_formats
 939                 if f.get('vcodec') == 'none']
 940             if audio_formats:
 941                 return audio_formats[-1]
 942         elif format_spec == 'worstaudio':
 943             audio_formats = [
 944                 f for f in available_formats
 945                 if f.get('vcodec') == 'none']
 946             if audio_formats:
 947                 return audio_formats[0]
 948         elif format_spec == 'bestvideo':
 949             video_formats = [
 950                 f for f in available_formats
 951                 if f.get('acodec') == 'none']
 952             if video_formats:
 953                 return video_formats[-1]
 954         elif format_spec == 'worstvideo':
 955             video_formats = [
 956                 f for f in available_formats
 957                 if f.get('acodec') == 'none']
 958             if video_formats:
 959                 return video_formats[0]
 960         else:
 961             extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
 962             if format_spec in extensions:
 963                 filter_f = lambda f: f['ext'] == format_spec
 964             else:
 965                 filter_f = lambda f: f['format_id'] == format_spec
 966             matches = list(filter(filter_f, available_formats))
 967             if matches:
 968                 return matches[-1]
 969         return None
 970
 971     def _calc_headers(self, info_dict):
 972         res = std_headers.copy()
 973
 974         add_headers = info_dict.get('http_headers')
 975         if add_headers:
 976             res.update(add_headers)
 977
 978         cookies = self._calc_cookies(info_dict)
 979         if cookies:
 980             res['Cookie'] = cookies
 981
 982         return res
 983
 984     def _calc_cookies(self, info_dict):
 985         pr = compat_urllib_request.Request(info_dict['url'])
 986         self.cookiejar.add_cookie_header(pr)
 987         return pr.get_header('Cookie')
 988
 989     def process_video_result(self, info_dict, download=True):
 990         assert info_dict.get('_type', 'video') == 'video'
 991
 992         if 'id' not in info_dict:
 993             raise ExtractorError('Missing "id" field in extractor result')
 994         if 'title' not in info_dict:
 995             raise ExtractorError('Missing "title" field in extractor result')
 996
 997         if 'playlist' not in info_dict:
 998             # It isn't part of a playlist
 999             info_dict['playlist'] = None
1000             info_dict['playlist_index'] = None
1001
1002         thumbnails = info_dict.get('thumbnails')
1003         if thumbnails is None:
1004             thumbnail = info_dict.get('thumbnail')
1005             if thumbnail:
1006                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1007         if thumbnails:
1008             thumbnails.sort(key=lambda t: (
1009                 t.get('preference'), t.get('width'), t.get('height'),
1010                 t.get('id'), t.get('url')))
1011             for i, t in enumerate(thumbnails):
1012                 if 'width' in t and 'height' in t:
1013                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1014                 if t.get('id') is None:
1015                     t['id'] = '%d' % i
1016
1017         if thumbnails and 'thumbnail' not in info_dict:
1018             info_dict['thumbnail'] = thumbnails[-1]['url']
1019
1020         if 'display_id' not in info_dict and 'id' in info_dict:
1021             info_dict['display_id'] = info_dict['id']
1022
1023         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1024             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1025             # see http://bugs.python.org/issue1646728)
1026             try:
1027                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1028                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1029             except (ValueError, OverflowError, OSError):
1030                 pass
1031
1032         if self.params.get('listsubtitles', False):
1033             if 'automatic_captions' in info_dict:
1034                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1035             self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
1036             return
1037         info_dict['requested_subtitles'] = self.process_subtitles(
1038             info_dict['id'], info_dict.get('subtitles'),
1039             info_dict.get('automatic_captions'))
1040
1041         # This extractors handle format selection themselves
1042         if info_dict['extractor'] in ['Youku']:
1043             if download:
1044                 self.process_info(info_dict)
1045             return info_dict
1046
1047         # We now pick which formats have to be downloaded
1048         if info_dict.get('formats') is None:
1049             # There's only one format available
1050             formats = [info_dict]
1051         else:
1052             formats = info_dict['formats']
1053
1054         if not formats:
1055             raise ExtractorError('No video formats found!')
1056
1057         formats_dict = {}
1058
1059         # We check that all the formats have the format and format_id fields
1060         for i, format in enumerate(formats):
1061             if 'url' not in format:
1062                 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1063
1064             if format.get('format_id') is None:
1065                 format['format_id'] = compat_str(i)
1066             format_id = format['format_id']
1067             if format_id not in formats_dict:
1068                 formats_dict[format_id] = []
1069             formats_dict[format_id].append(format)
1070
1071         # Make sure all formats have unique format_id
1072         for format_id, ambiguous_formats in formats_dict.items():
1073             if len(ambiguous_formats) > 1:
1074                 for i, format in enumerate(ambiguous_formats):
1075                     format['format_id'] = '%s-%d' % (format_id, i)
1076
1077         for i, format in enumerate(formats):
1078             if format.get('format') is None:
1079                 format['format'] = '{id} - {res}{note}'.format(
1080                     id=format['format_id'],
1081                     res=self.format_resolution(format),
1082                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1083                 )
1084             # Automatically determine file extension if missing
1085             if 'ext' not in format:
1086                 format['ext'] = determine_ext(format['url']).lower()
1087             # Add HTTP headers, so that external programs can use them from the
1088             # json output
1089             full_format_info = info_dict.copy()
1090             full_format_info.update(format)
1091             format['http_headers'] = self._calc_headers(full_format_info)
1092
1093         # TODO Central sorting goes here
1094
1095         if formats[0] is not info_dict:
1096             # only set the 'formats' fields if the original info_dict list them
1097             # otherwise we end up with a circular reference, the first (and unique)
1098             # element in the 'formats' field in info_dict is info_dict itself,
1099             # wich can't be exported to json
1100             info_dict['formats'] = formats
1101         if self.params.get('listformats'):
1102             self.list_formats(info_dict)
1103             return
1104         if self.params.get('list_thumbnails'):
1105             self.list_thumbnails(info_dict)
1106             return
1107
1108         req_format = self.params.get('format')
1109         if req_format is None:
1110             req_format_list = []
1111             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1112                     info_dict['extractor'] in ['youtube', 'ted']):
1113                 merger = FFmpegMergerPP(self)
1114                 if merger.available and merger.can_merge():
1115                     req_format_list.append('bestvideo+bestaudio')
1116             req_format_list.append('best')
1117             req_format = '/'.join(req_format_list)
1118         formats_to_download = []
1119         if req_format == 'all':
1120             formats_to_download = formats
1121         else:
1122             for rfstr in req_format.split(','):
1123                 # We can accept formats requested in the format: 34/5/best, we pick
1124                 # the first that is available, starting from left
1125                 req_formats = rfstr.split('/')
1126                 for rf in req_formats:
1127                     if re.match(r'.+?\+.+?', rf) is not None:
1128                         # Two formats have been requested like '137+139'
1129                         format_1, format_2 = rf.split('+')
1130                         formats_info = (self.select_format(format_1, formats),
1131                                         self.select_format(format_2, formats))
1132                         if all(formats_info):
1133                             # The first format must contain the video and the
1134                             # second the audio
1135                             if formats_info[0].get('vcodec') == 'none':
1136                                 self.report_error('The first format must '
1137                                                   'contain the video, try using '
1138                                                   '"-f %s+%s"' % (format_2, format_1))
1139                                 return
1140                             output_ext = (
1141                                 formats_info[0]['ext']
1142                                 if self.params.get('merge_output_format') is None
1143                                 else self.params['merge_output_format'])
1144                             selected_format = {
1145                                 'requested_formats': formats_info,
1146                                 'format': '%s+%s' % (formats_info[0].get('format'),
1147                                                      formats_info[1].get('format')),
1148                                 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1149                                                         formats_info[1].get('format_id')),
1150                                 'width': formats_info[0].get('width'),
1151                                 'height': formats_info[0].get('height'),
1152                                 'resolution': formats_info[0].get('resolution'),
1153                                 'fps': formats_info[0].get('fps'),
1154                                 'vcodec': formats_info[0].get('vcodec'),
1155                                 'vbr': formats_info[0].get('vbr'),
1156                                 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1157                                 'acodec': formats_info[1].get('acodec'),
1158                                 'abr': formats_info[1].get('abr'),
1159                                 'ext': output_ext,
1160                             }
1161                         else:
1162                             selected_format = None
1163                     else:
1164                         selected_format = self.select_format(rf, formats)
1165                     if selected_format is not None:
1166                         formats_to_download.append(selected_format)
1167                         break
1168         if not formats_to_download:
1169             raise ExtractorError('requested format not available',
1170                                  expected=True)
1171
1172         if download:
1173             if len(formats_to_download) > 1:
1174                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1175             for format in formats_to_download:
1176                 new_info = dict(info_dict)
1177                 new_info.update(format)
1178                 self.process_info(new_info)
1179         # We update the info dict with the best quality format (backwards compatibility)
1180         info_dict.update(formats_to_download[-1])
1181         return info_dict
1182
1183     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1184         """Select the requested subtitles and their format"""
1185         available_subs = {}
1186         if normal_subtitles and self.params.get('writesubtitles'):
1187             available_subs.update(normal_subtitles)
1188         if automatic_captions and self.params.get('writeautomaticsub'):
1189             for lang, cap_info in automatic_captions.items():
1190                 if lang not in available_subs:
1191                     available_subs[lang] = cap_info
1192
1193         if (not self.params.get('writesubtitles') and not
1194                 self.params.get('writeautomaticsub') or not
1195                 available_subs):
1196             return None
1197
1198         if self.params.get('allsubtitles', False):
1199             requested_langs = available_subs.keys()
1200         else:
1201             if self.params.get('subtitleslangs', False):
1202                 requested_langs = self.params.get('subtitleslangs')
1203             elif 'en' in available_subs:
1204                 requested_langs = ['en']
1205             else:
1206                 requested_langs = [list(available_subs.keys())[0]]
1207
1208         formats_query = self.params.get('subtitlesformat', 'best')
1209         formats_preference = formats_query.split('/') if formats_query else []
1210         subs = {}
1211         for lang in requested_langs:
1212             formats = available_subs.get(lang)
1213             if formats is None:
1214                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1215                 continue
1216             for ext in formats_preference:
1217                 if ext == 'best':
1218                     f = formats[-1]
1219                     break
1220                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1221                 if matches:
1222                     f = matches[-1]
1223                     break
1224             else:
1225                 f = formats[-1]
1226                 self.report_warning(
1227                     'No subtitle format found matching "%s" for language %s, '
1228                     'using %s' % (formats_query, lang, f['ext']))
1229             subs[lang] = f
1230         return subs
1231
1232     def process_info(self, info_dict):
1233         """Process a single resolved IE result."""
1234
1235         assert info_dict.get('_type', 'video') == 'video'
1236
1237         max_downloads = self.params.get('max_downloads')
1238         if max_downloads is not None:
1239             if self._num_downloads >= int(max_downloads):
1240                 raise MaxDownloadsReached()
1241
1242         info_dict['fulltitle'] = info_dict['title']
1243         if len(info_dict['title']) > 200:
1244             info_dict['title'] = info_dict['title'][:197] + '...'
1245
1246         if 'format' not in info_dict:
1247             info_dict['format'] = info_dict['ext']
1248
1249         reason = self._match_entry(info_dict, incomplete=False)
1250         if reason is not None:
1251             self.to_screen('[download] ' + reason)
1252             return
1253
1254         self._num_downloads += 1
1255
1256         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1257
1258         # Forced printings
1259         if self.params.get('forcetitle', False):
1260             self.to_stdout(info_dict['fulltitle'])
1261         if self.params.get('forceid', False):
1262             self.to_stdout(info_dict['id'])
1263         if self.params.get('forceurl', False):
1264             if info_dict.get('requested_formats') is not None:
1265                 for f in info_dict['requested_formats']:
1266                     self.to_stdout(f['url'] + f.get('play_path', ''))
1267             else:
1268                 # For RTMP URLs, also include the playpath
1269                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1270         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1271             self.to_stdout(info_dict['thumbnail'])
1272         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1273             self.to_stdout(info_dict['description'])
1274         if self.params.get('forcefilename', False) and filename is not None:
1275             self.to_stdout(filename)
1276         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1277             self.to_stdout(formatSeconds(info_dict['duration']))
1278         if self.params.get('forceformat', False):
1279             self.to_stdout(info_dict['format'])
1280         if self.params.get('forcejson', False):
1281             self.to_stdout(json.dumps(info_dict))
1282
1283         # Do nothing else if in simulate mode
1284         if self.params.get('simulate', False):
1285             return
1286
1287         if filename is None:
1288             return
1289
1290         try:
1291             dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1292             if dn and not os.path.exists(dn):
1293                 os.makedirs(dn)
1294         except (OSError, IOError) as err:
1295             self.report_error('unable to create directory ' + compat_str(err))
1296             return
1297
1298         if self.params.get('writedescription', False):
1299             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1300             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1301                 self.to_screen('[info] Video description is already present')
1302             elif info_dict.get('description') is None:
1303                 self.report_warning('There\'s no description to write.')
1304             else:
1305                 try:
1306                     self.to_screen('[info] Writing video description to: ' + descfn)
1307                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1308                         descfile.write(info_dict['description'])
1309                 except (OSError, IOError):
1310                     self.report_error('Cannot write description file ' + descfn)
1311                     return
1312
1313         if self.params.get('writeannotations', False):
1314             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1315             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1316                 self.to_screen('[info] Video annotations are already present')
1317             else:
1318                 try:
1319                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1320                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1321                         annofile.write(info_dict['annotations'])
1322                 except (KeyError, TypeError):
1323                     self.report_warning('There are no annotations to write.')
1324                 except (OSError, IOError):
1325                     self.report_error('Cannot write annotations file: ' + annofn)
1326                     return
1327
1328         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1329                                        self.params.get('writeautomaticsub')])
1330
1331         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1332             # subtitles download errors are already managed as troubles in relevant IE
1333             # that way it will silently go on when used with unsupporting IE
1334             subtitles = info_dict['requested_subtitles']
1335             ie = self.get_info_extractor(info_dict['extractor_key'])
1336             for sub_lang, sub_info in subtitles.items():
1337                 sub_format = sub_info['ext']
1338                 if sub_info.get('data') is not None:
1339                     sub_data = sub_info['data']
1340                 else:
1341                     try:
1342                         sub_data = ie._download_webpage(
1343                             sub_info['url'], info_dict['id'], note=False)
1344                     except ExtractorError as err:
1345                         self.report_warning('Unable to download subtitle for "%s": %s' %
1346                                             (sub_lang, compat_str(err.cause)))
1347                         continue
1348                 try:
1349                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1350                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1351                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1352                     else:
1353                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1354                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1355                             subfile.write(sub_data)
1356                 except (OSError, IOError):
1357                     self.report_error('Cannot write subtitles file ' + sub_filename)
1358                     return
1359
1360         if self.params.get('writeinfojson', False):
1361             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1362             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1363                 self.to_screen('[info] Video description metadata is already present')
1364             else:
1365                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1366                 try:
1367                     write_json_file(self.filter_requested_info(info_dict), infofn)
1368                 except (OSError, IOError):
1369                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1370                     return
1371
1372         self._write_thumbnails(info_dict, filename)
1373
1374         if not self.params.get('skip_download', False):
1375             try:
1376                 def dl(name, info):
1377                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1378                     for ph in self._progress_hooks:
1379                         fd.add_progress_hook(ph)
1380                     if self.params.get('verbose'):
1381                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1382                     return fd.download(name, info)
1383
1384                 if info_dict.get('requested_formats') is not None:
1385                     downloaded = []
1386                     success = True
1387                     merger = FFmpegMergerPP(self)
1388                     if not merger.available:
1389                         postprocessors = []
1390                         self.report_warning('You have requested multiple '
1391                                             'formats but ffmpeg or avconv are not installed.'
1392                                             ' The formats won\'t be merged.')
1393                     else:
1394                         postprocessors = [merger]
1395
1396                     def compatible_formats(formats):
1397                         video, audio = formats
1398                         # Check extension
1399                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1400                         if video_ext and audio_ext:
1401                             COMPATIBLE_EXTS = (
1402                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1403                                 ('webm')
1404                             )
1405                             for exts in COMPATIBLE_EXTS:
1406                                 if video_ext in exts and audio_ext in exts:
1407                                     return True
1408                         # TODO: Check acodec/vcodec
1409                         return False
1410
1411                     filename_real_ext = os.path.splitext(filename)[1][1:]
1412                     filename_wo_ext = (
1413                         os.path.splitext(filename)[0]
1414                         if filename_real_ext == info_dict['ext']
1415                         else filename)
1416                     requested_formats = info_dict['requested_formats']
1417                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1418                         info_dict['ext'] = 'mkv'
1419                         self.report_warning(
1420                             'Requested formats are incompatible for merge and will be merged into mkv.')
1421                     # Ensure filename always has a correct extension for successful merge
1422                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1423                     if os.path.exists(encodeFilename(filename)):
1424                         self.to_screen(
1425                             '[download] %s has already been downloaded and '
1426                             'merged' % filename)
1427                     else:
1428                         for f in requested_formats:
1429                             new_info = dict(info_dict)
1430                             new_info.update(f)
1431                             fname = self.prepare_filename(new_info)
1432                             fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1433                             downloaded.append(fname)
1434                             partial_success = dl(fname, new_info)
1435                             success = success and partial_success
1436                         info_dict['__postprocessors'] = postprocessors
1437                         info_dict['__files_to_merge'] = downloaded
1438                 else:
1439                     # Just a single file
1440                     success = dl(filename, info_dict)
1441             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1442                 self.report_error('unable to download video data: %s' % str(err))
1443                 return
1444             except (OSError, IOError) as err:
1445                 raise UnavailableVideoError(err)
1446             except (ContentTooShortError, ) as err:
1447                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1448                 return
1449
1450             if success:
1451                 # Fixup content
1452                 fixup_policy = self.params.get('fixup')
1453                 if fixup_policy is None:
1454                     fixup_policy = 'detect_or_warn'
1455
1456                 stretched_ratio = info_dict.get('stretched_ratio')
1457                 if stretched_ratio is not None and stretched_ratio != 1:
1458                     if fixup_policy == 'warn':
1459                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1460                             info_dict['id'], stretched_ratio))
1461                     elif fixup_policy == 'detect_or_warn':
1462                         stretched_pp = FFmpegFixupStretchedPP(self)
1463                         if stretched_pp.available:
1464                             info_dict.setdefault('__postprocessors', [])
1465                             info_dict['__postprocessors'].append(stretched_pp)
1466                         else:
1467                             self.report_warning(
1468                                 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1469                                     info_dict['id'], stretched_ratio))
1470                     else:
1471                         assert fixup_policy in ('ignore', 'never')
1472
1473                 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1474                     if fixup_policy == 'warn':
1475                         self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1476                             info_dict['id']))
1477                     elif fixup_policy == 'detect_or_warn':
1478                         fixup_pp = FFmpegFixupM4aPP(self)
1479                         if fixup_pp.available:
1480                             info_dict.setdefault('__postprocessors', [])
1481                             info_dict['__postprocessors'].append(fixup_pp)
1482                         else:
1483                             self.report_warning(
1484                                 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1485                                     info_dict['id']))
1486                     else:
1487                         assert fixup_policy in ('ignore', 'never')
1488
1489                 try:
1490                     self.post_process(filename, info_dict)
1491                 except (PostProcessingError) as err:
1492                     self.report_error('postprocessing: %s' % str(err))
1493                     return
1494                 self.record_download_archive(info_dict)
1495
1496     def download(self, url_list):
1497         """Download a given list of URLs."""
1498         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1499         if (len(url_list) > 1 and
1500                 '%' not in outtmpl and
1501                 self.params.get('max_downloads') != 1):
1502             raise SameFileError(outtmpl)
1503
1504         for url in url_list:
1505             try:
1506                 self._force_generic_extractor_required = self.params.get('force_generic_extractor', False)
1507                 # It also downloads the videos
1508                 res = self.extract_info(url)
1509             except UnavailableVideoError:
1510                 self.report_error('unable to download video')
1511             except MaxDownloadsReached:
1512                 self.to_screen('[info] Maximum number of downloaded files reached.')
1513                 raise
1514             else:
1515                 if self.params.get('dump_single_json', False):
1516                     self.to_stdout(json.dumps(res))
1517
1518         return self._download_retcode
1519
1520     def download_with_info_file(self, info_filename):
1521         with contextlib.closing(fileinput.FileInput(
1522                 [info_filename], mode='r',
1523                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1524             # FileInput doesn't have a read method, we can't call json.load
1525             info = self.filter_requested_info(json.loads('\n'.join(f)))
1526         try:
1527             self.process_ie_result(info, download=True)
1528         except DownloadError:
1529             webpage_url = info.get('webpage_url')
1530             if webpage_url is not None:
1531                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1532                 return self.download([webpage_url])
1533             else:
1534                 raise
1535         return self._download_retcode
1536
1537     @staticmethod
1538     def filter_requested_info(info_dict):
1539         return dict(
1540             (k, v) for k, v in info_dict.items()
1541             if k not in ['requested_formats', 'requested_subtitles'])
1542
1543     def post_process(self, filename, ie_info):
1544         """Run all the postprocessors on the given file."""
1545         info = dict(ie_info)
1546         info['filepath'] = filename
1547         pps_chain = []
1548         if ie_info.get('__postprocessors') is not None:
1549             pps_chain.extend(ie_info['__postprocessors'])
1550         pps_chain.extend(self._pps)
1551         for pp in pps_chain:
1552             files_to_delete = []
1553             try:
1554                 files_to_delete, info = pp.run(info)
1555             except PostProcessingError as e:
1556                 self.report_error(e.msg)
1557             if files_to_delete and not self.params.get('keepvideo', False):
1558                 for old_filename in files_to_delete:
1559                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1560                     try:
1561                         os.remove(encodeFilename(old_filename))
1562                     except (IOError, OSError):
1563                         self.report_warning('Unable to remove downloaded original file')
1564
1565     def _make_archive_id(self, info_dict):
1566         # Future-proof against any change in case
1567         # and backwards compatibility with prior versions
1568         extractor = info_dict.get('extractor_key')
1569         if extractor is None:
1570             if 'id' in info_dict:
1571                 extractor = info_dict.get('ie_key')  # key in a playlist
1572         if extractor is None:
1573             return None  # Incomplete video information
1574         return extractor.lower() + ' ' + info_dict['id']
1575
1576     def in_download_archive(self, info_dict):
1577         fn = self.params.get('download_archive')
1578         if fn is None:
1579             return False
1580
1581         vid_id = self._make_archive_id(info_dict)
1582         if vid_id is None:
1583             return False  # Incomplete video information
1584
1585         try:
1586             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1587                 for line in archive_file:
1588                     if line.strip() == vid_id:
1589                         return True
1590         except IOError as ioe:
1591             if ioe.errno != errno.ENOENT:
1592                 raise
1593         return False
1594
1595     def record_download_archive(self, info_dict):
1596         fn = self.params.get('download_archive')
1597         if fn is None:
1598             return
1599         vid_id = self._make_archive_id(info_dict)
1600         assert vid_id
1601         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1602             archive_file.write(vid_id + '\n')
1603
1604     @staticmethod
1605     def format_resolution(format, default='unknown'):
1606         if format.get('vcodec') == 'none':
1607             return 'audio only'
1608         if format.get('resolution') is not None:
1609             return format['resolution']
1610         if format.get('height') is not None:
1611             if format.get('width') is not None:
1612                 res = '%sx%s' % (format['width'], format['height'])
1613             else:
1614                 res = '%sp' % format['height']
1615         elif format.get('width') is not None:
1616             res = '?x%d' % format['width']
1617         else:
1618             res = default
1619         return res
1620
1621     def _format_note(self, fdict):
1622         res = ''
1623         if fdict.get('ext') in ['f4f', 'f4m']:
1624             res += '(unsupported) '
1625         if fdict.get('format_note') is not None:
1626             res += fdict['format_note'] + ' '
1627         if fdict.get('tbr') is not None:
1628             res += '%4dk ' % fdict['tbr']
1629         if fdict.get('container') is not None:
1630             if res:
1631                 res += ', '
1632             res += '%s container' % fdict['container']
1633         if (fdict.get('vcodec') is not None and
1634                 fdict.get('vcodec') != 'none'):
1635             if res:
1636                 res += ', '
1637             res += fdict['vcodec']
1638             if fdict.get('vbr') is not None:
1639                 res += '@'
1640         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1641             res += 'video@'
1642         if fdict.get('vbr') is not None:
1643             res += '%4dk' % fdict['vbr']
1644         if fdict.get('fps') is not None:
1645             res += ', %sfps' % fdict['fps']
1646         if fdict.get('acodec') is not None:
1647             if res:
1648                 res += ', '
1649             if fdict['acodec'] == 'none':
1650                 res += 'video only'
1651             else:
1652                 res += '%-5s' % fdict['acodec']
1653         elif fdict.get('abr') is not None:
1654             if res:
1655                 res += ', '
1656             res += 'audio'
1657         if fdict.get('abr') is not None:
1658             res += '@%3dk' % fdict['abr']
1659         if fdict.get('asr') is not None:
1660             res += ' (%5dHz)' % fdict['asr']
1661         if fdict.get('filesize') is not None:
1662             if res:
1663                 res += ', '
1664             res += format_bytes(fdict['filesize'])
1665         elif fdict.get('filesize_approx') is not None:
1666             if res:
1667                 res += ', '
1668             res += '~' + format_bytes(fdict['filesize_approx'])
1669         return res
1670
1671     def list_formats(self, info_dict):
1672         formats = info_dict.get('formats', [info_dict])
1673         table = [
1674             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1675             for f in formats
1676             if f.get('preference') is None or f['preference'] >= -1000]
1677         if len(formats) > 1:
1678             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1679
1680         header_line = ['format code', 'extension', 'resolution', 'note']
1681         self.to_screen(
1682             '[info] Available formats for %s:\n%s' %
1683             (info_dict['id'], render_table(header_line, table)))
1684
1685     def list_thumbnails(self, info_dict):
1686         thumbnails = info_dict.get('thumbnails')
1687         if not thumbnails:
1688             tn_url = info_dict.get('thumbnail')
1689             if tn_url:
1690                 thumbnails = [{'id': '0', 'url': tn_url}]
1691             else:
1692                 self.to_screen(
1693                     '[info] No thumbnails present for %s' % info_dict['id'])
1694                 return
1695
1696         self.to_screen(
1697             '[info] Thumbnails for %s:' % info_dict['id'])
1698         self.to_screen(render_table(
1699             ['ID', 'width', 'height', 'URL'],
1700             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1701
1702     def list_subtitles(self, video_id, subtitles, name='subtitles'):
1703         if not subtitles:
1704             self.to_screen('%s has no %s' % (video_id, name))
1705             return
1706         self.to_screen(
1707             'Available %s for %s:' % (name, video_id))
1708         self.to_screen(render_table(
1709             ['Language', 'formats'],
1710             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1711                 for lang, formats in subtitles.items()]))
1712
1713     def urlopen(self, req):
1714         """ Start an HTTP download """
1715
1716         # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1717         # always respected by websites, some tend to give out URLs with non percent-encoded
1718         # non-ASCII characters (see telemb.py, ard.py [#3412])
1719         # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1720         # To work around aforementioned issue we will replace request's original URL with
1721         # percent-encoded one
1722         req_is_string = isinstance(req, compat_basestring)
1723         url = req if req_is_string else req.get_full_url()
1724         url_escaped = escape_url(url)
1725
1726         # Substitute URL if any change after escaping
1727         if url != url_escaped:
1728             if req_is_string:
1729                 req = url_escaped
1730             else:
1731                 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
1732                 req = req_type(
1733                     url_escaped, data=req.data, headers=req.headers,
1734                     origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1735
1736         return self._opener.open(req, timeout=self._socket_timeout)
1737
1738     def print_debug_header(self):
1739         if not self.params.get('verbose'):
1740             return
1741
1742         if type('') is not compat_str:
1743             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1744             self.report_warning(
1745                 'Your Python is broken! Update to a newer and supported version')
1746
1747         stdout_encoding = getattr(
1748             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1749         encoding_str = (
1750             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1751                 locale.getpreferredencoding(),
1752                 sys.getfilesystemencoding(),
1753                 stdout_encoding,
1754                 self.get_encoding()))
1755         write_string(encoding_str, encoding=None)
1756
1757         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1758         try:
1759             sp = subprocess.Popen(
1760                 ['git', 'rev-parse', '--short', 'HEAD'],
1761                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1762                 cwd=os.path.dirname(os.path.abspath(__file__)))
1763             out, err = sp.communicate()
1764             out = out.decode().strip()
1765             if re.match('[0-9a-f]+', out):
1766                 self._write_string('[debug] Git HEAD: ' + out + '\n')
1767         except Exception:
1768             try:
1769                 sys.exc_clear()
1770             except Exception:
1771                 pass
1772         self._write_string('[debug] Python version %s - %s\n' % (
1773             platform.python_version(), platform_name()))
1774
1775         exe_versions = FFmpegPostProcessor.get_versions(self)
1776         exe_versions['rtmpdump'] = rtmpdump_version()
1777         exe_str = ', '.join(
1778             '%s %s' % (exe, v)
1779             for exe, v in sorted(exe_versions.items())
1780             if v
1781         )
1782         if not exe_str:
1783             exe_str = 'none'
1784         self._write_string('[debug] exe versions: %s\n' % exe_str)
1785
1786         proxy_map = {}
1787         for handler in self._opener.handlers:
1788             if hasattr(handler, 'proxies'):
1789                 proxy_map.update(handler.proxies)
1790         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1791
1792         if self.params.get('call_home', False):
1793             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1794             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1795             latest_version = self.urlopen(
1796                 'https://yt-dl.org/latest/version').read().decode('utf-8')
1797             if version_tuple(latest_version) > version_tuple(__version__):
1798                 self.report_warning(
1799                     'You are using an outdated version (newest version: %s)! '
1800                     'See https://yt-dl.org/update if you need help updating.' %
1801                     latest_version)
1802
1803     def _setup_opener(self):
1804         timeout_val = self.params.get('socket_timeout')
1805         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1806
1807         opts_cookiefile = self.params.get('cookiefile')
1808         opts_proxy = self.params.get('proxy')
1809
1810         if opts_cookiefile is None:
1811             self.cookiejar = compat_cookiejar.CookieJar()
1812         else:
1813             self.cookiejar = compat_cookiejar.MozillaCookieJar(
1814                 opts_cookiefile)
1815             if os.access(opts_cookiefile, os.R_OK):
1816                 self.cookiejar.load()
1817
1818         cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1819             self.cookiejar)
1820         if opts_proxy is not None:
1821             if opts_proxy == '':
1822                 proxies = {}
1823             else:
1824                 proxies = {'http': opts_proxy, 'https': opts_proxy}
1825         else:
1826             proxies = compat_urllib_request.getproxies()
1827             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1828             if 'http' in proxies and 'https' not in proxies:
1829                 proxies['https'] = proxies['http']
1830         proxy_handler = PerRequestProxyHandler(proxies)
1831
1832         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1833         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1834         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1835         opener = compat_urllib_request.build_opener(
1836             proxy_handler, https_handler, cookie_processor, ydlh)
1837
1838         # Delete the default user-agent header, which would otherwise apply in
1839         # cases where our custom HTTP handler doesn't come into play
1840         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1841         opener.addheaders = []
1842         self._opener = opener
1843
1844     def encode(self, s):
1845         if isinstance(s, bytes):
1846             return s  # Already encoded
1847
1848         try:
1849             return s.encode(self.get_encoding())
1850         except UnicodeEncodeError as err:
1851             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1852             raise
1853
1854     def get_encoding(self):
1855         encoding = self.params.get('encoding')
1856         if encoding is None:
1857             encoding = preferredencoding()
1858         return encoding
1859
1860     def _write_thumbnails(self, info_dict, filename):
1861         if self.params.get('writethumbnail', False):
1862             thumbnails = info_dict.get('thumbnails')
1863             if thumbnails:
1864                 thumbnails = [thumbnails[-1]]
1865         elif self.params.get('write_all_thumbnails', False):
1866             thumbnails = info_dict.get('thumbnails')
1867         else:
1868             return
1869
1870         if not thumbnails:
1871             # No thumbnails present, so return immediately
1872             return
1873
1874         for t in thumbnails:
1875             thumb_ext = determine_ext(t['url'], 'jpg')
1876             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
1877             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
1878             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
1879
1880             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
1881                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
1882                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
1883             else:
1884                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
1885                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
1886                 try:
1887                     uf = self.urlopen(t['url'])
1888                     with open(thumb_filename, 'wb') as thumbf:
1889                         shutil.copyfileobj(uf, thumbf)
1890                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
1891                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
1892                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1893                     self.report_warning('Unable to download thumbnail "%s": %s' %
1894                                         (t['url'], compat_str(err)))