_ Git - youtube-dl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # coding: utf-8
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import contextlib
   8 import copy
   9 import datetime
  10 import errno
  11 import fileinput
  12 import io
  13 import itertools
  14 import json
  15 import locale
  16 import operator
  17 import os
  18 import platform
  19 import re
  20 import shutil
  21 import subprocess
  22 import socket
  23 import sys
  24 import time
  25 import tokenize
  26 import traceback
  27 import random
  28
  29 from .compat import (
  30     compat_basestring,
  31     compat_cookiejar,
  32     compat_expanduser,
  33     compat_get_terminal_size,
  34     compat_http_client,
  35     compat_kwargs,
  36     compat_os_name,
  37     compat_str,
  38     compat_tokenize_tokenize,
  39     compat_urllib_error,
  40     compat_urllib_request,
  41     compat_urllib_request_DataHandler,
  42 )
  43 from .utils import (
  44     age_restricted,
  45     args_to_str,
  46     ContentTooShortError,
  47     date_from_str,
  48     DateRange,
  49     DEFAULT_OUTTMPL,
  50     determine_ext,
  51     determine_protocol,
  52     DownloadError,
  53     encode_compat_str,
  54     encodeFilename,
  55     error_to_compat_str,
  56     ExtractorError,
  57     format_bytes,
  58     formatSeconds,
  59     locked_file,
  60     make_HTTPS_handler,
  61     MaxDownloadsReached,
  62     PagedList,
  63     parse_filesize,
  64     PerRequestProxyHandler,
  65     platform_name,
  66     PostProcessingError,
  67     preferredencoding,
  68     prepend_extension,
  69     register_socks_protocols,
  70     render_table,
  71     replace_extension,
  72     SameFileError,
  73     sanitize_filename,
  74     sanitize_path,
  75     sanitize_url,
  76     sanitized_Request,
  77     std_headers,
  78     subtitles_filename,
  79     UnavailableVideoError,
  80     url_basename,
  81     version_tuple,
  82     write_json_file,
  83     write_string,
  84     YoutubeDLCookieProcessor,
  85     YoutubeDLHandler,
  86 )
  87 from .cache import Cache
  88 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
  89 from .downloader import get_suitable_downloader
  90 from .downloader.rtmp import rtmpdump_version
  91 from .postprocessor import (
  92     FFmpegFixupM3u8PP,
  93     FFmpegFixupM4aPP,
  94     FFmpegFixupStretchedPP,
  95     FFmpegMergerPP,
  96     FFmpegPostProcessor,
  97     get_postprocessor,
  98 )
  99 from .version import __version__
 100
 101 if compat_os_name == 'nt':
 102     import ctypes
 103
 104
 105 class YoutubeDL(object):
 106     """YoutubeDL class.
 107
 108     YoutubeDL objects are the ones responsible of downloading the
 109     actual video file and writing it to disk if the user has requested
 110     it, among some other tasks. In most cases there should be one per
 111     program. As, given a video URL, the downloader doesn't know how to
 112     extract all the needed information, task that InfoExtractors do, it
 113     has to pass the URL to one of them.
 114
 115     For this, YoutubeDL objects have a method that allows
 116     InfoExtractors to be registered in a given order. When it is passed
 117     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 118     finds that reports being able to handle it. The InfoExtractor extracts
 119     all the information about the video or videos the URL refers to, and
 120     YoutubeDL process the extracted information, possibly using a File
 121     Downloader to download the video.
 122
 123     YoutubeDL objects accept a lot of parameters. In order not to saturate
 124     the object constructor with arguments, it receives a dictionary of
 125     options instead. These options are available through the params
 126     attribute for the InfoExtractors to use. The YoutubeDL also
 127     registers itself as the downloader in charge for the InfoExtractors
 128     that are added to it, so this is a "mutual registration".
 129
 130     Available options:
 131
 132     username:          Username for authentication purposes.
 133     password:          Password for authentication purposes.
 134     videopassword:     Password for accessing a video.
 135     ap_mso:            Adobe Pass multiple-system operator identifier.
 136     ap_username:       Multiple-system operator account username.
 137     ap_password:       Multiple-system operator account password.
 138     usenetrc:          Use netrc for authentication instead.
 139     verbose:           Print additional info to stdout.
 140     quiet:             Do not print messages to stdout.
 141     no_warnings:       Do not print out anything for warnings.
 142     forceurl:          Force printing final URL.
 143     forcetitle:        Force printing title.
 144     forceid:           Force printing ID.
 145     forcethumbnail:    Force printing thumbnail URL.
 146     forcedescription:  Force printing description.
 147     forcefilename:     Force printing final filename.
 148     forceduration:     Force printing duration.
 149     forcejson:         Force printing info_dict as JSON.
 150     dump_single_json:  Force printing the info_dict of the whole playlist
 151                        (or video) as a single JSON line.
 152     simulate:          Do not download the video files.
 153     format:            Video format code. See options.py for more information.
 154     outtmpl:           Template for output names.
 155     restrictfilenames: Do not allow "&" and spaces in file names
 156     ignoreerrors:      Do not stop on download errors.
 157     force_generic_extractor: Force downloader to use the generic extractor
 158     nooverwrites:      Prevent overwriting files.
 159     playliststart:     Playlist item to start at.
 160     playlistend:       Playlist item to end at.
 161     playlist_items:    Specific indices of playlist to download.
 162     playlistreverse:   Download playlist items in reverse order.
 163     playlistrandom:    Download playlist items in random order.
 164     matchtitle:        Download only matching titles.
 165     rejecttitle:       Reject downloads for matching titles.
 166     logger:            Log messages to a logging.Logger instance.
 167     logtostderr:       Log messages to stderr instead of stdout.
 168     writedescription:  Write the video description to a .description file
 169     writeinfojson:     Write the video description to a .info.json file
 170     writeannotations:  Write the video annotations to a .annotations.xml file
 171     writethumbnail:    Write the thumbnail image to a file
 172     write_all_thumbnails:  Write all thumbnail formats to files
 173     writesubtitles:    Write the video subtitles to a file
 174     writeautomaticsub: Write the automatically generated subtitles to a file
 175     allsubtitles:      Downloads all the subtitles of the video
 176                        (requires writesubtitles or writeautomaticsub)
 177     listsubtitles:     Lists all available subtitles for the video
 178     subtitlesformat:   The format code for subtitles
 179     subtitleslangs:    List of languages of the subtitles to download
 180     keepvideo:         Keep the video file after post-processing
 181     daterange:         A DateRange object, download only if the upload_date is in the range.
 182     skip_download:     Skip the actual download of the video file
 183     cachedir:          Location of the cache files in the filesystem.
 184                        False to disable filesystem cache.
 185     noplaylist:        Download single video instead of a playlist if in doubt.
 186     age_limit:         An integer representing the user's age in years.
 187                        Unsuitable videos for the given age are skipped.
 188     min_views:         An integer representing the minimum view count the video
 189                        must have in order to not be skipped.
 190                        Videos without view count information are always
 191                        downloaded. None for no limit.
 192     max_views:         An integer representing the maximum view count.
 193                        Videos that are more popular than that are not
 194                        downloaded.
 195                        Videos without view count information are always
 196                        downloaded. None for no limit.
 197     download_archive:  File name of a file where all downloads are recorded.
 198                        Videos already present in the file are not downloaded
 199                        again.
 200     cookiefile:        File name where cookies should be read from and dumped to.
 201     nocheckcertificate:Do not verify SSL certificates
 202     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 203                        At the moment, this is only supported by YouTube.
 204     proxy:             URL of the proxy server to use
 205     geo_verification_proxy:  URL of the proxy to use for IP address verification
 206                        on geo-restricted sites. (Experimental)
 207     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 208     bidi_workaround:   Work around buggy terminals without bidirectional text
 209                        support, using fridibi
 210     debug_printtraffic:Print out sent and received HTTP traffic
 211     include_ads:       Download ads as well
 212     default_search:    Prepend this string if an input url is not valid.
 213                        'auto' for elaborate guessing
 214     encoding:          Use this encoding instead of the system-specified.
 215     extract_flat:      Do not resolve URLs, return the immediate result.
 216                        Pass in 'in_playlist' to only show this behavior for
 217                        playlist items.
 218     postprocessors:    A list of dictionaries, each with an entry
 219                        * key:  The name of the postprocessor. See
 220                                youtube_dl/postprocessor/__init__.py for a list.
 221                        as well as any further keyword arguments for the
 222                        postprocessor.
 223     progress_hooks:    A list of functions that get called on download
 224                        progress, with a dictionary with the entries
 225                        * status: One of "downloading", "error", or "finished".
 226                                  Check this first and ignore unknown values.
 227
 228                        If status is one of "downloading", or "finished", the
 229                        following properties may also be present:
 230                        * filename: The final filename (always present)
 231                        * tmpfilename: The filename we're currently writing to
 232                        * downloaded_bytes: Bytes on disk
 233                        * total_bytes: Size of the whole file, None if unknown
 234                        * total_bytes_estimate: Guess of the eventual file size,
 235                                                None if unavailable.
 236                        * elapsed: The number of seconds since download started.
 237                        * eta: The estimated time in seconds, None if unknown
 238                        * speed: The download speed in bytes/second, None if
 239                                 unknown
 240                        * fragment_index: The counter of the currently
 241                                          downloaded video fragment.
 242                        * fragment_count: The number of fragments (= individual
 243                                          files that will be merged)
 244
 245                        Progress hooks are guaranteed to be called at least once
 246                        (with status "finished") if the download is successful.
 247     merge_output_format: Extension to use when merging formats.
 248     fixup:             Automatically correct known faults of the file.
 249                        One of:
 250                        - "never": do nothing
 251                        - "warn": only emit a warning
 252                        - "detect_or_warn": check whether we can do anything
 253                                            about it, warn otherwise (default)
 254     source_address:    (Experimental) Client-side IP address to bind to.
 255     call_home:         Boolean, true iff we are allowed to contact the
 256                        youtube-dl servers for debugging.
 257     sleep_interval:    Number of seconds to sleep before each download when
 258                        used alone or a lower bound of a range for randomized
 259                        sleep before each download (minimum possible number
 260                        of seconds to sleep) when used along with
 261                        max_sleep_interval.
 262     max_sleep_interval:Upper bound of a range for randomized sleep before each
 263                        download (maximum possible number of seconds to sleep).
 264                        Must only be used along with sleep_interval.
 265                        Actual sleep time will be a random float from range
 266                        [sleep_interval; max_sleep_interval].
 267     listformats:       Print an overview of available video formats and exit.
 268     list_thumbnails:   Print a table of all thumbnails and exit.
 269     match_filter:      A function that gets called with the info_dict of
 270                        every video.
 271                        If it returns a message, the video is ignored.
 272                        If it returns None, the video is downloaded.
 273                        match_filter_func in utils.py is one example for this.
 274     no_color:          Do not emit color codes in output.
 275
 276     The following options determine which downloader is picked:
 277     external_downloader: Executable of the external downloader to call.
 278                        None or unset for standard (built-in) downloader.
 279     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
 280                        if True, otherwise use ffmpeg/avconv if False, otherwise
 281                        use downloader suggested by extractor if None.
 282
 283     The following parameters are not used by YoutubeDL itself, they are used by
 284     the downloader (see youtube_dl/downloader/common.py):
 285     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 286     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 287     xattr_set_filesize, external_downloader_args, hls_use_mpegts.
 288
 289     The following options are used by the post processors:
 290     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 291                        otherwise prefer avconv.
 292     postprocessor_args: A list of additional command-line arguments for the
 293                         postprocessor.
 294     """
 295
 296     params = None
 297     _ies = []
 298     _pps = []
 299     _download_retcode = None
 300     _num_downloads = None
 301     _screen_file = None
 302
 303     def __init__(self, params=None, auto_init=True):
 304         """Create a FileDownloader object with the given options."""
 305         if params is None:
 306             params = {}
 307         self._ies = []
 308         self._ies_instances = {}
 309         self._pps = []
 310         self._progress_hooks = []
 311         self._download_retcode = 0
 312         self._num_downloads = 0
 313         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 314         self._err_file = sys.stderr
 315         self.params = {
 316             # Default parameters
 317             'nocheckcertificate': False,
 318         }
 319         self.params.update(params)
 320         self.cache = Cache(self)
 321
 322         if self.params.get('cn_verification_proxy') is not None:
 323             self.report_warning('--cn-verification-proxy is deprecated. Use --geo-verification-proxy instead.')
 324             if self.params.get('geo_verification_proxy') is None:
 325                 self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
 326
 327         if params.get('bidi_workaround', False):
 328             try:
 329                 import pty
 330                 master, slave = pty.openpty()
 331                 width = compat_get_terminal_size().columns
 332                 if width is None:
 333                     width_args = []
 334                 else:
 335                     width_args = ['-w', str(width)]
 336                 sp_kwargs = dict(
 337                     stdin=subprocess.PIPE,
 338                     stdout=slave,
 339                     stderr=self._err_file)
 340                 try:
 341                     self._output_process = subprocess.Popen(
 342                         ['bidiv'] + width_args, **sp_kwargs
 343                     )
 344                 except OSError:
 345                     self._output_process = subprocess.Popen(
 346                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 347                 self._output_channel = os.fdopen(master, 'rb')
 348             except OSError as ose:
 349                 if ose.errno == errno.ENOENT:
 350                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 351                 else:
 352                     raise
 353
 354         if (sys.version_info >= (3,) and sys.platform != 'win32' and
 355                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
 356                 not params.get('restrictfilenames', False)):
 357             # On Python 3, the Unicode filesystem API will throw errors (#1474)
 358             self.report_warning(
 359                 'Assuming --restrict-filenames since file system encoding '
 360                 'cannot encode all characters. '
 361                 'Set the LC_ALL environment variable to fix this.')
 362             self.params['restrictfilenames'] = True
 363
 364         if isinstance(params.get('outtmpl'), bytes):
 365             self.report_warning(
 366                 'Parameter outtmpl is bytes, but should be a unicode string. '
 367                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 368
 369         self._setup_opener()
 370
 371         if auto_init:
 372             self.print_debug_header()
 373             self.add_default_info_extractors()
 374
 375         for pp_def_raw in self.params.get('postprocessors', []):
 376             pp_class = get_postprocessor(pp_def_raw['key'])
 377             pp_def = dict(pp_def_raw)
 378             del pp_def['key']
 379             pp = pp_class(self, **compat_kwargs(pp_def))
 380             self.add_post_processor(pp)
 381
 382         for ph in self.params.get('progress_hooks', []):
 383             self.add_progress_hook(ph)
 384
 385         register_socks_protocols()
 386
 387     def warn_if_short_id(self, argv):
 388         # short YouTube ID starting with dash?
 389         idxs = [
 390             i for i, a in enumerate(argv)
 391             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 392         if idxs:
 393             correct_argv = (
 394                 ['youtube-dl'] +
 395                 [a for i, a in enumerate(argv) if i not in idxs] +
 396                 ['--'] + [argv[i] for i in idxs]
 397             )
 398             self.report_warning(
 399                 'Long argument string detected. '
 400                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 401                 args_to_str(correct_argv))
 402
 403     def add_info_extractor(self, ie):
 404         """Add an InfoExtractor object to the end of the list."""
 405         self._ies.append(ie)
 406         if not isinstance(ie, type):
 407             self._ies_instances[ie.ie_key()] = ie
 408             ie.set_downloader(self)
 409
 410     def get_info_extractor(self, ie_key):
 411         """
 412         Get an instance of an IE with name ie_key, it will try to get one from
 413         the _ies list, if there's no instance it will create a new one and add
 414         it to the extractor list.
 415         """
 416         ie = self._ies_instances.get(ie_key)
 417         if ie is None:
 418             ie = get_info_extractor(ie_key)()
 419             self.add_info_extractor(ie)
 420         return ie
 421
 422     def add_default_info_extractors(self):
 423         """
 424         Add the InfoExtractors returned by gen_extractors to the end of the list
 425         """
 426         for ie in gen_extractor_classes():
 427             self.add_info_extractor(ie)
 428
 429     def add_post_processor(self, pp):
 430         """Add a PostProcessor object to the end of the chain."""
 431         self._pps.append(pp)
 432         pp.set_downloader(self)
 433
 434     def add_progress_hook(self, ph):
 435         """Add the progress hook (currently only for the file downloader)"""
 436         self._progress_hooks.append(ph)
 437
 438     def _bidi_workaround(self, message):
 439         if not hasattr(self, '_output_channel'):
 440             return message
 441
 442         assert hasattr(self, '_output_process')
 443         assert isinstance(message, compat_str)
 444         line_count = message.count('\n') + 1
 445         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 446         self._output_process.stdin.flush()
 447         res = ''.join(self._output_channel.readline().decode('utf-8')
 448                       for _ in range(line_count))
 449         return res[:-len('\n')]
 450
 451     def to_screen(self, message, skip_eol=False):
 452         """Print message to stdout if not in quiet mode."""
 453         return self.to_stdout(message, skip_eol, check_quiet=True)
 454
 455     def _write_string(self, s, out=None):
 456         write_string(s, out=out, encoding=self.params.get('encoding'))
 457
 458     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 459         """Print message to stdout if not in quiet mode."""
 460         if self.params.get('logger'):
 461             self.params['logger'].debug(message)
 462         elif not check_quiet or not self.params.get('quiet', False):
 463             message = self._bidi_workaround(message)
 464             terminator = ['\n', ''][skip_eol]
 465             output = message + terminator
 466
 467             self._write_string(output, self._screen_file)
 468
 469     def to_stderr(self, message):
 470         """Print message to stderr."""
 471         assert isinstance(message, compat_str)
 472         if self.params.get('logger'):
 473             self.params['logger'].error(message)
 474         else:
 475             message = self._bidi_workaround(message)
 476             output = message + '\n'
 477             self._write_string(output, self._err_file)
 478
 479     def to_console_title(self, message):
 480         if not self.params.get('consoletitle', False):
 481             return
 482         if compat_os_name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 483             # c_wchar_p() might not be necessary if `message` is
 484             # already of type unicode()
 485             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 486         elif 'TERM' in os.environ:
 487             self._write_string('\033]0;%s\007' % message, self._screen_file)
 488
 489     def save_console_title(self):
 490         if not self.params.get('consoletitle', False):
 491             return
 492         if 'TERM' in os.environ:
 493             # Save the title on stack
 494             self._write_string('\033[22;0t', self._screen_file)
 495
 496     def restore_console_title(self):
 497         if not self.params.get('consoletitle', False):
 498             return
 499         if 'TERM' in os.environ:
 500             # Restore the title from stack
 501             self._write_string('\033[23;0t', self._screen_file)
 502
 503     def __enter__(self):
 504         self.save_console_title()
 505         return self
 506
 507     def __exit__(self, *args):
 508         self.restore_console_title()
 509
 510         if self.params.get('cookiefile') is not None:
 511             self.cookiejar.save()
 512
 513     def trouble(self, message=None, tb=None):
 514         """Determine action to take when a download problem appears.
 515
 516         Depending on if the downloader has been configured to ignore
 517         download errors or not, this method may throw an exception or
 518         not when errors are found, after printing the message.
 519
 520         tb, if given, is additional traceback information.
 521         """
 522         if message is not None:
 523             self.to_stderr(message)
 524         if self.params.get('verbose'):
 525             if tb is None:
 526                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 527                     tb = ''
 528                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 529                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 530                     tb += encode_compat_str(traceback.format_exc())
 531                 else:
 532                     tb_data = traceback.format_list(traceback.extract_stack())
 533                     tb = ''.join(tb_data)
 534             self.to_stderr(tb)
 535         if not self.params.get('ignoreerrors', False):
 536             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 537                 exc_info = sys.exc_info()[1].exc_info
 538             else:
 539                 exc_info = sys.exc_info()
 540             raise DownloadError(message, exc_info)
 541         self._download_retcode = 1
 542
 543     def report_warning(self, message):
 544         '''
 545         Print the message to stderr, it will be prefixed with 'WARNING:'
 546         If stderr is a tty file the 'WARNING:' will be colored
 547         '''
 548         if self.params.get('logger') is not None:
 549             self.params['logger'].warning(message)
 550         else:
 551             if self.params.get('no_warnings'):
 552                 return
 553             if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 554                 _msg_header = '\033[0;33mWARNING:\033[0m'
 555             else:
 556                 _msg_header = 'WARNING:'
 557             warning_message = '%s %s' % (_msg_header, message)
 558             self.to_stderr(warning_message)
 559
 560     def report_error(self, message, tb=None):
 561         '''
 562         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 563         in red if stderr is a tty file.
 564         '''
 565         if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 566             _msg_header = '\033[0;31mERROR:\033[0m'
 567         else:
 568             _msg_header = 'ERROR:'
 569         error_message = '%s %s' % (_msg_header, message)
 570         self.trouble(error_message, tb)
 571
 572     def report_file_already_downloaded(self, file_name):
 573         """Report file has already been fully downloaded."""
 574         try:
 575             self.to_screen('[download] %s has already been downloaded' % file_name)
 576         except UnicodeEncodeError:
 577             self.to_screen('[download] The file has already been downloaded')
 578
 579     def prepare_filename(self, info_dict):
 580         """Generate the output filename."""
 581         try:
 582             template_dict = dict(info_dict)
 583
 584             template_dict['epoch'] = int(time.time())
 585             autonumber_size = self.params.get('autonumber_size')
 586             if autonumber_size is None:
 587                 autonumber_size = 5
 588             autonumber_templ = '%0' + str(autonumber_size) + 'd'
 589             template_dict['autonumber'] = autonumber_templ % (self.params.get('autonumber_start', 1) - 1 + self._num_downloads)
 590             if template_dict.get('playlist_index') is not None:
 591                 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
 592             if template_dict.get('resolution') is None:
 593                 if template_dict.get('width') and template_dict.get('height'):
 594                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 595                 elif template_dict.get('height'):
 596                     template_dict['resolution'] = '%sp' % template_dict['height']
 597                 elif template_dict.get('width'):
 598                     template_dict['resolution'] = '%dx?' % template_dict['width']
 599
 600             sanitize = lambda k, v: sanitize_filename(
 601                 compat_str(v),
 602                 restricted=self.params.get('restrictfilenames'),
 603                 is_id=(k == 'id'))
 604             template_dict = dict((k, sanitize(k, v))
 605                                  for k, v in template_dict.items()
 606                                  if v is not None and not isinstance(v, (list, tuple, dict)))
 607             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 608
 609             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 610             tmpl = compat_expanduser(outtmpl)
 611             filename = tmpl % template_dict
 612             # Temporary fix for #4787
 613             # 'Treat' all problem characters by passing filename through preferredencoding
 614             # to workaround encoding issues with subprocess on python2 @ Windows
 615             if sys.version_info < (3, 0) and sys.platform == 'win32':
 616                 filename = encodeFilename(filename, True).decode(preferredencoding())
 617             return sanitize_path(filename)
 618         except ValueError as err:
 619             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 620             return None
 621
 622     def _match_entry(self, info_dict, incomplete):
 623         """ Returns None iff the file should be downloaded """
 624
 625         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 626         if 'title' in info_dict:
 627             # This can happen when we're just evaluating the playlist
 628             title = info_dict['title']
 629             matchtitle = self.params.get('matchtitle', False)
 630             if matchtitle:
 631                 if not re.search(matchtitle, title, re.IGNORECASE):
 632                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 633             rejecttitle = self.params.get('rejecttitle', False)
 634             if rejecttitle:
 635                 if re.search(rejecttitle, title, re.IGNORECASE):
 636                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 637         date = info_dict.get('upload_date')
 638         if date is not None:
 639             dateRange = self.params.get('daterange', DateRange())
 640             if date not in dateRange:
 641                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 642         view_count = info_dict.get('view_count')
 643         if view_count is not None:
 644             min_views = self.params.get('min_views')
 645             if min_views is not None and view_count < min_views:
 646                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 647             max_views = self.params.get('max_views')
 648             if max_views is not None and view_count > max_views:
 649                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 650         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 651             return 'Skipping "%s" because it is age restricted' % video_title
 652         if self.in_download_archive(info_dict):
 653             return '%s has already been recorded in archive' % video_title
 654
 655         if not incomplete:
 656             match_filter = self.params.get('match_filter')
 657             if match_filter is not None:
 658                 ret = match_filter(info_dict)
 659                 if ret is not None:
 660                     return ret
 661
 662         return None
 663
 664     @staticmethod
 665     def add_extra_info(info_dict, extra_info):
 666         '''Set the keys from extra_info in info dict if they are missing'''
 667         for key, value in extra_info.items():
 668             info_dict.setdefault(key, value)
 669
 670     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 671                      process=True, force_generic_extractor=False):
 672         '''
 673         Returns a list with a dictionary for each video we find.
 674         If 'download', also downloads the videos.
 675         extra_info is a dict containing the extra values to add to each result
 676         '''
 677
 678         if not ie_key and force_generic_extractor:
 679             ie_key = 'Generic'
 680
 681         if ie_key:
 682             ies = [self.get_info_extractor(ie_key)]
 683         else:
 684             ies = self._ies
 685
 686         for ie in ies:
 687             if not ie.suitable(url):
 688                 continue
 689
 690             ie = self.get_info_extractor(ie.ie_key())
 691             if not ie.working():
 692                 self.report_warning('The program functionality for this site has been marked as broken, '
 693                                     'and will probably not work.')
 694
 695             try:
 696                 ie_result = ie.extract(url)
 697                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 698                     break
 699                 if isinstance(ie_result, list):
 700                     # Backwards compatibility: old IE result format
 701                     ie_result = {
 702                         '_type': 'compat_list',
 703                         'entries': ie_result,
 704                     }
 705                 self.add_default_extra_info(ie_result, ie, url)
 706                 if process:
 707                     return self.process_ie_result(ie_result, download, extra_info)
 708                 else:
 709                     return ie_result
 710             except ExtractorError as e:  # An error we somewhat expected
 711                 self.report_error(compat_str(e), e.format_traceback())
 712                 break
 713             except MaxDownloadsReached:
 714                 raise
 715             except Exception as e:
 716                 if self.params.get('ignoreerrors', False):
 717                     self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
 718                     break
 719                 else:
 720                     raise
 721         else:
 722             self.report_error('no suitable InfoExtractor for URL %s' % url)
 723
 724     def add_default_extra_info(self, ie_result, ie, url):
 725         self.add_extra_info(ie_result, {
 726             'extractor': ie.IE_NAME,
 727             'webpage_url': url,
 728             'webpage_url_basename': url_basename(url),
 729             'extractor_key': ie.ie_key(),
 730         })
 731
 732     def process_ie_result(self, ie_result, download=True, extra_info={}):
 733         """
 734         Take the result of the ie(may be modified) and resolve all unresolved
 735         references (URLs, playlist items).
 736
 737         It will also download the videos if 'download'.
 738         Returns the resolved ie_result.
 739         """
 740         result_type = ie_result.get('_type', 'video')
 741
 742         if result_type in ('url', 'url_transparent'):
 743             ie_result['url'] = sanitize_url(ie_result['url'])
 744             extract_flat = self.params.get('extract_flat', False)
 745             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 746                     extract_flat is True):
 747                 if self.params.get('forcejson', False):
 748                     self.to_stdout(json.dumps(ie_result))
 749                 return ie_result
 750
 751         if result_type == 'video':
 752             self.add_extra_info(ie_result, extra_info)
 753             return self.process_video_result(ie_result, download=download)
 754         elif result_type == 'url':
 755             # We have to add extra_info to the results because it may be
 756             # contained in a playlist
 757             return self.extract_info(ie_result['url'],
 758                                      download,
 759                                      ie_key=ie_result.get('ie_key'),
 760                                      extra_info=extra_info)
 761         elif result_type == 'url_transparent':
 762             # Use the information from the embedding page
 763             info = self.extract_info(
 764                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 765                 extra_info=extra_info, download=False, process=False)
 766
 767             force_properties = dict(
 768                 (k, v) for k, v in ie_result.items() if v is not None)
 769             for f in ('_type', 'url', 'ie_key'):
 770                 if f in force_properties:
 771                     del force_properties[f]
 772             new_result = info.copy()
 773             new_result.update(force_properties)
 774
 775             assert new_result.get('_type') != 'url_transparent'
 776
 777             return self.process_ie_result(
 778                 new_result, download=download, extra_info=extra_info)
 779         elif result_type == 'playlist' or result_type == 'multi_video':
 780             # We process each entry in the playlist
 781             playlist = ie_result.get('title') or ie_result.get('id')
 782             self.to_screen('[download] Downloading playlist: %s' % playlist)
 783
 784             playlist_results = []
 785
 786             playliststart = self.params.get('playliststart', 1) - 1
 787             playlistend = self.params.get('playlistend')
 788             # For backwards compatibility, interpret -1 as whole list
 789             if playlistend == -1:
 790                 playlistend = None
 791
 792             playlistitems_str = self.params.get('playlist_items')
 793             playlistitems = None
 794             if playlistitems_str is not None:
 795                 def iter_playlistitems(format):
 796                     for string_segment in format.split(','):
 797                         if '-' in string_segment:
 798                             start, end = string_segment.split('-')
 799                             for item in range(int(start), int(end) + 1):
 800                                 yield int(item)
 801                         else:
 802                             yield int(string_segment)
 803                 playlistitems = iter_playlistitems(playlistitems_str)
 804
 805             ie_entries = ie_result['entries']
 806             if isinstance(ie_entries, list):
 807                 n_all_entries = len(ie_entries)
 808                 if playlistitems:
 809                     entries = [
 810                         ie_entries[i - 1] for i in playlistitems
 811                         if -n_all_entries <= i - 1 < n_all_entries]
 812                 else:
 813                     entries = ie_entries[playliststart:playlistend]
 814                 n_entries = len(entries)
 815                 self.to_screen(
 816                     '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
 817                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 818             elif isinstance(ie_entries, PagedList):
 819                 if playlistitems:
 820                     entries = []
 821                     for item in playlistitems:
 822                         entries.extend(ie_entries.getslice(
 823                             item - 1, item
 824                         ))
 825                 else:
 826                     entries = ie_entries.getslice(
 827                         playliststart, playlistend)
 828                 n_entries = len(entries)
 829                 self.to_screen(
 830                     '[%s] playlist %s: Downloading %d videos' %
 831                     (ie_result['extractor'], playlist, n_entries))
 832             else:  # iterable
 833                 if playlistitems:
 834                     entry_list = list(ie_entries)
 835                     entries = [entry_list[i - 1] for i in playlistitems]
 836                 else:
 837                     entries = list(itertools.islice(
 838                         ie_entries, playliststart, playlistend))
 839                 n_entries = len(entries)
 840                 self.to_screen(
 841                     '[%s] playlist %s: Downloading %d videos' %
 842                     (ie_result['extractor'], playlist, n_entries))
 843
 844             if self.params.get('playlistreverse', False):
 845                 entries = entries[::-1]
 846
 847             if self.params.get('playlistrandom', False):
 848                 random.shuffle(entries)
 849
 850             for i, entry in enumerate(entries, 1):
 851                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 852                 extra = {
 853                     'n_entries': n_entries,
 854                     'playlist': playlist,
 855                     'playlist_id': ie_result.get('id'),
 856                     'playlist_title': ie_result.get('title'),
 857                     'playlist_index': i + playliststart,
 858                     'extractor': ie_result['extractor'],
 859                     'webpage_url': ie_result['webpage_url'],
 860                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 861                     'extractor_key': ie_result['extractor_key'],
 862                 }
 863
 864                 reason = self._match_entry(entry, incomplete=True)
 865                 if reason is not None:
 866                     self.to_screen('[download] ' + reason)
 867                     continue
 868
 869                 entry_result = self.process_ie_result(entry,
 870                                                       download=download,
 871                                                       extra_info=extra)
 872                 playlist_results.append(entry_result)
 873             ie_result['entries'] = playlist_results
 874             self.to_screen('[download] Finished downloading playlist: %s' % playlist)
 875             return ie_result
 876         elif result_type == 'compat_list':
 877             self.report_warning(
 878                 'Extractor %s returned a compat_list result. '
 879                 'It needs to be updated.' % ie_result.get('extractor'))
 880
 881             def _fixup(r):
 882                 self.add_extra_info(
 883                     r,
 884                     {
 885                         'extractor': ie_result['extractor'],
 886                         'webpage_url': ie_result['webpage_url'],
 887                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 888                         'extractor_key': ie_result['extractor_key'],
 889                     }
 890                 )
 891                 return r
 892             ie_result['entries'] = [
 893                 self.process_ie_result(_fixup(r), download, extra_info)
 894                 for r in ie_result['entries']
 895             ]
 896             return ie_result
 897         else:
 898             raise Exception('Invalid result type: %s' % result_type)
 899
 900     def _build_format_filter(self, filter_spec):
 901         " Returns a function to filter the formats according to the filter_spec "
 902
 903         OPERATORS = {
 904             '<': operator.lt,
 905             '<=': operator.le,
 906             '>': operator.gt,
 907             '>=': operator.ge,
 908             '=': operator.eq,
 909             '!=': operator.ne,
 910         }
 911         operator_rex = re.compile(r'''(?x)\s*
 912             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
 913             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
 914             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
 915             $
 916             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
 917         m = operator_rex.search(filter_spec)
 918         if m:
 919             try:
 920                 comparison_value = int(m.group('value'))
 921             except ValueError:
 922                 comparison_value = parse_filesize(m.group('value'))
 923                 if comparison_value is None:
 924                     comparison_value = parse_filesize(m.group('value') + 'B')
 925                 if comparison_value is None:
 926                     raise ValueError(
 927                         'Invalid value %r in format specification %r' % (
 928                             m.group('value'), filter_spec))
 929             op = OPERATORS[m.group('op')]
 930
 931         if not m:
 932             STR_OPERATORS = {
 933                 '=': operator.eq,
 934                 '!=': operator.ne,
 935                 '^=': lambda attr, value: attr.startswith(value),
 936                 '$=': lambda attr, value: attr.endswith(value),
 937                 '*=': lambda attr, value: value in attr,
 938             }
 939             str_operator_rex = re.compile(r'''(?x)
 940                 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
 941                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
 942                 \s*(?P<value>[a-zA-Z0-9._-]+)
 943                 \s*$
 944                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
 945             m = str_operator_rex.search(filter_spec)
 946             if m:
 947                 comparison_value = m.group('value')
 948                 op = STR_OPERATORS[m.group('op')]
 949
 950         if not m:
 951             raise ValueError('Invalid filter specification %r' % filter_spec)
 952
 953         def _filter(f):
 954             actual_value = f.get(m.group('key'))
 955             if actual_value is None:
 956                 return m.group('none_inclusive')
 957             return op(actual_value, comparison_value)
 958         return _filter
 959
 960     def build_format_selector(self, format_spec):
 961         def syntax_error(note, start):
 962             message = (
 963                 'Invalid format specification: '
 964                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
 965             return SyntaxError(message)
 966
 967         PICKFIRST = 'PICKFIRST'
 968         MERGE = 'MERGE'
 969         SINGLE = 'SINGLE'
 970         GROUP = 'GROUP'
 971         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
 972
 973         def _parse_filter(tokens):
 974             filter_parts = []
 975             for type, string, start, _, _ in tokens:
 976                 if type == tokenize.OP and string == ']':
 977                     return ''.join(filter_parts)
 978                 else:
 979                     filter_parts.append(string)
 980
 981         def _remove_unused_ops(tokens):
 982             # Remove operators that we don't use and join them with the surrounding strings
 983             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
 984             ALLOWED_OPS = ('/', '+', ',', '(', ')')
 985             last_string, last_start, last_end, last_line = None, None, None, None
 986             for type, string, start, end, line in tokens:
 987                 if type == tokenize.OP and string == '[':
 988                     if last_string:
 989                         yield tokenize.NAME, last_string, last_start, last_end, last_line
 990                         last_string = None
 991                     yield type, string, start, end, line
 992                     # everything inside brackets will be handled by _parse_filter
 993                     for type, string, start, end, line in tokens:
 994                         yield type, string, start, end, line
 995                         if type == tokenize.OP and string == ']':
 996                             break
 997                 elif type == tokenize.OP and string in ALLOWED_OPS:
 998                     if last_string:
 999                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1000                         last_string = None
1001                     yield type, string, start, end, line
1002                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
1003                     if not last_string:
1004                         last_string = string
1005                         last_start = start
1006                         last_end = end
1007                     else:
1008                         last_string += string
1009             if last_string:
1010                 yield tokenize.NAME, last_string, last_start, last_end, last_line
1011
1012         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
1013             selectors = []
1014             current_selector = None
1015             for type, string, start, _, _ in tokens:
1016                 # ENCODING is only defined in python 3.x
1017                 if type == getattr(tokenize, 'ENCODING', None):
1018                     continue
1019                 elif type in [tokenize.NAME, tokenize.NUMBER]:
1020                     current_selector = FormatSelector(SINGLE, string, [])
1021                 elif type == tokenize.OP:
1022                     if string == ')':
1023                         if not inside_group:
1024                             # ')' will be handled by the parentheses group
1025                             tokens.restore_last_token()
1026                         break
1027                     elif inside_merge and string in ['/', ',']:
1028                         tokens.restore_last_token()
1029                         break
1030                     elif inside_choice and string == ',':
1031                         tokens.restore_last_token()
1032                         break
1033                     elif string == ',':
1034                         if not current_selector:
1035                             raise syntax_error('"," must follow a format selector', start)
1036                         selectors.append(current_selector)
1037                         current_selector = None
1038                     elif string == '/':
1039                         if not current_selector:
1040                             raise syntax_error('"/" must follow a format selector', start)
1041                         first_choice = current_selector
1042                         second_choice = _parse_format_selection(tokens, inside_choice=True)
1043                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1044                     elif string == '[':
1045                         if not current_selector:
1046                             current_selector = FormatSelector(SINGLE, 'best', [])
1047                         format_filter = _parse_filter(tokens)
1048                         current_selector.filters.append(format_filter)
1049                     elif string == '(':
1050                         if current_selector:
1051                             raise syntax_error('Unexpected "("', start)
1052                         group = _parse_format_selection(tokens, inside_group=True)
1053                         current_selector = FormatSelector(GROUP, group, [])
1054                     elif string == '+':
1055                         video_selector = current_selector
1056                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
1057                         if not video_selector or not audio_selector:
1058                             raise syntax_error('"+" must be between two format selectors', start)
1059                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1060                     else:
1061                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1062                 elif type == tokenize.ENDMARKER:
1063                     break
1064             if current_selector:
1065                 selectors.append(current_selector)
1066             return selectors
1067
1068         def _build_selector_function(selector):
1069             if isinstance(selector, list):
1070                 fs = [_build_selector_function(s) for s in selector]
1071
1072                 def selector_function(ctx):
1073                     for f in fs:
1074                         for format in f(ctx):
1075                             yield format
1076                 return selector_function
1077             elif selector.type == GROUP:
1078                 selector_function = _build_selector_function(selector.selector)
1079             elif selector.type == PICKFIRST:
1080                 fs = [_build_selector_function(s) for s in selector.selector]
1081
1082                 def selector_function(ctx):
1083                     for f in fs:
1084                         picked_formats = list(f(ctx))
1085                         if picked_formats:
1086                             return picked_formats
1087                     return []
1088             elif selector.type == SINGLE:
1089                 format_spec = selector.selector
1090
1091                 def selector_function(ctx):
1092                     formats = list(ctx['formats'])
1093                     if not formats:
1094                         return
1095                     if format_spec == 'all':
1096                         for f in formats:
1097                             yield f
1098                     elif format_spec in ['best', 'worst', None]:
1099                         format_idx = 0 if format_spec == 'worst' else -1
1100                         audiovideo_formats = [
1101                             f for f in formats
1102                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1103                         if audiovideo_formats:
1104                             yield audiovideo_formats[format_idx]
1105                         # for extractors with incomplete formats (audio only (soundcloud)
1106                         # or video only (imgur)) we will fallback to best/worst
1107                         # {video,audio}-only format
1108                         elif ctx['incomplete_formats']:
1109                             yield formats[format_idx]
1110                     elif format_spec == 'bestaudio':
1111                         audio_formats = [
1112                             f for f in formats
1113                             if f.get('vcodec') == 'none']
1114                         if audio_formats:
1115                             yield audio_formats[-1]
1116                     elif format_spec == 'worstaudio':
1117                         audio_formats = [
1118                             f for f in formats
1119                             if f.get('vcodec') == 'none']
1120                         if audio_formats:
1121                             yield audio_formats[0]
1122                     elif format_spec == 'bestvideo':
1123                         video_formats = [
1124                             f for f in formats
1125                             if f.get('acodec') == 'none']
1126                         if video_formats:
1127                             yield video_formats[-1]
1128                     elif format_spec == 'worstvideo':
1129                         video_formats = [
1130                             f for f in formats
1131                             if f.get('acodec') == 'none']
1132                         if video_formats:
1133                             yield video_formats[0]
1134                     else:
1135                         extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1136                         if format_spec in extensions:
1137                             filter_f = lambda f: f['ext'] == format_spec
1138                         else:
1139                             filter_f = lambda f: f['format_id'] == format_spec
1140                         matches = list(filter(filter_f, formats))
1141                         if matches:
1142                             yield matches[-1]
1143             elif selector.type == MERGE:
1144                 def _merge(formats_info):
1145                     format_1, format_2 = [f['format_id'] for f in formats_info]
1146                     # The first format must contain the video and the
1147                     # second the audio
1148                     if formats_info[0].get('vcodec') == 'none':
1149                         self.report_error('The first format must '
1150                                           'contain the video, try using '
1151                                           '"-f %s+%s"' % (format_2, format_1))
1152                         return
1153                     # Formats must be opposite (video+audio)
1154                     if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1155                         self.report_error(
1156                             'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1157                             % (format_1, format_2))
1158                         return
1159                     output_ext = (
1160                         formats_info[0]['ext']
1161                         if self.params.get('merge_output_format') is None
1162                         else self.params['merge_output_format'])
1163                     return {
1164                         'requested_formats': formats_info,
1165                         'format': '%s+%s' % (formats_info[0].get('format'),
1166                                              formats_info[1].get('format')),
1167                         'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1168                                                 formats_info[1].get('format_id')),
1169                         'width': formats_info[0].get('width'),
1170                         'height': formats_info[0].get('height'),
1171                         'resolution': formats_info[0].get('resolution'),
1172                         'fps': formats_info[0].get('fps'),
1173                         'vcodec': formats_info[0].get('vcodec'),
1174                         'vbr': formats_info[0].get('vbr'),
1175                         'stretched_ratio': formats_info[0].get('stretched_ratio'),
1176                         'acodec': formats_info[1].get('acodec'),
1177                         'abr': formats_info[1].get('abr'),
1178                         'ext': output_ext,
1179                     }
1180                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1181
1182                 def selector_function(ctx):
1183                     for pair in itertools.product(
1184                             video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
1185                         yield _merge(pair)
1186
1187             filters = [self._build_format_filter(f) for f in selector.filters]
1188
1189             def final_selector(ctx):
1190                 ctx_copy = copy.deepcopy(ctx)
1191                 for _filter in filters:
1192                     ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
1193                 return selector_function(ctx_copy)
1194             return final_selector
1195
1196         stream = io.BytesIO(format_spec.encode('utf-8'))
1197         try:
1198             tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1199         except tokenize.TokenError:
1200             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1201
1202         class TokenIterator(object):
1203             def __init__(self, tokens):
1204                 self.tokens = tokens
1205                 self.counter = 0
1206
1207             def __iter__(self):
1208                 return self
1209
1210             def __next__(self):
1211                 if self.counter >= len(self.tokens):
1212                     raise StopIteration()
1213                 value = self.tokens[self.counter]
1214                 self.counter += 1
1215                 return value
1216
1217             next = __next__
1218
1219             def restore_last_token(self):
1220                 self.counter -= 1
1221
1222         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1223         return _build_selector_function(parsed_selector)
1224
1225     def _calc_headers(self, info_dict):
1226         res = std_headers.copy()
1227
1228         add_headers = info_dict.get('http_headers')
1229         if add_headers:
1230             res.update(add_headers)
1231
1232         cookies = self._calc_cookies(info_dict)
1233         if cookies:
1234             res['Cookie'] = cookies
1235
1236         return res
1237
1238     def _calc_cookies(self, info_dict):
1239         pr = sanitized_Request(info_dict['url'])
1240         self.cookiejar.add_cookie_header(pr)
1241         return pr.get_header('Cookie')
1242
1243     def process_video_result(self, info_dict, download=True):
1244         assert info_dict.get('_type', 'video') == 'video'
1245
1246         if 'id' not in info_dict:
1247             raise ExtractorError('Missing "id" field in extractor result')
1248         if 'title' not in info_dict:
1249             raise ExtractorError('Missing "title" field in extractor result')
1250
1251         if not isinstance(info_dict['id'], compat_str):
1252             self.report_warning('"id" field is not a string - forcing string conversion')
1253             info_dict['id'] = compat_str(info_dict['id'])
1254
1255         if 'playlist' not in info_dict:
1256             # It isn't part of a playlist
1257             info_dict['playlist'] = None
1258             info_dict['playlist_index'] = None
1259
1260         thumbnails = info_dict.get('thumbnails')
1261         if thumbnails is None:
1262             thumbnail = info_dict.get('thumbnail')
1263             if thumbnail:
1264                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1265         if thumbnails:
1266             thumbnails.sort(key=lambda t: (
1267                 t.get('preference') if t.get('preference') is not None else -1,
1268                 t.get('width') if t.get('width') is not None else -1,
1269                 t.get('height') if t.get('height') is not None else -1,
1270                 t.get('id') if t.get('id') is not None else '', t.get('url')))
1271             for i, t in enumerate(thumbnails):
1272                 t['url'] = sanitize_url(t['url'])
1273                 if t.get('width') and t.get('height'):
1274                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1275                 if t.get('id') is None:
1276                     t['id'] = '%d' % i
1277
1278         if self.params.get('list_thumbnails'):
1279             self.list_thumbnails(info_dict)
1280             return
1281
1282         thumbnail = info_dict.get('thumbnail')
1283         if thumbnail:
1284             info_dict['thumbnail'] = sanitize_url(thumbnail)
1285         elif thumbnails:
1286             info_dict['thumbnail'] = thumbnails[-1]['url']
1287
1288         if 'display_id' not in info_dict and 'id' in info_dict:
1289             info_dict['display_id'] = info_dict['id']
1290
1291         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1292             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1293             # see http://bugs.python.org/issue1646728)
1294             try:
1295                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1296                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1297             except (ValueError, OverflowError, OSError):
1298                 pass
1299
1300         # Auto generate title fields corresponding to the *_number fields when missing
1301         # in order to always have clean titles. This is very common for TV series.
1302         for field in ('chapter', 'season', 'episode'):
1303             if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1304                 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1305
1306         subtitles = info_dict.get('subtitles')
1307         if subtitles:
1308             for _, subtitle in subtitles.items():
1309                 for subtitle_format in subtitle:
1310                     if subtitle_format.get('url'):
1311                         subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1312                     if subtitle_format.get('ext') is None:
1313                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1314
1315         if self.params.get('listsubtitles', False):
1316             if 'automatic_captions' in info_dict:
1317                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1318             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1319             return
1320         info_dict['requested_subtitles'] = self.process_subtitles(
1321             info_dict['id'], subtitles,
1322             info_dict.get('automatic_captions'))
1323
1324         # We now pick which formats have to be downloaded
1325         if info_dict.get('formats') is None:
1326             # There's only one format available
1327             formats = [info_dict]
1328         else:
1329             formats = info_dict['formats']
1330
1331         if not formats:
1332             raise ExtractorError('No video formats found!')
1333
1334         formats_dict = {}
1335
1336         # We check that all the formats have the format and format_id fields
1337         for i, format in enumerate(formats):
1338             if 'url' not in format:
1339                 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1340
1341             format['url'] = sanitize_url(format['url'])
1342
1343             if format.get('format_id') is None:
1344                 format['format_id'] = compat_str(i)
1345             else:
1346                 # Sanitize format_id from characters used in format selector expression
1347                 format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
1348             format_id = format['format_id']
1349             if format_id not in formats_dict:
1350                 formats_dict[format_id] = []
1351             formats_dict[format_id].append(format)
1352
1353         # Make sure all formats have unique format_id
1354         for format_id, ambiguous_formats in formats_dict.items():
1355             if len(ambiguous_formats) > 1:
1356                 for i, format in enumerate(ambiguous_formats):
1357                     format['format_id'] = '%s-%d' % (format_id, i)
1358
1359         for i, format in enumerate(formats):
1360             if format.get('format') is None:
1361                 format['format'] = '{id} - {res}{note}'.format(
1362                     id=format['format_id'],
1363                     res=self.format_resolution(format),
1364                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1365                 )
1366             # Automatically determine file extension if missing
1367             if format.get('ext') is None:
1368                 format['ext'] = determine_ext(format['url']).lower()
1369             # Automatically determine protocol if missing (useful for format
1370             # selection purposes)
1371             if format.get('protocol') is None:
1372                 format['protocol'] = determine_protocol(format)
1373             # Add HTTP headers, so that external programs can use them from the
1374             # json output
1375             full_format_info = info_dict.copy()
1376             full_format_info.update(format)
1377             format['http_headers'] = self._calc_headers(full_format_info)
1378
1379         # TODO Central sorting goes here
1380
1381         if formats[0] is not info_dict:
1382             # only set the 'formats' fields if the original info_dict list them
1383             # otherwise we end up with a circular reference, the first (and unique)
1384             # element in the 'formats' field in info_dict is info_dict itself,
1385             # which can't be exported to json
1386             info_dict['formats'] = formats
1387         if self.params.get('listformats'):
1388             self.list_formats(info_dict)
1389             return
1390
1391         req_format = self.params.get('format')
1392         if req_format is None:
1393             req_format_list = []
1394             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1395                     not info_dict.get('is_live')):
1396                 merger = FFmpegMergerPP(self)
1397                 if merger.available and merger.can_merge():
1398                     req_format_list.append('bestvideo+bestaudio')
1399             req_format_list.append('best')
1400             req_format = '/'.join(req_format_list)
1401         format_selector = self.build_format_selector(req_format)
1402
1403         # While in format selection we may need to have an access to the original
1404         # format set in order to calculate some metrics or do some processing.
1405         # For now we need to be able to guess whether original formats provided
1406         # by extractor are incomplete or not (i.e. whether extractor provides only
1407         # video-only or audio-only formats) for proper formats selection for
1408         # extractors with such incomplete formats (see
1409         # https://github.com/rg3/youtube-dl/pull/5556).
1410         # Since formats may be filtered during format selection and may not match
1411         # the original formats the results may be incorrect. Thus original formats
1412         # or pre-calculated metrics should be passed to format selection routines
1413         # as well.
1414         # We will pass a context object containing all necessary additional data
1415         # instead of just formats.
1416         # This fixes incorrect format selection issue (see
1417         # https://github.com/rg3/youtube-dl/issues/10083).
1418         incomplete_formats = (
1419             # All formats are video-only or
1420             all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
1421             # all formats are audio-only
1422             all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
1423
1424         ctx = {
1425             'formats': formats,
1426             'incomplete_formats': incomplete_formats,
1427         }
1428
1429         formats_to_download = list(format_selector(ctx))
1430         if not formats_to_download:
1431             raise ExtractorError('requested format not available',
1432                                  expected=True)
1433
1434         if download:
1435             if len(formats_to_download) > 1:
1436                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1437             for format in formats_to_download:
1438                 new_info = dict(info_dict)
1439                 new_info.update(format)
1440                 self.process_info(new_info)
1441         # We update the info dict with the best quality format (backwards compatibility)
1442         info_dict.update(formats_to_download[-1])
1443         return info_dict
1444
1445     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1446         """Select the requested subtitles and their format"""
1447         available_subs = {}
1448         if normal_subtitles and self.params.get('writesubtitles'):
1449             available_subs.update(normal_subtitles)
1450         if automatic_captions and self.params.get('writeautomaticsub'):
1451             for lang, cap_info in automatic_captions.items():
1452                 if lang not in available_subs:
1453                     available_subs[lang] = cap_info
1454
1455         if (not self.params.get('writesubtitles') and not
1456                 self.params.get('writeautomaticsub') or not
1457                 available_subs):
1458             return None
1459
1460         if self.params.get('allsubtitles', False):
1461             requested_langs = available_subs.keys()
1462         else:
1463             if self.params.get('subtitleslangs', False):
1464                 requested_langs = self.params.get('subtitleslangs')
1465             elif 'en' in available_subs:
1466                 requested_langs = ['en']
1467             else:
1468                 requested_langs = [list(available_subs.keys())[0]]
1469
1470         formats_query = self.params.get('subtitlesformat', 'best')
1471         formats_preference = formats_query.split('/') if formats_query else []
1472         subs = {}
1473         for lang in requested_langs:
1474             formats = available_subs.get(lang)
1475             if formats is None:
1476                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1477                 continue
1478             for ext in formats_preference:
1479                 if ext == 'best':
1480                     f = formats[-1]
1481                     break
1482                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1483                 if matches:
1484                     f = matches[-1]
1485                     break
1486             else:
1487                 f = formats[-1]
1488                 self.report_warning(
1489                     'No subtitle format found matching "%s" for language %s, '
1490                     'using %s' % (formats_query, lang, f['ext']))
1491             subs[lang] = f
1492         return subs
1493
1494     def process_info(self, info_dict):
1495         """Process a single resolved IE result."""
1496
1497         assert info_dict.get('_type', 'video') == 'video'
1498
1499         max_downloads = self.params.get('max_downloads')
1500         if max_downloads is not None:
1501             if self._num_downloads >= int(max_downloads):
1502                 raise MaxDownloadsReached()
1503
1504         info_dict['fulltitle'] = info_dict['title']
1505         if len(info_dict['title']) > 200:
1506             info_dict['title'] = info_dict['title'][:197] + '...'
1507
1508         if 'format' not in info_dict:
1509             info_dict['format'] = info_dict['ext']
1510
1511         reason = self._match_entry(info_dict, incomplete=False)
1512         if reason is not None:
1513             self.to_screen('[download] ' + reason)
1514             return
1515
1516         self._num_downloads += 1
1517
1518         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1519
1520         # Forced printings
1521         if self.params.get('forcetitle', False):
1522             self.to_stdout(info_dict['fulltitle'])
1523         if self.params.get('forceid', False):
1524             self.to_stdout(info_dict['id'])
1525         if self.params.get('forceurl', False):
1526             if info_dict.get('requested_formats') is not None:
1527                 for f in info_dict['requested_formats']:
1528                     self.to_stdout(f['url'] + f.get('play_path', ''))
1529             else:
1530                 # For RTMP URLs, also include the playpath
1531                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1532         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1533             self.to_stdout(info_dict['thumbnail'])
1534         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1535             self.to_stdout(info_dict['description'])
1536         if self.params.get('forcefilename', False) and filename is not None:
1537             self.to_stdout(filename)
1538         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1539             self.to_stdout(formatSeconds(info_dict['duration']))
1540         if self.params.get('forceformat', False):
1541             self.to_stdout(info_dict['format'])
1542         if self.params.get('forcejson', False):
1543             self.to_stdout(json.dumps(info_dict))
1544
1545         # Do nothing else if in simulate mode
1546         if self.params.get('simulate', False):
1547             return
1548
1549         if filename is None:
1550             return
1551
1552         try:
1553             dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1554             if dn and not os.path.exists(dn):
1555                 os.makedirs(dn)
1556         except (OSError, IOError) as err:
1557             self.report_error('unable to create directory ' + error_to_compat_str(err))
1558             return
1559
1560         if self.params.get('writedescription', False):
1561             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1562             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1563                 self.to_screen('[info] Video description is already present')
1564             elif info_dict.get('description') is None:
1565                 self.report_warning('There\'s no description to write.')
1566             else:
1567                 try:
1568                     self.to_screen('[info] Writing video description to: ' + descfn)
1569                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1570                         descfile.write(info_dict['description'])
1571                 except (OSError, IOError):
1572                     self.report_error('Cannot write description file ' + descfn)
1573                     return
1574
1575         if self.params.get('writeannotations', False):
1576             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1577             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1578                 self.to_screen('[info] Video annotations are already present')
1579             else:
1580                 try:
1581                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1582                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1583                         annofile.write(info_dict['annotations'])
1584                 except (KeyError, TypeError):
1585                     self.report_warning('There are no annotations to write.')
1586                 except (OSError, IOError):
1587                     self.report_error('Cannot write annotations file: ' + annofn)
1588                     return
1589
1590         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1591                                        self.params.get('writeautomaticsub')])
1592
1593         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1594             # subtitles download errors are already managed as troubles in relevant IE
1595             # that way it will silently go on when used with unsupporting IE
1596             subtitles = info_dict['requested_subtitles']
1597             ie = self.get_info_extractor(info_dict['extractor_key'])
1598             for sub_lang, sub_info in subtitles.items():
1599                 sub_format = sub_info['ext']
1600                 if sub_info.get('data') is not None:
1601                     sub_data = sub_info['data']
1602                 else:
1603                     try:
1604                         sub_data = ie._download_webpage(
1605                             sub_info['url'], info_dict['id'], note=False)
1606                     except ExtractorError as err:
1607                         self.report_warning('Unable to download subtitle for "%s": %s' %
1608                                             (sub_lang, error_to_compat_str(err.cause)))
1609                         continue
1610                 try:
1611                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1612                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1613                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1614                     else:
1615                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1616                         # Use newline='' to prevent conversion of newline characters
1617                         # See https://github.com/rg3/youtube-dl/issues/10268
1618                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
1619                             subfile.write(sub_data)
1620                 except (OSError, IOError):
1621                     self.report_error('Cannot write subtitles file ' + sub_filename)
1622                     return
1623
1624         if self.params.get('writeinfojson', False):
1625             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1626             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1627                 self.to_screen('[info] Video description metadata is already present')
1628             else:
1629                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1630                 try:
1631                     write_json_file(self.filter_requested_info(info_dict), infofn)
1632                 except (OSError, IOError):
1633                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1634                     return
1635
1636         self._write_thumbnails(info_dict, filename)
1637
1638         if not self.params.get('skip_download', False):
1639             try:
1640                 def dl(name, info):
1641                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1642                     for ph in self._progress_hooks:
1643                         fd.add_progress_hook(ph)
1644                     if self.params.get('verbose'):
1645                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1646                     return fd.download(name, info)
1647
1648                 if info_dict.get('requested_formats') is not None:
1649                     downloaded = []
1650                     success = True
1651                     merger = FFmpegMergerPP(self)
1652                     if not merger.available:
1653                         postprocessors = []
1654                         self.report_warning('You have requested multiple '
1655                                             'formats but ffmpeg or avconv are not installed.'
1656                                             ' The formats won\'t be merged.')
1657                     else:
1658                         postprocessors = [merger]
1659
1660                     def compatible_formats(formats):
1661                         video, audio = formats
1662                         # Check extension
1663                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1664                         if video_ext and audio_ext:
1665                             COMPATIBLE_EXTS = (
1666                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'),
1667                                 ('webm')
1668                             )
1669                             for exts in COMPATIBLE_EXTS:
1670                                 if video_ext in exts and audio_ext in exts:
1671                                     return True
1672                         # TODO: Check acodec/vcodec
1673                         return False
1674
1675                     filename_real_ext = os.path.splitext(filename)[1][1:]
1676                     filename_wo_ext = (
1677                         os.path.splitext(filename)[0]
1678                         if filename_real_ext == info_dict['ext']
1679                         else filename)
1680                     requested_formats = info_dict['requested_formats']
1681                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1682                         info_dict['ext'] = 'mkv'
1683                         self.report_warning(
1684                             'Requested formats are incompatible for merge and will be merged into mkv.')
1685                     # Ensure filename always has a correct extension for successful merge
1686                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1687                     if os.path.exists(encodeFilename(filename)):
1688                         self.to_screen(
1689                             '[download] %s has already been downloaded and '
1690                             'merged' % filename)
1691                     else:
1692                         for f in requested_formats:
1693                             new_info = dict(info_dict)
1694                             new_info.update(f)
1695                             fname = self.prepare_filename(new_info)
1696                             fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1697                             downloaded.append(fname)
1698                             partial_success = dl(fname, new_info)
1699                             success = success and partial_success
1700                         info_dict['__postprocessors'] = postprocessors
1701                         info_dict['__files_to_merge'] = downloaded
1702                 else:
1703                     # Just a single file
1704                     success = dl(filename, info_dict)
1705             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1706                 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1707                 return
1708             except (OSError, IOError) as err:
1709                 raise UnavailableVideoError(err)
1710             except (ContentTooShortError, ) as err:
1711                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1712                 return
1713
1714             if success and filename != '-':
1715                 # Fixup content
1716                 fixup_policy = self.params.get('fixup')
1717                 if fixup_policy is None:
1718                     fixup_policy = 'detect_or_warn'
1719
1720                 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1721
1722                 stretched_ratio = info_dict.get('stretched_ratio')
1723                 if stretched_ratio is not None and stretched_ratio != 1:
1724                     if fixup_policy == 'warn':
1725                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1726                             info_dict['id'], stretched_ratio))
1727                     elif fixup_policy == 'detect_or_warn':
1728                         stretched_pp = FFmpegFixupStretchedPP(self)
1729                         if stretched_pp.available:
1730                             info_dict.setdefault('__postprocessors', [])
1731                             info_dict['__postprocessors'].append(stretched_pp)
1732                         else:
1733                             self.report_warning(
1734                                 '%s: Non-uniform pixel ratio (%s). %s'
1735                                 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1736                     else:
1737                         assert fixup_policy in ('ignore', 'never')
1738
1739                 if (info_dict.get('requested_formats') is None and
1740                         info_dict.get('container') == 'm4a_dash'):
1741                     if fixup_policy == 'warn':
1742                         self.report_warning(
1743                             '%s: writing DASH m4a. '
1744                             'Only some players support this container.'
1745                             % info_dict['id'])
1746                     elif fixup_policy == 'detect_or_warn':
1747                         fixup_pp = FFmpegFixupM4aPP(self)
1748                         if fixup_pp.available:
1749                             info_dict.setdefault('__postprocessors', [])
1750                             info_dict['__postprocessors'].append(fixup_pp)
1751                         else:
1752                             self.report_warning(
1753                                 '%s: writing DASH m4a. '
1754                                 'Only some players support this container. %s'
1755                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1756                     else:
1757                         assert fixup_policy in ('ignore', 'never')
1758
1759                 if (info_dict.get('protocol') == 'm3u8_native' or
1760                         info_dict.get('protocol') == 'm3u8' and
1761                         self.params.get('hls_prefer_native')):
1762                     if fixup_policy == 'warn':
1763                         self.report_warning('%s: malformated aac bitstream.' % (
1764                             info_dict['id']))
1765                     elif fixup_policy == 'detect_or_warn':
1766                         fixup_pp = FFmpegFixupM3u8PP(self)
1767                         if fixup_pp.available:
1768                             info_dict.setdefault('__postprocessors', [])
1769                             info_dict['__postprocessors'].append(fixup_pp)
1770                         else:
1771                             self.report_warning(
1772                                 '%s: malformated aac bitstream. %s'
1773                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1774                     else:
1775                         assert fixup_policy in ('ignore', 'never')
1776
1777                 try:
1778                     self.post_process(filename, info_dict)
1779                 except (PostProcessingError) as err:
1780                     self.report_error('postprocessing: %s' % str(err))
1781                     return
1782                 self.record_download_archive(info_dict)
1783
1784     def download(self, url_list):
1785         """Download a given list of URLs."""
1786         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1787         if (len(url_list) > 1 and
1788                 '%' not in outtmpl and
1789                 self.params.get('max_downloads') != 1):
1790             raise SameFileError(outtmpl)
1791
1792         for url in url_list:
1793             try:
1794                 # It also downloads the videos
1795                 res = self.extract_info(
1796                     url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1797             except UnavailableVideoError:
1798                 self.report_error('unable to download video')
1799             except MaxDownloadsReached:
1800                 self.to_screen('[info] Maximum number of downloaded files reached.')
1801                 raise
1802             else:
1803                 if self.params.get('dump_single_json', False):
1804                     self.to_stdout(json.dumps(res))
1805
1806         return self._download_retcode
1807
1808     def download_with_info_file(self, info_filename):
1809         with contextlib.closing(fileinput.FileInput(
1810                 [info_filename], mode='r',
1811                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1812             # FileInput doesn't have a read method, we can't call json.load
1813             info = self.filter_requested_info(json.loads('\n'.join(f)))
1814         try:
1815             self.process_ie_result(info, download=True)
1816         except DownloadError:
1817             webpage_url = info.get('webpage_url')
1818             if webpage_url is not None:
1819                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1820                 return self.download([webpage_url])
1821             else:
1822                 raise
1823         return self._download_retcode
1824
1825     @staticmethod
1826     def filter_requested_info(info_dict):
1827         return dict(
1828             (k, v) for k, v in info_dict.items()
1829             if k not in ['requested_formats', 'requested_subtitles'])
1830
1831     def post_process(self, filename, ie_info):
1832         """Run all the postprocessors on the given file."""
1833         info = dict(ie_info)
1834         info['filepath'] = filename
1835         pps_chain = []
1836         if ie_info.get('__postprocessors') is not None:
1837             pps_chain.extend(ie_info['__postprocessors'])
1838         pps_chain.extend(self._pps)
1839         for pp in pps_chain:
1840             files_to_delete = []
1841             try:
1842                 files_to_delete, info = pp.run(info)
1843             except PostProcessingError as e:
1844                 self.report_error(e.msg)
1845             if files_to_delete and not self.params.get('keepvideo', False):
1846                 for old_filename in files_to_delete:
1847                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1848                     try:
1849                         os.remove(encodeFilename(old_filename))
1850                     except (IOError, OSError):
1851                         self.report_warning('Unable to remove downloaded original file')
1852
1853     def _make_archive_id(self, info_dict):
1854         # Future-proof against any change in case
1855         # and backwards compatibility with prior versions
1856         extractor = info_dict.get('extractor_key')
1857         if extractor is None:
1858             if 'id' in info_dict:
1859                 extractor = info_dict.get('ie_key')  # key in a playlist
1860         if extractor is None:
1861             return None  # Incomplete video information
1862         return extractor.lower() + ' ' + info_dict['id']
1863
1864     def in_download_archive(self, info_dict):
1865         fn = self.params.get('download_archive')
1866         if fn is None:
1867             return False
1868
1869         vid_id = self._make_archive_id(info_dict)
1870         if vid_id is None:
1871             return False  # Incomplete video information
1872
1873         try:
1874             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1875                 for line in archive_file:
1876                     if line.strip() == vid_id:
1877                         return True
1878         except IOError as ioe:
1879             if ioe.errno != errno.ENOENT:
1880                 raise
1881         return False
1882
1883     def record_download_archive(self, info_dict):
1884         fn = self.params.get('download_archive')
1885         if fn is None:
1886             return
1887         vid_id = self._make_archive_id(info_dict)
1888         assert vid_id
1889         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1890             archive_file.write(vid_id + '\n')
1891
1892     @staticmethod
1893     def format_resolution(format, default='unknown'):
1894         if format.get('vcodec') == 'none':
1895             return 'audio only'
1896         if format.get('resolution') is not None:
1897             return format['resolution']
1898         if format.get('height') is not None:
1899             if format.get('width') is not None:
1900                 res = '%sx%s' % (format['width'], format['height'])
1901             else:
1902                 res = '%sp' % format['height']
1903         elif format.get('width') is not None:
1904             res = '%dx?' % format['width']
1905         else:
1906             res = default
1907         return res
1908
1909     def _format_note(self, fdict):
1910         res = ''
1911         if fdict.get('ext') in ['f4f', 'f4m']:
1912             res += '(unsupported) '
1913         if fdict.get('language'):
1914             if res:
1915                 res += ' '
1916             res += '[%s] ' % fdict['language']
1917         if fdict.get('format_note') is not None:
1918             res += fdict['format_note'] + ' '
1919         if fdict.get('tbr') is not None:
1920             res += '%4dk ' % fdict['tbr']
1921         if fdict.get('container') is not None:
1922             if res:
1923                 res += ', '
1924             res += '%s container' % fdict['container']
1925         if (fdict.get('vcodec') is not None and
1926                 fdict.get('vcodec') != 'none'):
1927             if res:
1928                 res += ', '
1929             res += fdict['vcodec']
1930             if fdict.get('vbr') is not None:
1931                 res += '@'
1932         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1933             res += 'video@'
1934         if fdict.get('vbr') is not None:
1935             res += '%4dk' % fdict['vbr']
1936         if fdict.get('fps') is not None:
1937             if res:
1938                 res += ', '
1939             res += '%sfps' % fdict['fps']
1940         if fdict.get('acodec') is not None:
1941             if res:
1942                 res += ', '
1943             if fdict['acodec'] == 'none':
1944                 res += 'video only'
1945             else:
1946                 res += '%-5s' % fdict['acodec']
1947         elif fdict.get('abr') is not None:
1948             if res:
1949                 res += ', '
1950             res += 'audio'
1951         if fdict.get('abr') is not None:
1952             res += '@%3dk' % fdict['abr']
1953         if fdict.get('asr') is not None:
1954             res += ' (%5dHz)' % fdict['asr']
1955         if fdict.get('filesize') is not None:
1956             if res:
1957                 res += ', '
1958             res += format_bytes(fdict['filesize'])
1959         elif fdict.get('filesize_approx') is not None:
1960             if res:
1961                 res += ', '
1962             res += '~' + format_bytes(fdict['filesize_approx'])
1963         return res
1964
1965     def list_formats(self, info_dict):
1966         formats = info_dict.get('formats', [info_dict])
1967         table = [
1968             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1969             for f in formats
1970             if f.get('preference') is None or f['preference'] >= -1000]
1971         if len(formats) > 1:
1972             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1973
1974         header_line = ['format code', 'extension', 'resolution', 'note']
1975         self.to_screen(
1976             '[info] Available formats for %s:\n%s' %
1977             (info_dict['id'], render_table(header_line, table)))
1978
1979     def list_thumbnails(self, info_dict):
1980         thumbnails = info_dict.get('thumbnails')
1981         if not thumbnails:
1982             self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
1983             return
1984
1985         self.to_screen(
1986             '[info] Thumbnails for %s:' % info_dict['id'])
1987         self.to_screen(render_table(
1988             ['ID', 'width', 'height', 'URL'],
1989             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1990
1991     def list_subtitles(self, video_id, subtitles, name='subtitles'):
1992         if not subtitles:
1993             self.to_screen('%s has no %s' % (video_id, name))
1994             return
1995         self.to_screen(
1996             'Available %s for %s:' % (name, video_id))
1997         self.to_screen(render_table(
1998             ['Language', 'formats'],
1999             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
2000                 for lang, formats in subtitles.items()]))
2001
2002     def urlopen(self, req):
2003         """ Start an HTTP download """
2004         if isinstance(req, compat_basestring):
2005             req = sanitized_Request(req)
2006         return self._opener.open(req, timeout=self._socket_timeout)
2007
2008     def print_debug_header(self):
2009         if not self.params.get('verbose'):
2010             return
2011
2012         if type('') is not compat_str:
2013             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
2014             self.report_warning(
2015                 'Your Python is broken! Update to a newer and supported version')
2016
2017         stdout_encoding = getattr(
2018             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
2019         encoding_str = (
2020             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2021                 locale.getpreferredencoding(),
2022                 sys.getfilesystemencoding(),
2023                 stdout_encoding,
2024                 self.get_encoding()))
2025         write_string(encoding_str, encoding=None)
2026
2027         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
2028         if _LAZY_LOADER:
2029             self._write_string('[debug] Lazy loading extractors enabled' + '\n')
2030         try:
2031             sp = subprocess.Popen(
2032                 ['git', 'rev-parse', '--short', 'HEAD'],
2033                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2034                 cwd=os.path.dirname(os.path.abspath(__file__)))
2035             out, err = sp.communicate()
2036             out = out.decode().strip()
2037             if re.match('[0-9a-f]+', out):
2038                 self._write_string('[debug] Git HEAD: ' + out + '\n')
2039         except Exception:
2040             try:
2041                 sys.exc_clear()
2042             except Exception:
2043                 pass
2044         self._write_string('[debug] Python version %s - %s\n' % (
2045             platform.python_version(), platform_name()))
2046
2047         exe_versions = FFmpegPostProcessor.get_versions(self)
2048         exe_versions['rtmpdump'] = rtmpdump_version()
2049         exe_str = ', '.join(
2050             '%s %s' % (exe, v)
2051             for exe, v in sorted(exe_versions.items())
2052             if v
2053         )
2054         if not exe_str:
2055             exe_str = 'none'
2056         self._write_string('[debug] exe versions: %s\n' % exe_str)
2057
2058         proxy_map = {}
2059         for handler in self._opener.handlers:
2060             if hasattr(handler, 'proxies'):
2061                 proxy_map.update(handler.proxies)
2062         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2063
2064         if self.params.get('call_home', False):
2065             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2066             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2067             latest_version = self.urlopen(
2068                 'https://yt-dl.org/latest/version').read().decode('utf-8')
2069             if version_tuple(latest_version) > version_tuple(__version__):
2070                 self.report_warning(
2071                     'You are using an outdated version (newest version: %s)! '
2072                     'See https://yt-dl.org/update if you need help updating.' %
2073                     latest_version)
2074
2075     def _setup_opener(self):
2076         timeout_val = self.params.get('socket_timeout')
2077         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2078
2079         opts_cookiefile = self.params.get('cookiefile')
2080         opts_proxy = self.params.get('proxy')
2081
2082         if opts_cookiefile is None:
2083             self.cookiejar = compat_cookiejar.CookieJar()
2084         else:
2085             opts_cookiefile = compat_expanduser(opts_cookiefile)
2086             self.cookiejar = compat_cookiejar.MozillaCookieJar(
2087                 opts_cookiefile)
2088             if os.access(opts_cookiefile, os.R_OK):
2089                 self.cookiejar.load()
2090
2091         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2092         if opts_proxy is not None:
2093             if opts_proxy == '':
2094                 proxies = {}
2095             else:
2096                 proxies = {'http': opts_proxy, 'https': opts_proxy}
2097         else:
2098             proxies = compat_urllib_request.getproxies()
2099             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2100             if 'http' in proxies and 'https' not in proxies:
2101                 proxies['https'] = proxies['http']
2102         proxy_handler = PerRequestProxyHandler(proxies)
2103
2104         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2105         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2106         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2107         data_handler = compat_urllib_request_DataHandler()
2108
2109         # When passing our own FileHandler instance, build_opener won't add the
2110         # default FileHandler and allows us to disable the file protocol, which
2111         # can be used for malicious purposes (see
2112         # https://github.com/rg3/youtube-dl/issues/8227)
2113         file_handler = compat_urllib_request.FileHandler()
2114
2115         def file_open(*args, **kwargs):
2116             raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2117         file_handler.file_open = file_open
2118
2119         opener = compat_urllib_request.build_opener(
2120             proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2121
2122         # Delete the default user-agent header, which would otherwise apply in
2123         # cases where our custom HTTP handler doesn't come into play
2124         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2125         opener.addheaders = []
2126         self._opener = opener
2127
2128     def encode(self, s):
2129         if isinstance(s, bytes):
2130             return s  # Already encoded
2131
2132         try:
2133             return s.encode(self.get_encoding())
2134         except UnicodeEncodeError as err:
2135             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2136             raise
2137
2138     def get_encoding(self):
2139         encoding = self.params.get('encoding')
2140         if encoding is None:
2141             encoding = preferredencoding()
2142         return encoding
2143
2144     def _write_thumbnails(self, info_dict, filename):
2145         if self.params.get('writethumbnail', False):
2146             thumbnails = info_dict.get('thumbnails')
2147             if thumbnails:
2148                 thumbnails = [thumbnails[-1]]
2149         elif self.params.get('write_all_thumbnails', False):
2150             thumbnails = info_dict.get('thumbnails')
2151         else:
2152             return
2153
2154         if not thumbnails:
2155             # No thumbnails present, so return immediately
2156             return
2157
2158         for t in thumbnails:
2159             thumb_ext = determine_ext(t['url'], 'jpg')
2160             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2161             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2162             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2163
2164             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2165                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2166                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2167             else:
2168                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2169                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2170                 try:
2171                     uf = self.urlopen(t['url'])
2172                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2173                         shutil.copyfileobj(uf, thumbf)
2174                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2175                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2176                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2177                     self.report_warning('Unable to download thumbnail "%s": %s' %
2178                                         (t['url'], error_to_compat_str(err)))