_ Git - youtube-dl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # coding: utf-8
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import contextlib
   8 import copy
   9 import datetime
  10 import errno
  11 import fileinput
  12 import io
  13 import itertools
  14 import json
  15 import locale
  16 import operator
  17 import os
  18 import platform
  19 import re
  20 import shutil
  21 import subprocess
  22 import socket
  23 import sys
  24 import time
  25 import tokenize
  26 import traceback
  27 import random
  28
  29 from string import ascii_letters
  30
  31 from .compat import (
  32     compat_basestring,
  33     compat_cookiejar,
  34     compat_get_terminal_size,
  35     compat_http_client,
  36     compat_kwargs,
  37     compat_numeric_types,
  38     compat_os_name,
  39     compat_str,
  40     compat_tokenize_tokenize,
  41     compat_urllib_error,
  42     compat_urllib_request,
  43     compat_urllib_request_DataHandler,
  44 )
  45 from .utils import (
  46     age_restricted,
  47     args_to_str,
  48     ContentTooShortError,
  49     date_from_str,
  50     DateRange,
  51     DEFAULT_OUTTMPL,
  52     determine_ext,
  53     determine_protocol,
  54     DownloadError,
  55     encode_compat_str,
  56     encodeFilename,
  57     error_to_compat_str,
  58     expand_path,
  59     ExtractorError,
  60     format_bytes,
  61     formatSeconds,
  62     GeoRestrictedError,
  63     int_or_none,
  64     ISO3166Utils,
  65     locked_file,
  66     make_HTTPS_handler,
  67     MaxDownloadsReached,
  68     PagedList,
  69     parse_filesize,
  70     PerRequestProxyHandler,
  71     platform_name,
  72     PostProcessingError,
  73     preferredencoding,
  74     prepend_extension,
  75     register_socks_protocols,
  76     render_table,
  77     replace_extension,
  78     SameFileError,
  79     sanitize_filename,
  80     sanitize_path,
  81     sanitize_url,
  82     sanitized_Request,
  83     std_headers,
  84     subtitles_filename,
  85     UnavailableVideoError,
  86     url_basename,
  87     version_tuple,
  88     write_json_file,
  89     write_string,
  90     YoutubeDLCookieProcessor,
  91     YoutubeDLHandler,
  92 )
  93 from .cache import Cache
  94 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
  95 from .downloader import get_suitable_downloader
  96 from .downloader.rtmp import rtmpdump_version
  97 from .postprocessor import (
  98     FFmpegFixupM3u8PP,
  99     FFmpegFixupM4aPP,
 100     FFmpegFixupStretchedPP,
 101     FFmpegMergerPP,
 102     FFmpegPostProcessor,
 103     get_postprocessor,
 104 )
 105 from .version import __version__
 106
 107 if compat_os_name == 'nt':
 108     import ctypes
 109
 110
 111 class YoutubeDL(object):
 112     """YoutubeDL class.
 113
 114     YoutubeDL objects are the ones responsible of downloading the
 115     actual video file and writing it to disk if the user has requested
 116     it, among some other tasks. In most cases there should be one per
 117     program. As, given a video URL, the downloader doesn't know how to
 118     extract all the needed information, task that InfoExtractors do, it
 119     has to pass the URL to one of them.
 120
 121     For this, YoutubeDL objects have a method that allows
 122     InfoExtractors to be registered in a given order. When it is passed
 123     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 124     finds that reports being able to handle it. The InfoExtractor extracts
 125     all the information about the video or videos the URL refers to, and
 126     YoutubeDL process the extracted information, possibly using a File
 127     Downloader to download the video.
 128
 129     YoutubeDL objects accept a lot of parameters. In order not to saturate
 130     the object constructor with arguments, it receives a dictionary of
 131     options instead. These options are available through the params
 132     attribute for the InfoExtractors to use. The YoutubeDL also
 133     registers itself as the downloader in charge for the InfoExtractors
 134     that are added to it, so this is a "mutual registration".
 135
 136     Available options:
 137
 138     username:          Username for authentication purposes.
 139     password:          Password for authentication purposes.
 140     videopassword:     Password for accessing a video.
 141     ap_mso:            Adobe Pass multiple-system operator identifier.
 142     ap_username:       Multiple-system operator account username.
 143     ap_password:       Multiple-system operator account password.
 144     usenetrc:          Use netrc for authentication instead.
 145     verbose:           Print additional info to stdout.
 146     quiet:             Do not print messages to stdout.
 147     no_warnings:       Do not print out anything for warnings.
 148     forceurl:          Force printing final URL.
 149     forcetitle:        Force printing title.
 150     forceid:           Force printing ID.
 151     forcethumbnail:    Force printing thumbnail URL.
 152     forcedescription:  Force printing description.
 153     forcefilename:     Force printing final filename.
 154     forceduration:     Force printing duration.
 155     forcejson:         Force printing info_dict as JSON.
 156     dump_single_json:  Force printing the info_dict of the whole playlist
 157                        (or video) as a single JSON line.
 158     simulate:          Do not download the video files.
 159     format:            Video format code. See options.py for more information.
 160     outtmpl:           Template for output names.
 161     restrictfilenames: Do not allow "&" and spaces in file names
 162     ignoreerrors:      Do not stop on download errors.
 163     force_generic_extractor: Force downloader to use the generic extractor
 164     nooverwrites:      Prevent overwriting files.
 165     playliststart:     Playlist item to start at.
 166     playlistend:       Playlist item to end at.
 167     playlist_items:    Specific indices of playlist to download.
 168     playlistreverse:   Download playlist items in reverse order.
 169     playlistrandom:    Download playlist items in random order.
 170     matchtitle:        Download only matching titles.
 171     rejecttitle:       Reject downloads for matching titles.
 172     logger:            Log messages to a logging.Logger instance.
 173     logtostderr:       Log messages to stderr instead of stdout.
 174     writedescription:  Write the video description to a .description file
 175     writeinfojson:     Write the video description to a .info.json file
 176     writeannotations:  Write the video annotations to a .annotations.xml file
 177     writethumbnail:    Write the thumbnail image to a file
 178     write_all_thumbnails:  Write all thumbnail formats to files
 179     writesubtitles:    Write the video subtitles to a file
 180     writeautomaticsub: Write the automatically generated subtitles to a file
 181     allsubtitles:      Downloads all the subtitles of the video
 182                        (requires writesubtitles or writeautomaticsub)
 183     listsubtitles:     Lists all available subtitles for the video
 184     subtitlesformat:   The format code for subtitles
 185     subtitleslangs:    List of languages of the subtitles to download
 186     keepvideo:         Keep the video file after post-processing
 187     daterange:         A DateRange object, download only if the upload_date is in the range.
 188     skip_download:     Skip the actual download of the video file
 189     cachedir:          Location of the cache files in the filesystem.
 190                        False to disable filesystem cache.
 191     noplaylist:        Download single video instead of a playlist if in doubt.
 192     age_limit:         An integer representing the user's age in years.
 193                        Unsuitable videos for the given age are skipped.
 194     min_views:         An integer representing the minimum view count the video
 195                        must have in order to not be skipped.
 196                        Videos without view count information are always
 197                        downloaded. None for no limit.
 198     max_views:         An integer representing the maximum view count.
 199                        Videos that are more popular than that are not
 200                        downloaded.
 201                        Videos without view count information are always
 202                        downloaded. None for no limit.
 203     download_archive:  File name of a file where all downloads are recorded.
 204                        Videos already present in the file are not downloaded
 205                        again.
 206     cookiefile:        File name where cookies should be read from and dumped to.
 207     nocheckcertificate:Do not verify SSL certificates
 208     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 209                        At the moment, this is only supported by YouTube.
 210     proxy:             URL of the proxy server to use
 211     geo_verification_proxy:  URL of the proxy to use for IP address verification
 212                        on geo-restricted sites. (Experimental)
 213     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 214     bidi_workaround:   Work around buggy terminals without bidirectional text
 215                        support, using fridibi
 216     debug_printtraffic:Print out sent and received HTTP traffic
 217     include_ads:       Download ads as well
 218     default_search:    Prepend this string if an input url is not valid.
 219                        'auto' for elaborate guessing
 220     encoding:          Use this encoding instead of the system-specified.
 221     extract_flat:      Do not resolve URLs, return the immediate result.
 222                        Pass in 'in_playlist' to only show this behavior for
 223                        playlist items.
 224     postprocessors:    A list of dictionaries, each with an entry
 225                        * key:  The name of the postprocessor. See
 226                                youtube_dl/postprocessor/__init__.py for a list.
 227                        as well as any further keyword arguments for the
 228                        postprocessor.
 229     progress_hooks:    A list of functions that get called on download
 230                        progress, with a dictionary with the entries
 231                        * status: One of "downloading", "error", or "finished".
 232                                  Check this first and ignore unknown values.
 233
 234                        If status is one of "downloading", or "finished", the
 235                        following properties may also be present:
 236                        * filename: The final filename (always present)
 237                        * tmpfilename: The filename we're currently writing to
 238                        * downloaded_bytes: Bytes on disk
 239                        * total_bytes: Size of the whole file, None if unknown
 240                        * total_bytes_estimate: Guess of the eventual file size,
 241                                                None if unavailable.
 242                        * elapsed: The number of seconds since download started.
 243                        * eta: The estimated time in seconds, None if unknown
 244                        * speed: The download speed in bytes/second, None if
 245                                 unknown
 246                        * fragment_index: The counter of the currently
 247                                          downloaded video fragment.
 248                        * fragment_count: The number of fragments (= individual
 249                                          files that will be merged)
 250
 251                        Progress hooks are guaranteed to be called at least once
 252                        (with status "finished") if the download is successful.
 253     merge_output_format: Extension to use when merging formats.
 254     fixup:             Automatically correct known faults of the file.
 255                        One of:
 256                        - "never": do nothing
 257                        - "warn": only emit a warning
 258                        - "detect_or_warn": check whether we can do anything
 259                                            about it, warn otherwise (default)
 260     source_address:    (Experimental) Client-side IP address to bind to.
 261     call_home:         Boolean, true iff we are allowed to contact the
 262                        youtube-dl servers for debugging.
 263     sleep_interval:    Number of seconds to sleep before each download when
 264                        used alone or a lower bound of a range for randomized
 265                        sleep before each download (minimum possible number
 266                        of seconds to sleep) when used along with
 267                        max_sleep_interval.
 268     max_sleep_interval:Upper bound of a range for randomized sleep before each
 269                        download (maximum possible number of seconds to sleep).
 270                        Must only be used along with sleep_interval.
 271                        Actual sleep time will be a random float from range
 272                        [sleep_interval; max_sleep_interval].
 273     listformats:       Print an overview of available video formats and exit.
 274     list_thumbnails:   Print a table of all thumbnails and exit.
 275     match_filter:      A function that gets called with the info_dict of
 276                        every video.
 277                        If it returns a message, the video is ignored.
 278                        If it returns None, the video is downloaded.
 279                        match_filter_func in utils.py is one example for this.
 280     no_color:          Do not emit color codes in output.
 281     geo_bypass:        Bypass geographic restriction via faking X-Forwarded-For
 282                        HTTP header (experimental)
 283     geo_bypass_country:
 284                        Two-letter ISO 3166-2 country code that will be used for
 285                        explicit geographic restriction bypassing via faking
 286                        X-Forwarded-For HTTP header (experimental)
 287
 288     The following options determine which downloader is picked:
 289     external_downloader: Executable of the external downloader to call.
 290                        None or unset for standard (built-in) downloader.
 291     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
 292                        if True, otherwise use ffmpeg/avconv if False, otherwise
 293                        use downloader suggested by extractor if None.
 294
 295     The following parameters are not used by YoutubeDL itself, they are used by
 296     the downloader (see youtube_dl/downloader/common.py):
 297     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 298     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 299     xattr_set_filesize, external_downloader_args, hls_use_mpegts.
 300
 301     The following options are used by the post processors:
 302     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 303                        otherwise prefer avconv.
 304     postprocessor_args: A list of additional command-line arguments for the
 305                         postprocessor.
 306     """
 307
 308     _NUMERIC_FIELDS = set((
 309         'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
 310         'timestamp', 'upload_year', 'upload_month', 'upload_day',
 311         'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
 312         'average_rating', 'comment_count', 'age_limit',
 313         'start_time', 'end_time',
 314         'chapter_number', 'season_number', 'episode_number',
 315         'track_number', 'disc_number', 'release_year',
 316         'playlist_index',
 317     ))
 318
 319     params = None
 320     _ies = []
 321     _pps = []
 322     _download_retcode = None
 323     _num_downloads = None
 324     _screen_file = None
 325
 326     def __init__(self, params=None, auto_init=True):
 327         """Create a FileDownloader object with the given options."""
 328         if params is None:
 329             params = {}
 330         self._ies = []
 331         self._ies_instances = {}
 332         self._pps = []
 333         self._progress_hooks = []
 334         self._download_retcode = 0
 335         self._num_downloads = 0
 336         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 337         self._err_file = sys.stderr
 338         self.params = {
 339             # Default parameters
 340             'nocheckcertificate': False,
 341         }
 342         self.params.update(params)
 343         self.cache = Cache(self)
 344
 345         def check_deprecated(param, option, suggestion):
 346             if self.params.get(param) is not None:
 347                 self.report_warning(
 348                     '%s is deprecated. Use %s instead.' % (option, suggestion))
 349                 return True
 350             return False
 351
 352         if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
 353             if self.params.get('geo_verification_proxy') is None:
 354                 self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
 355
 356         check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
 357         check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
 358         check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
 359
 360         if params.get('bidi_workaround', False):
 361             try:
 362                 import pty
 363                 master, slave = pty.openpty()
 364                 width = compat_get_terminal_size().columns
 365                 if width is None:
 366                     width_args = []
 367                 else:
 368                     width_args = ['-w', str(width)]
 369                 sp_kwargs = dict(
 370                     stdin=subprocess.PIPE,
 371                     stdout=slave,
 372                     stderr=self._err_file)
 373                 try:
 374                     self._output_process = subprocess.Popen(
 375                         ['bidiv'] + width_args, **sp_kwargs
 376                     )
 377                 except OSError:
 378                     self._output_process = subprocess.Popen(
 379                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 380                 self._output_channel = os.fdopen(master, 'rb')
 381             except OSError as ose:
 382                 if ose.errno == errno.ENOENT:
 383                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 384                 else:
 385                     raise
 386
 387         if (sys.platform != 'win32' and
 388                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
 389                 not params.get('restrictfilenames', False)):
 390             # Unicode filesystem API will throw errors (#1474, #13027)
 391             self.report_warning(
 392                 'Assuming --restrict-filenames since file system encoding '
 393                 'cannot encode all characters. '
 394                 'Set the LC_ALL environment variable to fix this.')
 395             self.params['restrictfilenames'] = True
 396
 397         if isinstance(params.get('outtmpl'), bytes):
 398             self.report_warning(
 399                 'Parameter outtmpl is bytes, but should be a unicode string. '
 400                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 401
 402         self._setup_opener()
 403
 404         if auto_init:
 405             self.print_debug_header()
 406             self.add_default_info_extractors()
 407
 408         for pp_def_raw in self.params.get('postprocessors', []):
 409             pp_class = get_postprocessor(pp_def_raw['key'])
 410             pp_def = dict(pp_def_raw)
 411             del pp_def['key']
 412             pp = pp_class(self, **compat_kwargs(pp_def))
 413             self.add_post_processor(pp)
 414
 415         for ph in self.params.get('progress_hooks', []):
 416             self.add_progress_hook(ph)
 417
 418         register_socks_protocols()
 419
 420     def warn_if_short_id(self, argv):
 421         # short YouTube ID starting with dash?
 422         idxs = [
 423             i for i, a in enumerate(argv)
 424             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 425         if idxs:
 426             correct_argv = (
 427                 ['youtube-dl'] +
 428                 [a for i, a in enumerate(argv) if i not in idxs] +
 429                 ['--'] + [argv[i] for i in idxs]
 430             )
 431             self.report_warning(
 432                 'Long argument string detected. '
 433                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 434                 args_to_str(correct_argv))
 435
 436     def add_info_extractor(self, ie):
 437         """Add an InfoExtractor object to the end of the list."""
 438         self._ies.append(ie)
 439         if not isinstance(ie, type):
 440             self._ies_instances[ie.ie_key()] = ie
 441             ie.set_downloader(self)
 442
 443     def get_info_extractor(self, ie_key):
 444         """
 445         Get an instance of an IE with name ie_key, it will try to get one from
 446         the _ies list, if there's no instance it will create a new one and add
 447         it to the extractor list.
 448         """
 449         ie = self._ies_instances.get(ie_key)
 450         if ie is None:
 451             ie = get_info_extractor(ie_key)()
 452             self.add_info_extractor(ie)
 453         return ie
 454
 455     def add_default_info_extractors(self):
 456         """
 457         Add the InfoExtractors returned by gen_extractors to the end of the list
 458         """
 459         for ie in gen_extractor_classes():
 460             self.add_info_extractor(ie)
 461
 462     def add_post_processor(self, pp):
 463         """Add a PostProcessor object to the end of the chain."""
 464         self._pps.append(pp)
 465         pp.set_downloader(self)
 466
 467     def add_progress_hook(self, ph):
 468         """Add the progress hook (currently only for the file downloader)"""
 469         self._progress_hooks.append(ph)
 470
 471     def _bidi_workaround(self, message):
 472         if not hasattr(self, '_output_channel'):
 473             return message
 474
 475         assert hasattr(self, '_output_process')
 476         assert isinstance(message, compat_str)
 477         line_count = message.count('\n') + 1
 478         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 479         self._output_process.stdin.flush()
 480         res = ''.join(self._output_channel.readline().decode('utf-8')
 481                       for _ in range(line_count))
 482         return res[:-len('\n')]
 483
 484     def to_screen(self, message, skip_eol=False):
 485         """Print message to stdout if not in quiet mode."""
 486         return self.to_stdout(message, skip_eol, check_quiet=True)
 487
 488     def _write_string(self, s, out=None):
 489         write_string(s, out=out, encoding=self.params.get('encoding'))
 490
 491     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 492         """Print message to stdout if not in quiet mode."""
 493         if self.params.get('logger'):
 494             self.params['logger'].debug(message)
 495         elif not check_quiet or not self.params.get('quiet', False):
 496             message = self._bidi_workaround(message)
 497             terminator = ['\n', ''][skip_eol]
 498             output = message + terminator
 499
 500             self._write_string(output, self._screen_file)
 501
 502     def to_stderr(self, message):
 503         """Print message to stderr."""
 504         assert isinstance(message, compat_str)
 505         if self.params.get('logger'):
 506             self.params['logger'].error(message)
 507         else:
 508             message = self._bidi_workaround(message)
 509             output = message + '\n'
 510             self._write_string(output, self._err_file)
 511
 512     def to_console_title(self, message):
 513         if not self.params.get('consoletitle', False):
 514             return
 515         if compat_os_name == 'nt':
 516             if ctypes.windll.kernel32.GetConsoleWindow():
 517                 # c_wchar_p() might not be necessary if `message` is
 518                 # already of type unicode()
 519                 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 520         elif 'TERM' in os.environ:
 521             self._write_string('\033]0;%s\007' % message, self._screen_file)
 522
 523     def save_console_title(self):
 524         if not self.params.get('consoletitle', False):
 525             return
 526         if compat_os_name != 'nt' and 'TERM' in os.environ:
 527             # Save the title on stack
 528             self._write_string('\033[22;0t', self._screen_file)
 529
 530     def restore_console_title(self):
 531         if not self.params.get('consoletitle', False):
 532             return
 533         if compat_os_name != 'nt' and 'TERM' in os.environ:
 534             # Restore the title from stack
 535             self._write_string('\033[23;0t', self._screen_file)
 536
 537     def __enter__(self):
 538         self.save_console_title()
 539         return self
 540
 541     def __exit__(self, *args):
 542         self.restore_console_title()
 543
 544         if self.params.get('cookiefile') is not None:
 545             self.cookiejar.save()
 546
 547     def trouble(self, message=None, tb=None):
 548         """Determine action to take when a download problem appears.
 549
 550         Depending on if the downloader has been configured to ignore
 551         download errors or not, this method may throw an exception or
 552         not when errors are found, after printing the message.
 553
 554         tb, if given, is additional traceback information.
 555         """
 556         if message is not None:
 557             self.to_stderr(message)
 558         if self.params.get('verbose'):
 559             if tb is None:
 560                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 561                     tb = ''
 562                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 563                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 564                     tb += encode_compat_str(traceback.format_exc())
 565                 else:
 566                     tb_data = traceback.format_list(traceback.extract_stack())
 567                     tb = ''.join(tb_data)
 568             self.to_stderr(tb)
 569         if not self.params.get('ignoreerrors', False):
 570             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 571                 exc_info = sys.exc_info()[1].exc_info
 572             else:
 573                 exc_info = sys.exc_info()
 574             raise DownloadError(message, exc_info)
 575         self._download_retcode = 1
 576
 577     def report_warning(self, message):
 578         '''
 579         Print the message to stderr, it will be prefixed with 'WARNING:'
 580         If stderr is a tty file the 'WARNING:' will be colored
 581         '''
 582         if self.params.get('logger') is not None:
 583             self.params['logger'].warning(message)
 584         else:
 585             if self.params.get('no_warnings'):
 586                 return
 587             if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 588                 _msg_header = '\033[0;33mWARNING:\033[0m'
 589             else:
 590                 _msg_header = 'WARNING:'
 591             warning_message = '%s %s' % (_msg_header, message)
 592             self.to_stderr(warning_message)
 593
 594     def report_error(self, message, tb=None):
 595         '''
 596         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 597         in red if stderr is a tty file.
 598         '''
 599         if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 600             _msg_header = '\033[0;31mERROR:\033[0m'
 601         else:
 602             _msg_header = 'ERROR:'
 603         error_message = '%s %s' % (_msg_header, message)
 604         self.trouble(error_message, tb)
 605
 606     def report_file_already_downloaded(self, file_name):
 607         """Report file has already been fully downloaded."""
 608         try:
 609             self.to_screen('[download] %s has already been downloaded' % file_name)
 610         except UnicodeEncodeError:
 611             self.to_screen('[download] The file has already been downloaded')
 612
 613     def prepare_filename(self, info_dict):
 614         """Generate the output filename."""
 615         try:
 616             template_dict = dict(info_dict)
 617
 618             template_dict['epoch'] = int(time.time())
 619             autonumber_size = self.params.get('autonumber_size')
 620             if autonumber_size is None:
 621                 autonumber_size = 5
 622             template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
 623             if template_dict.get('resolution') is None:
 624                 if template_dict.get('width') and template_dict.get('height'):
 625                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 626                 elif template_dict.get('height'):
 627                     template_dict['resolution'] = '%sp' % template_dict['height']
 628                 elif template_dict.get('width'):
 629                     template_dict['resolution'] = '%dx?' % template_dict['width']
 630
 631             sanitize = lambda k, v: sanitize_filename(
 632                 compat_str(v),
 633                 restricted=self.params.get('restrictfilenames'),
 634                 is_id=(k == 'id' or k.endswith('_id')))
 635             template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
 636                                  for k, v in template_dict.items()
 637                                  if v is not None and not isinstance(v, (list, tuple, dict)))
 638             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 639
 640             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 641
 642             # For fields playlist_index and autonumber convert all occurrences
 643             # of %(field)s to %(field)0Nd for backward compatibility
 644             field_size_compat_map = {
 645                 'playlist_index': len(str(template_dict['n_entries'])),
 646                 'autonumber': autonumber_size,
 647             }
 648             FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
 649             mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl)
 650             if mobj:
 651                 outtmpl = re.sub(
 652                     FIELD_SIZE_COMPAT_RE,
 653                     r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
 654                     outtmpl)
 655
 656             # Missing numeric fields used together with integer presentation types
 657             # in format specification will break the argument substitution since
 658             # string 'NA' is returned for missing fields. We will patch output
 659             # template for missing fields to meet string presentation type.
 660             for numeric_field in self._NUMERIC_FIELDS:
 661                 if numeric_field not in template_dict:
 662                     # As of [1] format syntax is:
 663                     #  %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
 664                     # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
 665                     FORMAT_RE = r'''(?x)
 666                         (?<!%)
 667                         %
 668                         \({0}\)  # mapping key
 669                         (?:[#0\-+ ]+)?  # conversion flags (optional)
 670                         (?:\d+)?  # minimum field width (optional)
 671                         (?:\.\d+)?  # precision (optional)
 672                         [hlL]?  # length modifier (optional)
 673                         [diouxXeEfFgGcrs%]  # conversion type
 674                     '''
 675                     outtmpl = re.sub(
 676                         FORMAT_RE.format(numeric_field),
 677                         r'%({0})s'.format(numeric_field), outtmpl)
 678
 679             # expand_path translates '%%' into '%' and '$$' into '$'
 680             # correspondingly that is not what we want since we need to keep
 681             # '%%' intact for template dict substitution step. Working around
 682             # with boundary-alike separator hack.
 683             sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
 684             outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
 685
 686             # outtmpl should be expand_path'ed before template dict substitution
 687             # because meta fields may contain env variables we don't want to
 688             # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
 689             # title "Hello $PATH", we don't want `$PATH` to be expanded.
 690             filename = expand_path(outtmpl).replace(sep, '') % template_dict
 691
 692             # Temporary fix for #4787
 693             # 'Treat' all problem characters by passing filename through preferredencoding
 694             # to workaround encoding issues with subprocess on python2 @ Windows
 695             if sys.version_info < (3, 0) and sys.platform == 'win32':
 696                 filename = encodeFilename(filename, True).decode(preferredencoding())
 697             return sanitize_path(filename)
 698         except ValueError as err:
 699             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 700             return None
 701
 702     def _match_entry(self, info_dict, incomplete):
 703         """ Returns None iff the file should be downloaded """
 704
 705         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 706         if 'title' in info_dict:
 707             # This can happen when we're just evaluating the playlist
 708             title = info_dict['title']
 709             matchtitle = self.params.get('matchtitle', False)
 710             if matchtitle:
 711                 if not re.search(matchtitle, title, re.IGNORECASE):
 712                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 713             rejecttitle = self.params.get('rejecttitle', False)
 714             if rejecttitle:
 715                 if re.search(rejecttitle, title, re.IGNORECASE):
 716                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 717         date = info_dict.get('upload_date')
 718         if date is not None:
 719             dateRange = self.params.get('daterange', DateRange())
 720             if date not in dateRange:
 721                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 722         view_count = info_dict.get('view_count')
 723         if view_count is not None:
 724             min_views = self.params.get('min_views')
 725             if min_views is not None and view_count < min_views:
 726                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 727             max_views = self.params.get('max_views')
 728             if max_views is not None and view_count > max_views:
 729                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 730         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 731             return 'Skipping "%s" because it is age restricted' % video_title
 732         if self.in_download_archive(info_dict):
 733             return '%s has already been recorded in archive' % video_title
 734
 735         if not incomplete:
 736             match_filter = self.params.get('match_filter')
 737             if match_filter is not None:
 738                 ret = match_filter(info_dict)
 739                 if ret is not None:
 740                     return ret
 741
 742         return None
 743
 744     @staticmethod
 745     def add_extra_info(info_dict, extra_info):
 746         '''Set the keys from extra_info in info dict if they are missing'''
 747         for key, value in extra_info.items():
 748             info_dict.setdefault(key, value)
 749
 750     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 751                      process=True, force_generic_extractor=False):
 752         '''
 753         Returns a list with a dictionary for each video we find.
 754         If 'download', also downloads the videos.
 755         extra_info is a dict containing the extra values to add to each result
 756         '''
 757
 758         if not ie_key and force_generic_extractor:
 759             ie_key = 'Generic'
 760
 761         if ie_key:
 762             ies = [self.get_info_extractor(ie_key)]
 763         else:
 764             ies = self._ies
 765
 766         for ie in ies:
 767             if not ie.suitable(url):
 768                 continue
 769
 770             ie = self.get_info_extractor(ie.ie_key())
 771             if not ie.working():
 772                 self.report_warning('The program functionality for this site has been marked as broken, '
 773                                     'and will probably not work.')
 774
 775             try:
 776                 ie_result = ie.extract(url)
 777                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 778                     break
 779                 if isinstance(ie_result, list):
 780                     # Backwards compatibility: old IE result format
 781                     ie_result = {
 782                         '_type': 'compat_list',
 783                         'entries': ie_result,
 784                     }
 785                 self.add_default_extra_info(ie_result, ie, url)
 786                 if process:
 787                     return self.process_ie_result(ie_result, download, extra_info)
 788                 else:
 789                     return ie_result
 790             except GeoRestrictedError as e:
 791                 msg = e.msg
 792                 if e.countries:
 793                     msg += '\nThis video is available in %s.' % ', '.join(
 794                         map(ISO3166Utils.short2full, e.countries))
 795                 msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
 796                 self.report_error(msg)
 797                 break
 798             except ExtractorError as e:  # An error we somewhat expected
 799                 self.report_error(compat_str(e), e.format_traceback())
 800                 break
 801             except MaxDownloadsReached:
 802                 raise
 803             except Exception as e:
 804                 if self.params.get('ignoreerrors', False):
 805                     self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
 806                     break
 807                 else:
 808                     raise
 809         else:
 810             self.report_error('no suitable InfoExtractor for URL %s' % url)
 811
 812     def add_default_extra_info(self, ie_result, ie, url):
 813         self.add_extra_info(ie_result, {
 814             'extractor': ie.IE_NAME,
 815             'webpage_url': url,
 816             'webpage_url_basename': url_basename(url),
 817             'extractor_key': ie.ie_key(),
 818         })
 819
 820     def process_ie_result(self, ie_result, download=True, extra_info={}):
 821         """
 822         Take the result of the ie(may be modified) and resolve all unresolved
 823         references (URLs, playlist items).
 824
 825         It will also download the videos if 'download'.
 826         Returns the resolved ie_result.
 827         """
 828         result_type = ie_result.get('_type', 'video')
 829
 830         if result_type in ('url', 'url_transparent'):
 831             ie_result['url'] = sanitize_url(ie_result['url'])
 832             extract_flat = self.params.get('extract_flat', False)
 833             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 834                     extract_flat is True):
 835                 if self.params.get('forcejson', False):
 836                     self.to_stdout(json.dumps(ie_result))
 837                 return ie_result
 838
 839         if result_type == 'video':
 840             self.add_extra_info(ie_result, extra_info)
 841             return self.process_video_result(ie_result, download=download)
 842         elif result_type == 'url':
 843             # We have to add extra_info to the results because it may be
 844             # contained in a playlist
 845             return self.extract_info(ie_result['url'],
 846                                      download,
 847                                      ie_key=ie_result.get('ie_key'),
 848                                      extra_info=extra_info)
 849         elif result_type == 'url_transparent':
 850             # Use the information from the embedding page
 851             info = self.extract_info(
 852                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 853                 extra_info=extra_info, download=False, process=False)
 854
 855             # extract_info may return None when ignoreerrors is enabled and
 856             # extraction failed with an error, don't crash and return early
 857             # in this case
 858             if not info:
 859                 return info
 860
 861             force_properties = dict(
 862                 (k, v) for k, v in ie_result.items() if v is not None)
 863             for f in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
 864                 if f in force_properties:
 865                     del force_properties[f]
 866             new_result = info.copy()
 867             new_result.update(force_properties)
 868
 869             # Extracted info may not be a video result (i.e.
 870             # info.get('_type', 'video') != video) but rather an url or
 871             # url_transparent. In such cases outer metadata (from ie_result)
 872             # should be propagated to inner one (info). For this to happen
 873             # _type of info should be overridden with url_transparent. This
 874             # fixes issue from https://github.com/rg3/youtube-dl/pull/11163.
 875             if new_result.get('_type') == 'url':
 876                 new_result['_type'] = 'url_transparent'
 877
 878             return self.process_ie_result(
 879                 new_result, download=download, extra_info=extra_info)
 880         elif result_type in ('playlist', 'multi_video'):
 881             # We process each entry in the playlist
 882             playlist = ie_result.get('title') or ie_result.get('id')
 883             self.to_screen('[download] Downloading playlist: %s' % playlist)
 884
 885             playlist_results = []
 886
 887             playliststart = self.params.get('playliststart', 1) - 1
 888             playlistend = self.params.get('playlistend')
 889             # For backwards compatibility, interpret -1 as whole list
 890             if playlistend == -1:
 891                 playlistend = None
 892
 893             playlistitems_str = self.params.get('playlist_items')
 894             playlistitems = None
 895             if playlistitems_str is not None:
 896                 def iter_playlistitems(format):
 897                     for string_segment in format.split(','):
 898                         if '-' in string_segment:
 899                             start, end = string_segment.split('-')
 900                             for item in range(int(start), int(end) + 1):
 901                                 yield int(item)
 902                         else:
 903                             yield int(string_segment)
 904                 playlistitems = iter_playlistitems(playlistitems_str)
 905
 906             ie_entries = ie_result['entries']
 907             if isinstance(ie_entries, list):
 908                 n_all_entries = len(ie_entries)
 909                 if playlistitems:
 910                     entries = [
 911                         ie_entries[i - 1] for i in playlistitems
 912                         if -n_all_entries <= i - 1 < n_all_entries]
 913                 else:
 914                     entries = ie_entries[playliststart:playlistend]
 915                 n_entries = len(entries)
 916                 self.to_screen(
 917                     '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
 918                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 919             elif isinstance(ie_entries, PagedList):
 920                 if playlistitems:
 921                     entries = []
 922                     for item in playlistitems:
 923                         entries.extend(ie_entries.getslice(
 924                             item - 1, item
 925                         ))
 926                 else:
 927                     entries = ie_entries.getslice(
 928                         playliststart, playlistend)
 929                 n_entries = len(entries)
 930                 self.to_screen(
 931                     '[%s] playlist %s: Downloading %d videos' %
 932                     (ie_result['extractor'], playlist, n_entries))
 933             else:  # iterable
 934                 if playlistitems:
 935                     entry_list = list(ie_entries)
 936                     entries = [entry_list[i - 1] for i in playlistitems]
 937                 else:
 938                     entries = list(itertools.islice(
 939                         ie_entries, playliststart, playlistend))
 940                 n_entries = len(entries)
 941                 self.to_screen(
 942                     '[%s] playlist %s: Downloading %d videos' %
 943                     (ie_result['extractor'], playlist, n_entries))
 944
 945             if self.params.get('playlistreverse', False):
 946                 entries = entries[::-1]
 947
 948             if self.params.get('playlistrandom', False):
 949                 random.shuffle(entries)
 950
 951             x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
 952
 953             for i, entry in enumerate(entries, 1):
 954                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 955                 # This __x_forwarded_for_ip thing is a bit ugly but requires
 956                 # minimal changes
 957                 if x_forwarded_for:
 958                     entry['__x_forwarded_for_ip'] = x_forwarded_for
 959                 extra = {
 960                     'n_entries': n_entries,
 961                     'playlist': playlist,
 962                     'playlist_id': ie_result.get('id'),
 963                     'playlist_title': ie_result.get('title'),
 964                     'playlist_index': i + playliststart,
 965                     'extractor': ie_result['extractor'],
 966                     'webpage_url': ie_result['webpage_url'],
 967                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 968                     'extractor_key': ie_result['extractor_key'],
 969                 }
 970
 971                 reason = self._match_entry(entry, incomplete=True)
 972                 if reason is not None:
 973                     self.to_screen('[download] ' + reason)
 974                     continue
 975
 976                 entry_result = self.process_ie_result(entry,
 977                                                       download=download,
 978                                                       extra_info=extra)
 979                 playlist_results.append(entry_result)
 980             ie_result['entries'] = playlist_results
 981             self.to_screen('[download] Finished downloading playlist: %s' % playlist)
 982             return ie_result
 983         elif result_type == 'compat_list':
 984             self.report_warning(
 985                 'Extractor %s returned a compat_list result. '
 986                 'It needs to be updated.' % ie_result.get('extractor'))
 987
 988             def _fixup(r):
 989                 self.add_extra_info(
 990                     r,
 991                     {
 992                         'extractor': ie_result['extractor'],
 993                         'webpage_url': ie_result['webpage_url'],
 994                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 995                         'extractor_key': ie_result['extractor_key'],
 996                     }
 997                 )
 998                 return r
 999             ie_result['entries'] = [
1000                 self.process_ie_result(_fixup(r), download, extra_info)
1001                 for r in ie_result['entries']
1002             ]
1003             return ie_result
1004         else:
1005             raise Exception('Invalid result type: %s' % result_type)
1006
1007     def _build_format_filter(self, filter_spec):
1008         " Returns a function to filter the formats according to the filter_spec "
1009
1010         OPERATORS = {
1011             '<': operator.lt,
1012             '<=': operator.le,
1013             '>': operator.gt,
1014             '>=': operator.ge,
1015             '=': operator.eq,
1016             '!=': operator.ne,
1017         }
1018         operator_rex = re.compile(r'''(?x)\s*
1019             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
1020             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1021             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
1022             $
1023             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
1024         m = operator_rex.search(filter_spec)
1025         if m:
1026             try:
1027                 comparison_value = int(m.group('value'))
1028             except ValueError:
1029                 comparison_value = parse_filesize(m.group('value'))
1030                 if comparison_value is None:
1031                     comparison_value = parse_filesize(m.group('value') + 'B')
1032                 if comparison_value is None:
1033                     raise ValueError(
1034                         'Invalid value %r in format specification %r' % (
1035                             m.group('value'), filter_spec))
1036             op = OPERATORS[m.group('op')]
1037
1038         if not m:
1039             STR_OPERATORS = {
1040                 '=': operator.eq,
1041                 '!=': operator.ne,
1042                 '^=': lambda attr, value: attr.startswith(value),
1043                 '$=': lambda attr, value: attr.endswith(value),
1044                 '*=': lambda attr, value: value in attr,
1045             }
1046             str_operator_rex = re.compile(r'''(?x)
1047                 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
1048                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
1049                 \s*(?P<value>[a-zA-Z0-9._-]+)
1050                 \s*$
1051                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
1052             m = str_operator_rex.search(filter_spec)
1053             if m:
1054                 comparison_value = m.group('value')
1055                 op = STR_OPERATORS[m.group('op')]
1056
1057         if not m:
1058             raise ValueError('Invalid filter specification %r' % filter_spec)
1059
1060         def _filter(f):
1061             actual_value = f.get(m.group('key'))
1062             if actual_value is None:
1063                 return m.group('none_inclusive')
1064             return op(actual_value, comparison_value)
1065         return _filter
1066
1067     def _default_format_spec(self, info_dict, download=True):
1068         req_format_list = []
1069
1070         def can_have_partial_formats():
1071             if self.params.get('simulate', False):
1072                 return True
1073             if not download:
1074                 return True
1075             if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
1076                 return False
1077             if info_dict.get('is_live'):
1078                 return False
1079             merger = FFmpegMergerPP(self)
1080             return merger.available and merger.can_merge()
1081         if can_have_partial_formats():
1082             req_format_list.append('bestvideo+bestaudio')
1083         req_format_list.append('best')
1084         return '/'.join(req_format_list)
1085
1086     def build_format_selector(self, format_spec):
1087         def syntax_error(note, start):
1088             message = (
1089                 'Invalid format specification: '
1090                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
1091             return SyntaxError(message)
1092
1093         PICKFIRST = 'PICKFIRST'
1094         MERGE = 'MERGE'
1095         SINGLE = 'SINGLE'
1096         GROUP = 'GROUP'
1097         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1098
1099         def _parse_filter(tokens):
1100             filter_parts = []
1101             for type, string, start, _, _ in tokens:
1102                 if type == tokenize.OP and string == ']':
1103                     return ''.join(filter_parts)
1104                 else:
1105                     filter_parts.append(string)
1106
1107         def _remove_unused_ops(tokens):
1108             # Remove operators that we don't use and join them with the surrounding strings
1109             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1110             ALLOWED_OPS = ('/', '+', ',', '(', ')')
1111             last_string, last_start, last_end, last_line = None, None, None, None
1112             for type, string, start, end, line in tokens:
1113                 if type == tokenize.OP and string == '[':
1114                     if last_string:
1115                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1116                         last_string = None
1117                     yield type, string, start, end, line
1118                     # everything inside brackets will be handled by _parse_filter
1119                     for type, string, start, end, line in tokens:
1120                         yield type, string, start, end, line
1121                         if type == tokenize.OP and string == ']':
1122                             break
1123                 elif type == tokenize.OP and string in ALLOWED_OPS:
1124                     if last_string:
1125                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1126                         last_string = None
1127                     yield type, string, start, end, line
1128                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
1129                     if not last_string:
1130                         last_string = string
1131                         last_start = start
1132                         last_end = end
1133                     else:
1134                         last_string += string
1135             if last_string:
1136                 yield tokenize.NAME, last_string, last_start, last_end, last_line
1137
1138         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
1139             selectors = []
1140             current_selector = None
1141             for type, string, start, _, _ in tokens:
1142                 # ENCODING is only defined in python 3.x
1143                 if type == getattr(tokenize, 'ENCODING', None):
1144                     continue
1145                 elif type in [tokenize.NAME, tokenize.NUMBER]:
1146                     current_selector = FormatSelector(SINGLE, string, [])
1147                 elif type == tokenize.OP:
1148                     if string == ')':
1149                         if not inside_group:
1150                             # ')' will be handled by the parentheses group
1151                             tokens.restore_last_token()
1152                         break
1153                     elif inside_merge and string in ['/', ',']:
1154                         tokens.restore_last_token()
1155                         break
1156                     elif inside_choice and string == ',':
1157                         tokens.restore_last_token()
1158                         break
1159                     elif string == ',':
1160                         if not current_selector:
1161                             raise syntax_error('"," must follow a format selector', start)
1162                         selectors.append(current_selector)
1163                         current_selector = None
1164                     elif string == '/':
1165                         if not current_selector:
1166                             raise syntax_error('"/" must follow a format selector', start)
1167                         first_choice = current_selector
1168                         second_choice = _parse_format_selection(tokens, inside_choice=True)
1169                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1170                     elif string == '[':
1171                         if not current_selector:
1172                             current_selector = FormatSelector(SINGLE, 'best', [])
1173                         format_filter = _parse_filter(tokens)
1174                         current_selector.filters.append(format_filter)
1175                     elif string == '(':
1176                         if current_selector:
1177                             raise syntax_error('Unexpected "("', start)
1178                         group = _parse_format_selection(tokens, inside_group=True)
1179                         current_selector = FormatSelector(GROUP, group, [])
1180                     elif string == '+':
1181                         video_selector = current_selector
1182                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
1183                         if not video_selector or not audio_selector:
1184                             raise syntax_error('"+" must be between two format selectors', start)
1185                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1186                     else:
1187                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1188                 elif type == tokenize.ENDMARKER:
1189                     break
1190             if current_selector:
1191                 selectors.append(current_selector)
1192             return selectors
1193
1194         def _build_selector_function(selector):
1195             if isinstance(selector, list):
1196                 fs = [_build_selector_function(s) for s in selector]
1197
1198                 def selector_function(ctx):
1199                     for f in fs:
1200                         for format in f(ctx):
1201                             yield format
1202                 return selector_function
1203             elif selector.type == GROUP:
1204                 selector_function = _build_selector_function(selector.selector)
1205             elif selector.type == PICKFIRST:
1206                 fs = [_build_selector_function(s) for s in selector.selector]
1207
1208                 def selector_function(ctx):
1209                     for f in fs:
1210                         picked_formats = list(f(ctx))
1211                         if picked_formats:
1212                             return picked_formats
1213                     return []
1214             elif selector.type == SINGLE:
1215                 format_spec = selector.selector
1216
1217                 def selector_function(ctx):
1218                     formats = list(ctx['formats'])
1219                     if not formats:
1220                         return
1221                     if format_spec == 'all':
1222                         for f in formats:
1223                             yield f
1224                     elif format_spec in ['best', 'worst', None]:
1225                         format_idx = 0 if format_spec == 'worst' else -1
1226                         audiovideo_formats = [
1227                             f for f in formats
1228                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1229                         if audiovideo_formats:
1230                             yield audiovideo_formats[format_idx]
1231                         # for extractors with incomplete formats (audio only (soundcloud)
1232                         # or video only (imgur)) we will fallback to best/worst
1233                         # {video,audio}-only format
1234                         elif ctx['incomplete_formats']:
1235                             yield formats[format_idx]
1236                     elif format_spec == 'bestaudio':
1237                         audio_formats = [
1238                             f for f in formats
1239                             if f.get('vcodec') == 'none']
1240                         if audio_formats:
1241                             yield audio_formats[-1]
1242                     elif format_spec == 'worstaudio':
1243                         audio_formats = [
1244                             f for f in formats
1245                             if f.get('vcodec') == 'none']
1246                         if audio_formats:
1247                             yield audio_formats[0]
1248                     elif format_spec == 'bestvideo':
1249                         video_formats = [
1250                             f for f in formats
1251                             if f.get('acodec') == 'none']
1252                         if video_formats:
1253                             yield video_formats[-1]
1254                     elif format_spec == 'worstvideo':
1255                         video_formats = [
1256                             f for f in formats
1257                             if f.get('acodec') == 'none']
1258                         if video_formats:
1259                             yield video_formats[0]
1260                     else:
1261                         extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1262                         if format_spec in extensions:
1263                             filter_f = lambda f: f['ext'] == format_spec
1264                         else:
1265                             filter_f = lambda f: f['format_id'] == format_spec
1266                         matches = list(filter(filter_f, formats))
1267                         if matches:
1268                             yield matches[-1]
1269             elif selector.type == MERGE:
1270                 def _merge(formats_info):
1271                     format_1, format_2 = [f['format_id'] for f in formats_info]
1272                     # The first format must contain the video and the
1273                     # second the audio
1274                     if formats_info[0].get('vcodec') == 'none':
1275                         self.report_error('The first format must '
1276                                           'contain the video, try using '
1277                                           '"-f %s+%s"' % (format_2, format_1))
1278                         return
1279                     # Formats must be opposite (video+audio)
1280                     if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1281                         self.report_error(
1282                             'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1283                             % (format_1, format_2))
1284                         return
1285                     output_ext = (
1286                         formats_info[0]['ext']
1287                         if self.params.get('merge_output_format') is None
1288                         else self.params['merge_output_format'])
1289                     return {
1290                         'requested_formats': formats_info,
1291                         'format': '%s+%s' % (formats_info[0].get('format'),
1292                                              formats_info[1].get('format')),
1293                         'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1294                                                 formats_info[1].get('format_id')),
1295                         'width': formats_info[0].get('width'),
1296                         'height': formats_info[0].get('height'),
1297                         'resolution': formats_info[0].get('resolution'),
1298                         'fps': formats_info[0].get('fps'),
1299                         'vcodec': formats_info[0].get('vcodec'),
1300                         'vbr': formats_info[0].get('vbr'),
1301                         'stretched_ratio': formats_info[0].get('stretched_ratio'),
1302                         'acodec': formats_info[1].get('acodec'),
1303                         'abr': formats_info[1].get('abr'),
1304                         'ext': output_ext,
1305                     }
1306                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1307
1308                 def selector_function(ctx):
1309                     for pair in itertools.product(
1310                             video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
1311                         yield _merge(pair)
1312
1313             filters = [self._build_format_filter(f) for f in selector.filters]
1314
1315             def final_selector(ctx):
1316                 ctx_copy = copy.deepcopy(ctx)
1317                 for _filter in filters:
1318                     ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
1319                 return selector_function(ctx_copy)
1320             return final_selector
1321
1322         stream = io.BytesIO(format_spec.encode('utf-8'))
1323         try:
1324             tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1325         except tokenize.TokenError:
1326             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1327
1328         class TokenIterator(object):
1329             def __init__(self, tokens):
1330                 self.tokens = tokens
1331                 self.counter = 0
1332
1333             def __iter__(self):
1334                 return self
1335
1336             def __next__(self):
1337                 if self.counter >= len(self.tokens):
1338                     raise StopIteration()
1339                 value = self.tokens[self.counter]
1340                 self.counter += 1
1341                 return value
1342
1343             next = __next__
1344
1345             def restore_last_token(self):
1346                 self.counter -= 1
1347
1348         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1349         return _build_selector_function(parsed_selector)
1350
1351     def _calc_headers(self, info_dict):
1352         res = std_headers.copy()
1353
1354         add_headers = info_dict.get('http_headers')
1355         if add_headers:
1356             res.update(add_headers)
1357
1358         cookies = self._calc_cookies(info_dict)
1359         if cookies:
1360             res['Cookie'] = cookies
1361
1362         if 'X-Forwarded-For' not in res:
1363             x_forwarded_for_ip = info_dict.get('__x_forwarded_for_ip')
1364             if x_forwarded_for_ip:
1365                 res['X-Forwarded-For'] = x_forwarded_for_ip
1366
1367         return res
1368
1369     def _calc_cookies(self, info_dict):
1370         pr = sanitized_Request(info_dict['url'])
1371         self.cookiejar.add_cookie_header(pr)
1372         return pr.get_header('Cookie')
1373
1374     def process_video_result(self, info_dict, download=True):
1375         assert info_dict.get('_type', 'video') == 'video'
1376
1377         if 'id' not in info_dict:
1378             raise ExtractorError('Missing "id" field in extractor result')
1379         if 'title' not in info_dict:
1380             raise ExtractorError('Missing "title" field in extractor result')
1381
1382         def report_force_conversion(field, field_not, conversion):
1383             self.report_warning(
1384                 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1385                 % (field, field_not, conversion))
1386
1387         def sanitize_string_field(info, string_field):
1388             field = info.get(string_field)
1389             if field is None or isinstance(field, compat_str):
1390                 return
1391             report_force_conversion(string_field, 'a string', 'string')
1392             info[string_field] = compat_str(field)
1393
1394         def sanitize_numeric_fields(info):
1395             for numeric_field in self._NUMERIC_FIELDS:
1396                 field = info.get(numeric_field)
1397                 if field is None or isinstance(field, compat_numeric_types):
1398                     continue
1399                 report_force_conversion(numeric_field, 'numeric', 'int')
1400                 info[numeric_field] = int_or_none(field)
1401
1402         sanitize_string_field(info_dict, 'id')
1403         sanitize_numeric_fields(info_dict)
1404
1405         if 'playlist' not in info_dict:
1406             # It isn't part of a playlist
1407             info_dict['playlist'] = None
1408             info_dict['playlist_index'] = None
1409
1410         thumbnails = info_dict.get('thumbnails')
1411         if thumbnails is None:
1412             thumbnail = info_dict.get('thumbnail')
1413             if thumbnail:
1414                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1415         if thumbnails:
1416             thumbnails.sort(key=lambda t: (
1417                 t.get('preference') if t.get('preference') is not None else -1,
1418                 t.get('width') if t.get('width') is not None else -1,
1419                 t.get('height') if t.get('height') is not None else -1,
1420                 t.get('id') if t.get('id') is not None else '', t.get('url')))
1421             for i, t in enumerate(thumbnails):
1422                 t['url'] = sanitize_url(t['url'])
1423                 if t.get('width') and t.get('height'):
1424                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1425                 if t.get('id') is None:
1426                     t['id'] = '%d' % i
1427
1428         if self.params.get('list_thumbnails'):
1429             self.list_thumbnails(info_dict)
1430             return
1431
1432         thumbnail = info_dict.get('thumbnail')
1433         if thumbnail:
1434             info_dict['thumbnail'] = sanitize_url(thumbnail)
1435         elif thumbnails:
1436             info_dict['thumbnail'] = thumbnails[-1]['url']
1437
1438         if 'display_id' not in info_dict and 'id' in info_dict:
1439             info_dict['display_id'] = info_dict['id']
1440
1441         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1442             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1443             # see http://bugs.python.org/issue1646728)
1444             try:
1445                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1446                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1447             except (ValueError, OverflowError, OSError):
1448                 pass
1449
1450         # Auto generate title fields corresponding to the *_number fields when missing
1451         # in order to always have clean titles. This is very common for TV series.
1452         for field in ('chapter', 'season', 'episode'):
1453             if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1454                 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1455
1456         subtitles = info_dict.get('subtitles')
1457         if subtitles:
1458             for _, subtitle in subtitles.items():
1459                 for subtitle_format in subtitle:
1460                     if subtitle_format.get('url'):
1461                         subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1462                     if subtitle_format.get('ext') is None:
1463                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1464
1465         if self.params.get('listsubtitles', False):
1466             if 'automatic_captions' in info_dict:
1467                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1468             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1469             return
1470         info_dict['requested_subtitles'] = self.process_subtitles(
1471             info_dict['id'], subtitles,
1472             info_dict.get('automatic_captions'))
1473
1474         # We now pick which formats have to be downloaded
1475         if info_dict.get('formats') is None:
1476             # There's only one format available
1477             formats = [info_dict]
1478         else:
1479             formats = info_dict['formats']
1480
1481         if not formats:
1482             raise ExtractorError('No video formats found!')
1483
1484         def is_wellformed(f):
1485             url = f.get('url')
1486             if not url:
1487                 self.report_warning(
1488                     '"url" field is missing or empty - skipping format, '
1489                     'there is an error in extractor')
1490                 return False
1491             if isinstance(url, bytes):
1492                 sanitize_string_field(f, 'url')
1493             return True
1494
1495         # Filter out malformed formats for better extraction robustness
1496         formats = list(filter(is_wellformed, formats))
1497
1498         formats_dict = {}
1499
1500         # We check that all the formats have the format and format_id fields
1501         for i, format in enumerate(formats):
1502             sanitize_string_field(format, 'format_id')
1503             sanitize_numeric_fields(format)
1504             format['url'] = sanitize_url(format['url'])
1505             if not format.get('format_id'):
1506                 format['format_id'] = compat_str(i)
1507             else:
1508                 # Sanitize format_id from characters used in format selector expression
1509                 format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
1510             format_id = format['format_id']
1511             if format_id not in formats_dict:
1512                 formats_dict[format_id] = []
1513             formats_dict[format_id].append(format)
1514
1515         # Make sure all formats have unique format_id
1516         for format_id, ambiguous_formats in formats_dict.items():
1517             if len(ambiguous_formats) > 1:
1518                 for i, format in enumerate(ambiguous_formats):
1519                     format['format_id'] = '%s-%d' % (format_id, i)
1520
1521         for i, format in enumerate(formats):
1522             if format.get('format') is None:
1523                 format['format'] = '{id} - {res}{note}'.format(
1524                     id=format['format_id'],
1525                     res=self.format_resolution(format),
1526                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1527                 )
1528             # Automatically determine file extension if missing
1529             if format.get('ext') is None:
1530                 format['ext'] = determine_ext(format['url']).lower()
1531             # Automatically determine protocol if missing (useful for format
1532             # selection purposes)
1533             if format.get('protocol') is None:
1534                 format['protocol'] = determine_protocol(format)
1535             # Add HTTP headers, so that external programs can use them from the
1536             # json output
1537             full_format_info = info_dict.copy()
1538             full_format_info.update(format)
1539             format['http_headers'] = self._calc_headers(full_format_info)
1540         # Remove private housekeeping stuff
1541         if '__x_forwarded_for_ip' in info_dict:
1542             del info_dict['__x_forwarded_for_ip']
1543
1544         # TODO Central sorting goes here
1545
1546         if formats[0] is not info_dict:
1547             # only set the 'formats' fields if the original info_dict list them
1548             # otherwise we end up with a circular reference, the first (and unique)
1549             # element in the 'formats' field in info_dict is info_dict itself,
1550             # which can't be exported to json
1551             info_dict['formats'] = formats
1552         if self.params.get('listformats'):
1553             self.list_formats(info_dict)
1554             return
1555
1556         req_format = self.params.get('format')
1557         if req_format is None:
1558             req_format = self._default_format_spec(info_dict, download=download)
1559             if self.params.get('verbose'):
1560                 self.to_stdout('[debug] Default format spec: %s' % req_format)
1561
1562         format_selector = self.build_format_selector(req_format)
1563
1564         # While in format selection we may need to have an access to the original
1565         # format set in order to calculate some metrics or do some processing.
1566         # For now we need to be able to guess whether original formats provided
1567         # by extractor are incomplete or not (i.e. whether extractor provides only
1568         # video-only or audio-only formats) for proper formats selection for
1569         # extractors with such incomplete formats (see
1570         # https://github.com/rg3/youtube-dl/pull/5556).
1571         # Since formats may be filtered during format selection and may not match
1572         # the original formats the results may be incorrect. Thus original formats
1573         # or pre-calculated metrics should be passed to format selection routines
1574         # as well.
1575         # We will pass a context object containing all necessary additional data
1576         # instead of just formats.
1577         # This fixes incorrect format selection issue (see
1578         # https://github.com/rg3/youtube-dl/issues/10083).
1579         incomplete_formats = (
1580             # All formats are video-only or
1581             all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
1582             # all formats are audio-only
1583             all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
1584
1585         ctx = {
1586             'formats': formats,
1587             'incomplete_formats': incomplete_formats,
1588         }
1589
1590         formats_to_download = list(format_selector(ctx))
1591         if not formats_to_download:
1592             raise ExtractorError('requested format not available',
1593                                  expected=True)
1594
1595         if download:
1596             if len(formats_to_download) > 1:
1597                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1598             for format in formats_to_download:
1599                 new_info = dict(info_dict)
1600                 new_info.update(format)
1601                 self.process_info(new_info)
1602         # We update the info dict with the best quality format (backwards compatibility)
1603         info_dict.update(formats_to_download[-1])
1604         return info_dict
1605
1606     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1607         """Select the requested subtitles and their format"""
1608         available_subs = {}
1609         if normal_subtitles and self.params.get('writesubtitles'):
1610             available_subs.update(normal_subtitles)
1611         if automatic_captions and self.params.get('writeautomaticsub'):
1612             for lang, cap_info in automatic_captions.items():
1613                 if lang not in available_subs:
1614                     available_subs[lang] = cap_info
1615
1616         if (not self.params.get('writesubtitles') and not
1617                 self.params.get('writeautomaticsub') or not
1618                 available_subs):
1619             return None
1620
1621         if self.params.get('allsubtitles', False):
1622             requested_langs = available_subs.keys()
1623         else:
1624             if self.params.get('subtitleslangs', False):
1625                 requested_langs = self.params.get('subtitleslangs')
1626             elif 'en' in available_subs:
1627                 requested_langs = ['en']
1628             else:
1629                 requested_langs = [list(available_subs.keys())[0]]
1630
1631         formats_query = self.params.get('subtitlesformat', 'best')
1632         formats_preference = formats_query.split('/') if formats_query else []
1633         subs = {}
1634         for lang in requested_langs:
1635             formats = available_subs.get(lang)
1636             if formats is None:
1637                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1638                 continue
1639             for ext in formats_preference:
1640                 if ext == 'best':
1641                     f = formats[-1]
1642                     break
1643                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1644                 if matches:
1645                     f = matches[-1]
1646                     break
1647             else:
1648                 f = formats[-1]
1649                 self.report_warning(
1650                     'No subtitle format found matching "%s" for language %s, '
1651                     'using %s' % (formats_query, lang, f['ext']))
1652             subs[lang] = f
1653         return subs
1654
1655     def process_info(self, info_dict):
1656         """Process a single resolved IE result."""
1657
1658         assert info_dict.get('_type', 'video') == 'video'
1659
1660         max_downloads = self.params.get('max_downloads')
1661         if max_downloads is not None:
1662             if self._num_downloads >= int(max_downloads):
1663                 raise MaxDownloadsReached()
1664
1665         info_dict['fulltitle'] = info_dict['title']
1666         if len(info_dict['title']) > 200:
1667             info_dict['title'] = info_dict['title'][:197] + '...'
1668
1669         if 'format' not in info_dict:
1670             info_dict['format'] = info_dict['ext']
1671
1672         reason = self._match_entry(info_dict, incomplete=False)
1673         if reason is not None:
1674             self.to_screen('[download] ' + reason)
1675             return
1676
1677         self._num_downloads += 1
1678
1679         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1680
1681         # Forced printings
1682         if self.params.get('forcetitle', False):
1683             self.to_stdout(info_dict['fulltitle'])
1684         if self.params.get('forceid', False):
1685             self.to_stdout(info_dict['id'])
1686         if self.params.get('forceurl', False):
1687             if info_dict.get('requested_formats') is not None:
1688                 for f in info_dict['requested_formats']:
1689                     self.to_stdout(f['url'] + f.get('play_path', ''))
1690             else:
1691                 # For RTMP URLs, also include the playpath
1692                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1693         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1694             self.to_stdout(info_dict['thumbnail'])
1695         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1696             self.to_stdout(info_dict['description'])
1697         if self.params.get('forcefilename', False) and filename is not None:
1698             self.to_stdout(filename)
1699         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1700             self.to_stdout(formatSeconds(info_dict['duration']))
1701         if self.params.get('forceformat', False):
1702             self.to_stdout(info_dict['format'])
1703         if self.params.get('forcejson', False):
1704             self.to_stdout(json.dumps(info_dict))
1705
1706         # Do nothing else if in simulate mode
1707         if self.params.get('simulate', False):
1708             return
1709
1710         if filename is None:
1711             return
1712
1713         try:
1714             dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1715             if dn and not os.path.exists(dn):
1716                 os.makedirs(dn)
1717         except (OSError, IOError) as err:
1718             self.report_error('unable to create directory ' + error_to_compat_str(err))
1719             return
1720
1721         if self.params.get('writedescription', False):
1722             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1723             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1724                 self.to_screen('[info] Video description is already present')
1725             elif info_dict.get('description') is None:
1726                 self.report_warning('There\'s no description to write.')
1727             else:
1728                 try:
1729                     self.to_screen('[info] Writing video description to: ' + descfn)
1730                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1731                         descfile.write(info_dict['description'])
1732                 except (OSError, IOError):
1733                     self.report_error('Cannot write description file ' + descfn)
1734                     return
1735
1736         if self.params.get('writeannotations', False):
1737             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1738             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1739                 self.to_screen('[info] Video annotations are already present')
1740             else:
1741                 try:
1742                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1743                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1744                         annofile.write(info_dict['annotations'])
1745                 except (KeyError, TypeError):
1746                     self.report_warning('There are no annotations to write.')
1747                 except (OSError, IOError):
1748                     self.report_error('Cannot write annotations file: ' + annofn)
1749                     return
1750
1751         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1752                                        self.params.get('writeautomaticsub')])
1753
1754         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1755             # subtitles download errors are already managed as troubles in relevant IE
1756             # that way it will silently go on when used with unsupporting IE
1757             subtitles = info_dict['requested_subtitles']
1758             ie = self.get_info_extractor(info_dict['extractor_key'])
1759             for sub_lang, sub_info in subtitles.items():
1760                 sub_format = sub_info['ext']
1761                 if sub_info.get('data') is not None:
1762                     sub_data = sub_info['data']
1763                 else:
1764                     try:
1765                         sub_data = ie._download_webpage(
1766                             sub_info['url'], info_dict['id'], note=False)
1767                     except ExtractorError as err:
1768                         self.report_warning('Unable to download subtitle for "%s": %s' %
1769                                             (sub_lang, error_to_compat_str(err.cause)))
1770                         continue
1771                 try:
1772                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1773                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1774                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1775                     else:
1776                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1777                         # Use newline='' to prevent conversion of newline characters
1778                         # See https://github.com/rg3/youtube-dl/issues/10268
1779                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
1780                             subfile.write(sub_data)
1781                 except (OSError, IOError):
1782                     self.report_error('Cannot write subtitles file ' + sub_filename)
1783                     return
1784
1785         if self.params.get('writeinfojson', False):
1786             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1787             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1788                 self.to_screen('[info] Video description metadata is already present')
1789             else:
1790                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1791                 try:
1792                     write_json_file(self.filter_requested_info(info_dict), infofn)
1793                 except (OSError, IOError):
1794                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1795                     return
1796
1797         self._write_thumbnails(info_dict, filename)
1798
1799         if not self.params.get('skip_download', False):
1800             try:
1801                 def dl(name, info):
1802                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1803                     for ph in self._progress_hooks:
1804                         fd.add_progress_hook(ph)
1805                     if self.params.get('verbose'):
1806                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1807                     return fd.download(name, info)
1808
1809                 if info_dict.get('requested_formats') is not None:
1810                     downloaded = []
1811                     success = True
1812                     merger = FFmpegMergerPP(self)
1813                     if not merger.available:
1814                         postprocessors = []
1815                         self.report_warning('You have requested multiple '
1816                                             'formats but ffmpeg or avconv are not installed.'
1817                                             ' The formats won\'t be merged.')
1818                     else:
1819                         postprocessors = [merger]
1820
1821                     def compatible_formats(formats):
1822                         video, audio = formats
1823                         # Check extension
1824                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1825                         if video_ext and audio_ext:
1826                             COMPATIBLE_EXTS = (
1827                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'),
1828                                 ('webm')
1829                             )
1830                             for exts in COMPATIBLE_EXTS:
1831                                 if video_ext in exts and audio_ext in exts:
1832                                     return True
1833                         # TODO: Check acodec/vcodec
1834                         return False
1835
1836                     filename_real_ext = os.path.splitext(filename)[1][1:]
1837                     filename_wo_ext = (
1838                         os.path.splitext(filename)[0]
1839                         if filename_real_ext == info_dict['ext']
1840                         else filename)
1841                     requested_formats = info_dict['requested_formats']
1842                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1843                         info_dict['ext'] = 'mkv'
1844                         self.report_warning(
1845                             'Requested formats are incompatible for merge and will be merged into mkv.')
1846                     # Ensure filename always has a correct extension for successful merge
1847                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1848                     if os.path.exists(encodeFilename(filename)):
1849                         self.to_screen(
1850                             '[download] %s has already been downloaded and '
1851                             'merged' % filename)
1852                     else:
1853                         for f in requested_formats:
1854                             new_info = dict(info_dict)
1855                             new_info.update(f)
1856                             fname = self.prepare_filename(new_info)
1857                             fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1858                             downloaded.append(fname)
1859                             partial_success = dl(fname, new_info)
1860                             success = success and partial_success
1861                         info_dict['__postprocessors'] = postprocessors
1862                         info_dict['__files_to_merge'] = downloaded
1863                 else:
1864                     # Just a single file
1865                     success = dl(filename, info_dict)
1866             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1867                 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1868                 return
1869             except (OSError, IOError) as err:
1870                 raise UnavailableVideoError(err)
1871             except (ContentTooShortError, ) as err:
1872                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1873                 return
1874
1875             if success and filename != '-':
1876                 # Fixup content
1877                 fixup_policy = self.params.get('fixup')
1878                 if fixup_policy is None:
1879                     fixup_policy = 'detect_or_warn'
1880
1881                 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1882
1883                 stretched_ratio = info_dict.get('stretched_ratio')
1884                 if stretched_ratio is not None and stretched_ratio != 1:
1885                     if fixup_policy == 'warn':
1886                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1887                             info_dict['id'], stretched_ratio))
1888                     elif fixup_policy == 'detect_or_warn':
1889                         stretched_pp = FFmpegFixupStretchedPP(self)
1890                         if stretched_pp.available:
1891                             info_dict.setdefault('__postprocessors', [])
1892                             info_dict['__postprocessors'].append(stretched_pp)
1893                         else:
1894                             self.report_warning(
1895                                 '%s: Non-uniform pixel ratio (%s). %s'
1896                                 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1897                     else:
1898                         assert fixup_policy in ('ignore', 'never')
1899
1900                 if (info_dict.get('requested_formats') is None and
1901                         info_dict.get('container') == 'm4a_dash'):
1902                     if fixup_policy == 'warn':
1903                         self.report_warning(
1904                             '%s: writing DASH m4a. '
1905                             'Only some players support this container.'
1906                             % info_dict['id'])
1907                     elif fixup_policy == 'detect_or_warn':
1908                         fixup_pp = FFmpegFixupM4aPP(self)
1909                         if fixup_pp.available:
1910                             info_dict.setdefault('__postprocessors', [])
1911                             info_dict['__postprocessors'].append(fixup_pp)
1912                         else:
1913                             self.report_warning(
1914                                 '%s: writing DASH m4a. '
1915                                 'Only some players support this container. %s'
1916                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1917                     else:
1918                         assert fixup_policy in ('ignore', 'never')
1919
1920                 if (info_dict.get('protocol') == 'm3u8_native' or
1921                         info_dict.get('protocol') == 'm3u8' and
1922                         self.params.get('hls_prefer_native')):
1923                     if fixup_policy == 'warn':
1924                         self.report_warning('%s: malformed AAC bitstream detected.' % (
1925                             info_dict['id']))
1926                     elif fixup_policy == 'detect_or_warn':
1927                         fixup_pp = FFmpegFixupM3u8PP(self)
1928                         if fixup_pp.available:
1929                             info_dict.setdefault('__postprocessors', [])
1930                             info_dict['__postprocessors'].append(fixup_pp)
1931                         else:
1932                             self.report_warning(
1933                                 '%s: malformed AAC bitstream detected. %s'
1934                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1935                     else:
1936                         assert fixup_policy in ('ignore', 'never')
1937
1938                 try:
1939                     self.post_process(filename, info_dict)
1940                 except (PostProcessingError) as err:
1941                     self.report_error('postprocessing: %s' % str(err))
1942                     return
1943                 self.record_download_archive(info_dict)
1944
1945     def download(self, url_list):
1946         """Download a given list of URLs."""
1947         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1948         if (len(url_list) > 1 and
1949                 outtmpl != '-' and
1950                 '%' not in outtmpl and
1951                 self.params.get('max_downloads') != 1):
1952             raise SameFileError(outtmpl)
1953
1954         for url in url_list:
1955             try:
1956                 # It also downloads the videos
1957                 res = self.extract_info(
1958                     url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1959             except UnavailableVideoError:
1960                 self.report_error('unable to download video')
1961             except MaxDownloadsReached:
1962                 self.to_screen('[info] Maximum number of downloaded files reached.')
1963                 raise
1964             else:
1965                 if self.params.get('dump_single_json', False):
1966                     self.to_stdout(json.dumps(res))
1967
1968         return self._download_retcode
1969
1970     def download_with_info_file(self, info_filename):
1971         with contextlib.closing(fileinput.FileInput(
1972                 [info_filename], mode='r',
1973                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1974             # FileInput doesn't have a read method, we can't call json.load
1975             info = self.filter_requested_info(json.loads('\n'.join(f)))
1976         try:
1977             self.process_ie_result(info, download=True)
1978         except DownloadError:
1979             webpage_url = info.get('webpage_url')
1980             if webpage_url is not None:
1981                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1982                 return self.download([webpage_url])
1983             else:
1984                 raise
1985         return self._download_retcode
1986
1987     @staticmethod
1988     def filter_requested_info(info_dict):
1989         return dict(
1990             (k, v) for k, v in info_dict.items()
1991             if k not in ['requested_formats', 'requested_subtitles'])
1992
1993     def post_process(self, filename, ie_info):
1994         """Run all the postprocessors on the given file."""
1995         info = dict(ie_info)
1996         info['filepath'] = filename
1997         pps_chain = []
1998         if ie_info.get('__postprocessors') is not None:
1999             pps_chain.extend(ie_info['__postprocessors'])
2000         pps_chain.extend(self._pps)
2001         for pp in pps_chain:
2002             files_to_delete = []
2003             try:
2004                 files_to_delete, info = pp.run(info)
2005             except PostProcessingError as e:
2006                 self.report_error(e.msg)
2007             if files_to_delete and not self.params.get('keepvideo', False):
2008                 for old_filename in files_to_delete:
2009                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
2010                     try:
2011                         os.remove(encodeFilename(old_filename))
2012                     except (IOError, OSError):
2013                         self.report_warning('Unable to remove downloaded original file')
2014
2015     def _make_archive_id(self, info_dict):
2016         # Future-proof against any change in case
2017         # and backwards compatibility with prior versions
2018         extractor = info_dict.get('extractor_key')
2019         if extractor is None:
2020             if 'id' in info_dict:
2021                 extractor = info_dict.get('ie_key')  # key in a playlist
2022         if extractor is None:
2023             return None  # Incomplete video information
2024         return extractor.lower() + ' ' + info_dict['id']
2025
2026     def in_download_archive(self, info_dict):
2027         fn = self.params.get('download_archive')
2028         if fn is None:
2029             return False
2030
2031         vid_id = self._make_archive_id(info_dict)
2032         if vid_id is None:
2033             return False  # Incomplete video information
2034
2035         try:
2036             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
2037                 for line in archive_file:
2038                     if line.strip() == vid_id:
2039                         return True
2040         except IOError as ioe:
2041             if ioe.errno != errno.ENOENT:
2042                 raise
2043         return False
2044
2045     def record_download_archive(self, info_dict):
2046         fn = self.params.get('download_archive')
2047         if fn is None:
2048             return
2049         vid_id = self._make_archive_id(info_dict)
2050         assert vid_id
2051         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
2052             archive_file.write(vid_id + '\n')
2053
2054     @staticmethod
2055     def format_resolution(format, default='unknown'):
2056         if format.get('vcodec') == 'none':
2057             return 'audio only'
2058         if format.get('resolution') is not None:
2059             return format['resolution']
2060         if format.get('height') is not None:
2061             if format.get('width') is not None:
2062                 res = '%sx%s' % (format['width'], format['height'])
2063             else:
2064                 res = '%sp' % format['height']
2065         elif format.get('width') is not None:
2066             res = '%dx?' % format['width']
2067         else:
2068             res = default
2069         return res
2070
2071     def _format_note(self, fdict):
2072         res = ''
2073         if fdict.get('ext') in ['f4f', 'f4m']:
2074             res += '(unsupported) '
2075         if fdict.get('language'):
2076             if res:
2077                 res += ' '
2078             res += '[%s] ' % fdict['language']
2079         if fdict.get('format_note') is not None:
2080             res += fdict['format_note'] + ' '
2081         if fdict.get('tbr') is not None:
2082             res += '%4dk ' % fdict['tbr']
2083         if fdict.get('container') is not None:
2084             if res:
2085                 res += ', '
2086             res += '%s container' % fdict['container']
2087         if (fdict.get('vcodec') is not None and
2088                 fdict.get('vcodec') != 'none'):
2089             if res:
2090                 res += ', '
2091             res += fdict['vcodec']
2092             if fdict.get('vbr') is not None:
2093                 res += '@'
2094         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
2095             res += 'video@'
2096         if fdict.get('vbr') is not None:
2097             res += '%4dk' % fdict['vbr']
2098         if fdict.get('fps') is not None:
2099             if res:
2100                 res += ', '
2101             res += '%sfps' % fdict['fps']
2102         if fdict.get('acodec') is not None:
2103             if res:
2104                 res += ', '
2105             if fdict['acodec'] == 'none':
2106                 res += 'video only'
2107             else:
2108                 res += '%-5s' % fdict['acodec']
2109         elif fdict.get('abr') is not None:
2110             if res:
2111                 res += ', '
2112             res += 'audio'
2113         if fdict.get('abr') is not None:
2114             res += '@%3dk' % fdict['abr']
2115         if fdict.get('asr') is not None:
2116             res += ' (%5dHz)' % fdict['asr']
2117         if fdict.get('filesize') is not None:
2118             if res:
2119                 res += ', '
2120             res += format_bytes(fdict['filesize'])
2121         elif fdict.get('filesize_approx') is not None:
2122             if res:
2123                 res += ', '
2124             res += '~' + format_bytes(fdict['filesize_approx'])
2125         return res
2126
2127     def list_formats(self, info_dict):
2128         formats = info_dict.get('formats', [info_dict])
2129         table = [
2130             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
2131             for f in formats
2132             if f.get('preference') is None or f['preference'] >= -1000]
2133         if len(formats) > 1:
2134             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
2135
2136         header_line = ['format code', 'extension', 'resolution', 'note']
2137         self.to_screen(
2138             '[info] Available formats for %s:\n%s' %
2139             (info_dict['id'], render_table(header_line, table)))
2140
2141     def list_thumbnails(self, info_dict):
2142         thumbnails = info_dict.get('thumbnails')
2143         if not thumbnails:
2144             self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
2145             return
2146
2147         self.to_screen(
2148             '[info] Thumbnails for %s:' % info_dict['id'])
2149         self.to_screen(render_table(
2150             ['ID', 'width', 'height', 'URL'],
2151             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
2152
2153     def list_subtitles(self, video_id, subtitles, name='subtitles'):
2154         if not subtitles:
2155             self.to_screen('%s has no %s' % (video_id, name))
2156             return
2157         self.to_screen(
2158             'Available %s for %s:' % (name, video_id))
2159         self.to_screen(render_table(
2160             ['Language', 'formats'],
2161             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
2162                 for lang, formats in subtitles.items()]))
2163
2164     def urlopen(self, req):
2165         """ Start an HTTP download """
2166         if isinstance(req, compat_basestring):
2167             req = sanitized_Request(req)
2168         return self._opener.open(req, timeout=self._socket_timeout)
2169
2170     def print_debug_header(self):
2171         if not self.params.get('verbose'):
2172             return
2173
2174         if type('') is not compat_str:
2175             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
2176             self.report_warning(
2177                 'Your Python is broken! Update to a newer and supported version')
2178
2179         stdout_encoding = getattr(
2180             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
2181         encoding_str = (
2182             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2183                 locale.getpreferredencoding(),
2184                 sys.getfilesystemencoding(),
2185                 stdout_encoding,
2186                 self.get_encoding()))
2187         write_string(encoding_str, encoding=None)
2188
2189         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
2190         if _LAZY_LOADER:
2191             self._write_string('[debug] Lazy loading extractors enabled' + '\n')
2192         try:
2193             sp = subprocess.Popen(
2194                 ['git', 'rev-parse', '--short', 'HEAD'],
2195                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2196                 cwd=os.path.dirname(os.path.abspath(__file__)))
2197             out, err = sp.communicate()
2198             out = out.decode().strip()
2199             if re.match('[0-9a-f]+', out):
2200                 self._write_string('[debug] Git HEAD: ' + out + '\n')
2201         except Exception:
2202             try:
2203                 sys.exc_clear()
2204             except Exception:
2205                 pass
2206         self._write_string('[debug] Python version %s - %s\n' % (
2207             platform.python_version(), platform_name()))
2208
2209         exe_versions = FFmpegPostProcessor.get_versions(self)
2210         exe_versions['rtmpdump'] = rtmpdump_version()
2211         exe_str = ', '.join(
2212             '%s %s' % (exe, v)
2213             for exe, v in sorted(exe_versions.items())
2214             if v
2215         )
2216         if not exe_str:
2217             exe_str = 'none'
2218         self._write_string('[debug] exe versions: %s\n' % exe_str)
2219
2220         proxy_map = {}
2221         for handler in self._opener.handlers:
2222             if hasattr(handler, 'proxies'):
2223                 proxy_map.update(handler.proxies)
2224         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2225
2226         if self.params.get('call_home', False):
2227             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2228             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2229             latest_version = self.urlopen(
2230                 'https://yt-dl.org/latest/version').read().decode('utf-8')
2231             if version_tuple(latest_version) > version_tuple(__version__):
2232                 self.report_warning(
2233                     'You are using an outdated version (newest version: %s)! '
2234                     'See https://yt-dl.org/update if you need help updating.' %
2235                     latest_version)
2236
2237     def _setup_opener(self):
2238         timeout_val = self.params.get('socket_timeout')
2239         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2240
2241         opts_cookiefile = self.params.get('cookiefile')
2242         opts_proxy = self.params.get('proxy')
2243
2244         if opts_cookiefile is None:
2245             self.cookiejar = compat_cookiejar.CookieJar()
2246         else:
2247             opts_cookiefile = expand_path(opts_cookiefile)
2248             self.cookiejar = compat_cookiejar.MozillaCookieJar(
2249                 opts_cookiefile)
2250             if os.access(opts_cookiefile, os.R_OK):
2251                 self.cookiejar.load()
2252
2253         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2254         if opts_proxy is not None:
2255             if opts_proxy == '':
2256                 proxies = {}
2257             else:
2258                 proxies = {'http': opts_proxy, 'https': opts_proxy}
2259         else:
2260             proxies = compat_urllib_request.getproxies()
2261             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2262             if 'http' in proxies and 'https' not in proxies:
2263                 proxies['https'] = proxies['http']
2264         proxy_handler = PerRequestProxyHandler(proxies)
2265
2266         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2267         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2268         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2269         data_handler = compat_urllib_request_DataHandler()
2270
2271         # When passing our own FileHandler instance, build_opener won't add the
2272         # default FileHandler and allows us to disable the file protocol, which
2273         # can be used for malicious purposes (see
2274         # https://github.com/rg3/youtube-dl/issues/8227)
2275         file_handler = compat_urllib_request.FileHandler()
2276
2277         def file_open(*args, **kwargs):
2278             raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2279         file_handler.file_open = file_open
2280
2281         opener = compat_urllib_request.build_opener(
2282             proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2283
2284         # Delete the default user-agent header, which would otherwise apply in
2285         # cases where our custom HTTP handler doesn't come into play
2286         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2287         opener.addheaders = []
2288         self._opener = opener
2289
2290     def encode(self, s):
2291         if isinstance(s, bytes):
2292             return s  # Already encoded
2293
2294         try:
2295             return s.encode(self.get_encoding())
2296         except UnicodeEncodeError as err:
2297             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2298             raise
2299
2300     def get_encoding(self):
2301         encoding = self.params.get('encoding')
2302         if encoding is None:
2303             encoding = preferredencoding()
2304         return encoding
2305
2306     def _write_thumbnails(self, info_dict, filename):
2307         if self.params.get('writethumbnail', False):
2308             thumbnails = info_dict.get('thumbnails')
2309             if thumbnails:
2310                 thumbnails = [thumbnails[-1]]
2311         elif self.params.get('write_all_thumbnails', False):
2312             thumbnails = info_dict.get('thumbnails')
2313         else:
2314             return
2315
2316         if not thumbnails:
2317             # No thumbnails present, so return immediately
2318             return
2319
2320         for t in thumbnails:
2321             thumb_ext = determine_ext(t['url'], 'jpg')
2322             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2323             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2324             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2325
2326             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2327                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2328                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2329             else:
2330                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2331                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2332                 try:
2333                     uf = self.urlopen(t['url'])
2334                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2335                         shutil.copyfileobj(uf, thumbf)
2336                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2337                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2338                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2339                     self.report_warning('Unable to download thumbnail "%s": %s' %
2340                                         (t['url'], error_to_compat_str(err)))