_ Git - youtube-dl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # coding: utf-8
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import contextlib
   8 import copy
   9 import datetime
  10 import errno
  11 import fileinput
  12 import io
  13 import itertools
  14 import json
  15 import locale
  16 import operator
  17 import os
  18 import platform
  19 import re
  20 import shutil
  21 import subprocess
  22 import socket
  23 import sys
  24 import time
  25 import tokenize
  26 import traceback
  27 import random
  28
  29 from string import ascii_letters
  30
  31 from .compat import (
  32     compat_basestring,
  33     compat_cookiejar,
  34     compat_get_terminal_size,
  35     compat_http_client,
  36     compat_kwargs,
  37     compat_numeric_types,
  38     compat_os_name,
  39     compat_str,
  40     compat_tokenize_tokenize,
  41     compat_urllib_error,
  42     compat_urllib_request,
  43     compat_urllib_request_DataHandler,
  44 )
  45 from .utils import (
  46     age_restricted,
  47     args_to_str,
  48     ContentTooShortError,
  49     date_from_str,
  50     DateRange,
  51     DEFAULT_OUTTMPL,
  52     determine_ext,
  53     determine_protocol,
  54     DownloadError,
  55     encode_compat_str,
  56     encodeFilename,
  57     error_to_compat_str,
  58     expand_path,
  59     ExtractorError,
  60     format_bytes,
  61     formatSeconds,
  62     GeoRestrictedError,
  63     int_or_none,
  64     ISO3166Utils,
  65     locked_file,
  66     make_HTTPS_handler,
  67     MaxDownloadsReached,
  68     PagedList,
  69     parse_filesize,
  70     PerRequestProxyHandler,
  71     platform_name,
  72     PostProcessingError,
  73     preferredencoding,
  74     prepend_extension,
  75     register_socks_protocols,
  76     render_table,
  77     replace_extension,
  78     SameFileError,
  79     sanitize_filename,
  80     sanitize_path,
  81     sanitize_url,
  82     sanitized_Request,
  83     std_headers,
  84     subtitles_filename,
  85     UnavailableVideoError,
  86     url_basename,
  87     version_tuple,
  88     write_json_file,
  89     write_string,
  90     YoutubeDLCookieProcessor,
  91     YoutubeDLHandler,
  92 )
  93 from .cache import Cache
  94 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
  95 from .downloader import get_suitable_downloader
  96 from .downloader.rtmp import rtmpdump_version
  97 from .postprocessor import (
  98     FFmpegFixupM3u8PP,
  99     FFmpegFixupM4aPP,
 100     FFmpegFixupStretchedPP,
 101     FFmpegMergerPP,
 102     FFmpegPostProcessor,
 103     get_postprocessor,
 104 )
 105 from .version import __version__
 106
 107 if compat_os_name == 'nt':
 108     import ctypes
 109
 110
 111 class YoutubeDL(object):
 112     """YoutubeDL class.
 113
 114     YoutubeDL objects are the ones responsible of downloading the
 115     actual video file and writing it to disk if the user has requested
 116     it, among some other tasks. In most cases there should be one per
 117     program. As, given a video URL, the downloader doesn't know how to
 118     extract all the needed information, task that InfoExtractors do, it
 119     has to pass the URL to one of them.
 120
 121     For this, YoutubeDL objects have a method that allows
 122     InfoExtractors to be registered in a given order. When it is passed
 123     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 124     finds that reports being able to handle it. The InfoExtractor extracts
 125     all the information about the video or videos the URL refers to, and
 126     YoutubeDL process the extracted information, possibly using a File
 127     Downloader to download the video.
 128
 129     YoutubeDL objects accept a lot of parameters. In order not to saturate
 130     the object constructor with arguments, it receives a dictionary of
 131     options instead. These options are available through the params
 132     attribute for the InfoExtractors to use. The YoutubeDL also
 133     registers itself as the downloader in charge for the InfoExtractors
 134     that are added to it, so this is a "mutual registration".
 135
 136     Available options:
 137
 138     username:          Username for authentication purposes.
 139     password:          Password for authentication purposes.
 140     videopassword:     Password for accessing a video.
 141     ap_mso:            Adobe Pass multiple-system operator identifier.
 142     ap_username:       Multiple-system operator account username.
 143     ap_password:       Multiple-system operator account password.
 144     usenetrc:          Use netrc for authentication instead.
 145     verbose:           Print additional info to stdout.
 146     quiet:             Do not print messages to stdout.
 147     no_warnings:       Do not print out anything for warnings.
 148     forceurl:          Force printing final URL.
 149     forcetitle:        Force printing title.
 150     forceid:           Force printing ID.
 151     forcethumbnail:    Force printing thumbnail URL.
 152     forcedescription:  Force printing description.
 153     forcefilename:     Force printing final filename.
 154     forceduration:     Force printing duration.
 155     forcejson:         Force printing info_dict as JSON.
 156     dump_single_json:  Force printing the info_dict of the whole playlist
 157                        (or video) as a single JSON line.
 158     simulate:          Do not download the video files.
 159     format:            Video format code. See options.py for more information.
 160     outtmpl:           Template for output names.
 161     restrictfilenames: Do not allow "&" and spaces in file names
 162     ignoreerrors:      Do not stop on download errors.
 163     force_generic_extractor: Force downloader to use the generic extractor
 164     nooverwrites:      Prevent overwriting files.
 165     playliststart:     Playlist item to start at.
 166     playlistend:       Playlist item to end at.
 167     playlist_items:    Specific indices of playlist to download.
 168     playlistreverse:   Download playlist items in reverse order.
 169     playlistrandom:    Download playlist items in random order.
 170     matchtitle:        Download only matching titles.
 171     rejecttitle:       Reject downloads for matching titles.
 172     logger:            Log messages to a logging.Logger instance.
 173     logtostderr:       Log messages to stderr instead of stdout.
 174     writedescription:  Write the video description to a .description file
 175     writeinfojson:     Write the video description to a .info.json file
 176     writeannotations:  Write the video annotations to a .annotations.xml file
 177     writethumbnail:    Write the thumbnail image to a file
 178     write_all_thumbnails:  Write all thumbnail formats to files
 179     writesubtitles:    Write the video subtitles to a file
 180     writeautomaticsub: Write the automatically generated subtitles to a file
 181     allsubtitles:      Downloads all the subtitles of the video
 182                        (requires writesubtitles or writeautomaticsub)
 183     listsubtitles:     Lists all available subtitles for the video
 184     subtitlesformat:   The format code for subtitles
 185     subtitleslangs:    List of languages of the subtitles to download
 186     keepvideo:         Keep the video file after post-processing
 187     daterange:         A DateRange object, download only if the upload_date is in the range.
 188     skip_download:     Skip the actual download of the video file
 189     cachedir:          Location of the cache files in the filesystem.
 190                        False to disable filesystem cache.
 191     noplaylist:        Download single video instead of a playlist if in doubt.
 192     age_limit:         An integer representing the user's age in years.
 193                        Unsuitable videos for the given age are skipped.
 194     min_views:         An integer representing the minimum view count the video
 195                        must have in order to not be skipped.
 196                        Videos without view count information are always
 197                        downloaded. None for no limit.
 198     max_views:         An integer representing the maximum view count.
 199                        Videos that are more popular than that are not
 200                        downloaded.
 201                        Videos without view count information are always
 202                        downloaded. None for no limit.
 203     download_archive:  File name of a file where all downloads are recorded.
 204                        Videos already present in the file are not downloaded
 205                        again.
 206     cookiefile:        File name where cookies should be read from and dumped to.
 207     nocheckcertificate:Do not verify SSL certificates
 208     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 209                        At the moment, this is only supported by YouTube.
 210     proxy:             URL of the proxy server to use
 211     geo_verification_proxy:  URL of the proxy to use for IP address verification
 212                        on geo-restricted sites. (Experimental)
 213     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 214     bidi_workaround:   Work around buggy terminals without bidirectional text
 215                        support, using fridibi
 216     debug_printtraffic:Print out sent and received HTTP traffic
 217     include_ads:       Download ads as well
 218     default_search:    Prepend this string if an input url is not valid.
 219                        'auto' for elaborate guessing
 220     encoding:          Use this encoding instead of the system-specified.
 221     extract_flat:      Do not resolve URLs, return the immediate result.
 222                        Pass in 'in_playlist' to only show this behavior for
 223                        playlist items.
 224     postprocessors:    A list of dictionaries, each with an entry
 225                        * key:  The name of the postprocessor. See
 226                                youtube_dl/postprocessor/__init__.py for a list.
 227                        as well as any further keyword arguments for the
 228                        postprocessor.
 229     progress_hooks:    A list of functions that get called on download
 230                        progress, with a dictionary with the entries
 231                        * status: One of "downloading", "error", or "finished".
 232                                  Check this first and ignore unknown values.
 233
 234                        If status is one of "downloading", or "finished", the
 235                        following properties may also be present:
 236                        * filename: The final filename (always present)
 237                        * tmpfilename: The filename we're currently writing to
 238                        * downloaded_bytes: Bytes on disk
 239                        * total_bytes: Size of the whole file, None if unknown
 240                        * total_bytes_estimate: Guess of the eventual file size,
 241                                                None if unavailable.
 242                        * elapsed: The number of seconds since download started.
 243                        * eta: The estimated time in seconds, None if unknown
 244                        * speed: The download speed in bytes/second, None if
 245                                 unknown
 246                        * fragment_index: The counter of the currently
 247                                          downloaded video fragment.
 248                        * fragment_count: The number of fragments (= individual
 249                                          files that will be merged)
 250
 251                        Progress hooks are guaranteed to be called at least once
 252                        (with status "finished") if the download is successful.
 253     merge_output_format: Extension to use when merging formats.
 254     fixup:             Automatically correct known faults of the file.
 255                        One of:
 256                        - "never": do nothing
 257                        - "warn": only emit a warning
 258                        - "detect_or_warn": check whether we can do anything
 259                                            about it, warn otherwise (default)
 260     source_address:    (Experimental) Client-side IP address to bind to.
 261     call_home:         Boolean, true iff we are allowed to contact the
 262                        youtube-dl servers for debugging.
 263     sleep_interval:    Number of seconds to sleep before each download when
 264                        used alone or a lower bound of a range for randomized
 265                        sleep before each download (minimum possible number
 266                        of seconds to sleep) when used along with
 267                        max_sleep_interval.
 268     max_sleep_interval:Upper bound of a range for randomized sleep before each
 269                        download (maximum possible number of seconds to sleep).
 270                        Must only be used along with sleep_interval.
 271                        Actual sleep time will be a random float from range
 272                        [sleep_interval; max_sleep_interval].
 273     listformats:       Print an overview of available video formats and exit.
 274     list_thumbnails:   Print a table of all thumbnails and exit.
 275     match_filter:      A function that gets called with the info_dict of
 276                        every video.
 277                        If it returns a message, the video is ignored.
 278                        If it returns None, the video is downloaded.
 279                        match_filter_func in utils.py is one example for this.
 280     no_color:          Do not emit color codes in output.
 281     geo_bypass:        Bypass geographic restriction via faking X-Forwarded-For
 282                        HTTP header (experimental)
 283     geo_bypass_country:
 284                        Two-letter ISO 3166-2 country code that will be used for
 285                        explicit geographic restriction bypassing via faking
 286                        X-Forwarded-For HTTP header (experimental)
 287
 288     The following options determine which downloader is picked:
 289     external_downloader: Executable of the external downloader to call.
 290                        None or unset for standard (built-in) downloader.
 291     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
 292                        if True, otherwise use ffmpeg/avconv if False, otherwise
 293                        use downloader suggested by extractor if None.
 294
 295     The following parameters are not used by YoutubeDL itself, they are used by
 296     the downloader (see youtube_dl/downloader/common.py):
 297     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 298     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 299     xattr_set_filesize, external_downloader_args, hls_use_mpegts.
 300
 301     The following options are used by the post processors:
 302     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 303                        otherwise prefer avconv.
 304     postprocessor_args: A list of additional command-line arguments for the
 305                         postprocessor.
 306     """
 307
 308     _NUMERIC_FIELDS = set((
 309         'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
 310         'timestamp', 'upload_year', 'upload_month', 'upload_day',
 311         'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
 312         'average_rating', 'comment_count', 'age_limit',
 313         'start_time', 'end_time',
 314         'chapter_number', 'season_number', 'episode_number',
 315         'track_number', 'disc_number', 'release_year',
 316         'playlist_index',
 317     ))
 318
 319     params = None
 320     _ies = []
 321     _pps = []
 322     _download_retcode = None
 323     _num_downloads = None
 324     _screen_file = None
 325
 326     def __init__(self, params=None, auto_init=True):
 327         """Create a FileDownloader object with the given options."""
 328         if params is None:
 329             params = {}
 330         self._ies = []
 331         self._ies_instances = {}
 332         self._pps = []
 333         self._progress_hooks = []
 334         self._download_retcode = 0
 335         self._num_downloads = 0
 336         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 337         self._err_file = sys.stderr
 338         self.params = {
 339             # Default parameters
 340             'nocheckcertificate': False,
 341         }
 342         self.params.update(params)
 343         self.cache = Cache(self)
 344
 345         def check_deprecated(param, option, suggestion):
 346             if self.params.get(param) is not None:
 347                 self.report_warning(
 348                     '%s is deprecated. Use %s instead.' % (option, suggestion))
 349                 return True
 350             return False
 351
 352         if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
 353             if self.params.get('geo_verification_proxy') is None:
 354                 self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
 355
 356         check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
 357         check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
 358         check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
 359
 360         if params.get('bidi_workaround', False):
 361             try:
 362                 import pty
 363                 master, slave = pty.openpty()
 364                 width = compat_get_terminal_size().columns
 365                 if width is None:
 366                     width_args = []
 367                 else:
 368                     width_args = ['-w', str(width)]
 369                 sp_kwargs = dict(
 370                     stdin=subprocess.PIPE,
 371                     stdout=slave,
 372                     stderr=self._err_file)
 373                 try:
 374                     self._output_process = subprocess.Popen(
 375                         ['bidiv'] + width_args, **sp_kwargs
 376                     )
 377                 except OSError:
 378                     self._output_process = subprocess.Popen(
 379                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 380                 self._output_channel = os.fdopen(master, 'rb')
 381             except OSError as ose:
 382                 if ose.errno == errno.ENOENT:
 383                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 384                 else:
 385                     raise
 386
 387         if (sys.platform != 'win32' and
 388                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
 389                 not params.get('restrictfilenames', False)):
 390             # Unicode filesystem API will throw errors (#1474, #13027)
 391             self.report_warning(
 392                 'Assuming --restrict-filenames since file system encoding '
 393                 'cannot encode all characters. '
 394                 'Set the LC_ALL environment variable to fix this.')
 395             self.params['restrictfilenames'] = True
 396
 397         if isinstance(params.get('outtmpl'), bytes):
 398             self.report_warning(
 399                 'Parameter outtmpl is bytes, but should be a unicode string. '
 400                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 401
 402         self._setup_opener()
 403
 404         if auto_init:
 405             self.print_debug_header()
 406             self.add_default_info_extractors()
 407
 408         for pp_def_raw in self.params.get('postprocessors', []):
 409             pp_class = get_postprocessor(pp_def_raw['key'])
 410             pp_def = dict(pp_def_raw)
 411             del pp_def['key']
 412             pp = pp_class(self, **compat_kwargs(pp_def))
 413             self.add_post_processor(pp)
 414
 415         for ph in self.params.get('progress_hooks', []):
 416             self.add_progress_hook(ph)
 417
 418         register_socks_protocols()
 419
 420     def warn_if_short_id(self, argv):
 421         # short YouTube ID starting with dash?
 422         idxs = [
 423             i for i, a in enumerate(argv)
 424             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 425         if idxs:
 426             correct_argv = (
 427                 ['youtube-dl'] +
 428                 [a for i, a in enumerate(argv) if i not in idxs] +
 429                 ['--'] + [argv[i] for i in idxs]
 430             )
 431             self.report_warning(
 432                 'Long argument string detected. '
 433                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 434                 args_to_str(correct_argv))
 435
 436     def add_info_extractor(self, ie):
 437         """Add an InfoExtractor object to the end of the list."""
 438         self._ies.append(ie)
 439         if not isinstance(ie, type):
 440             self._ies_instances[ie.ie_key()] = ie
 441             ie.set_downloader(self)
 442
 443     def get_info_extractor(self, ie_key):
 444         """
 445         Get an instance of an IE with name ie_key, it will try to get one from
 446         the _ies list, if there's no instance it will create a new one and add
 447         it to the extractor list.
 448         """
 449         ie = self._ies_instances.get(ie_key)
 450         if ie is None:
 451             ie = get_info_extractor(ie_key)()
 452             self.add_info_extractor(ie)
 453         return ie
 454
 455     def add_default_info_extractors(self):
 456         """
 457         Add the InfoExtractors returned by gen_extractors to the end of the list
 458         """
 459         for ie in gen_extractor_classes():
 460             self.add_info_extractor(ie)
 461
 462     def add_post_processor(self, pp):
 463         """Add a PostProcessor object to the end of the chain."""
 464         self._pps.append(pp)
 465         pp.set_downloader(self)
 466
 467     def add_progress_hook(self, ph):
 468         """Add the progress hook (currently only for the file downloader)"""
 469         self._progress_hooks.append(ph)
 470
 471     def _bidi_workaround(self, message):
 472         if not hasattr(self, '_output_channel'):
 473             return message
 474
 475         assert hasattr(self, '_output_process')
 476         assert isinstance(message, compat_str)
 477         line_count = message.count('\n') + 1
 478         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 479         self._output_process.stdin.flush()
 480         res = ''.join(self._output_channel.readline().decode('utf-8')
 481                       for _ in range(line_count))
 482         return res[:-len('\n')]
 483
 484     def to_screen(self, message, skip_eol=False):
 485         """Print message to stdout if not in quiet mode."""
 486         return self.to_stdout(message, skip_eol, check_quiet=True)
 487
 488     def _write_string(self, s, out=None):
 489         write_string(s, out=out, encoding=self.params.get('encoding'))
 490
 491     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 492         """Print message to stdout if not in quiet mode."""
 493         if self.params.get('logger'):
 494             self.params['logger'].debug(message)
 495         elif not check_quiet or not self.params.get('quiet', False):
 496             message = self._bidi_workaround(message)
 497             terminator = ['\n', ''][skip_eol]
 498             output = message + terminator
 499
 500             self._write_string(output, self._screen_file)
 501
 502     def to_stderr(self, message):
 503         """Print message to stderr."""
 504         assert isinstance(message, compat_str)
 505         if self.params.get('logger'):
 506             self.params['logger'].error(message)
 507         else:
 508             message = self._bidi_workaround(message)
 509             output = message + '\n'
 510             self._write_string(output, self._err_file)
 511
 512     def to_console_title(self, message):
 513         if not self.params.get('consoletitle', False):
 514             return
 515         if compat_os_name == 'nt':
 516             if ctypes.windll.kernel32.GetConsoleWindow():
 517                 # c_wchar_p() might not be necessary if `message` is
 518                 # already of type unicode()
 519                 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 520         elif 'TERM' in os.environ:
 521             self._write_string('\033]0;%s\007' % message, self._screen_file)
 522
 523     def save_console_title(self):
 524         if not self.params.get('consoletitle', False):
 525             return
 526         if compat_os_name != 'nt' and 'TERM' in os.environ:
 527             # Save the title on stack
 528             self._write_string('\033[22;0t', self._screen_file)
 529
 530     def restore_console_title(self):
 531         if not self.params.get('consoletitle', False):
 532             return
 533         if compat_os_name != 'nt' and 'TERM' in os.environ:
 534             # Restore the title from stack
 535             self._write_string('\033[23;0t', self._screen_file)
 536
 537     def __enter__(self):
 538         self.save_console_title()
 539         return self
 540
 541     def __exit__(self, *args):
 542         self.restore_console_title()
 543
 544         if self.params.get('cookiefile') is not None:
 545             self.cookiejar.save()
 546
 547     def trouble(self, message=None, tb=None):
 548         """Determine action to take when a download problem appears.
 549
 550         Depending on if the downloader has been configured to ignore
 551         download errors or not, this method may throw an exception or
 552         not when errors are found, after printing the message.
 553
 554         tb, if given, is additional traceback information.
 555         """
 556         if message is not None:
 557             self.to_stderr(message)
 558         if self.params.get('verbose'):
 559             if tb is None:
 560                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 561                     tb = ''
 562                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 563                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 564                     tb += encode_compat_str(traceback.format_exc())
 565                 else:
 566                     tb_data = traceback.format_list(traceback.extract_stack())
 567                     tb = ''.join(tb_data)
 568             self.to_stderr(tb)
 569         if not self.params.get('ignoreerrors', False):
 570             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 571                 exc_info = sys.exc_info()[1].exc_info
 572             else:
 573                 exc_info = sys.exc_info()
 574             raise DownloadError(message, exc_info)
 575         self._download_retcode = 1
 576
 577     def report_warning(self, message):
 578         '''
 579         Print the message to stderr, it will be prefixed with 'WARNING:'
 580         If stderr is a tty file the 'WARNING:' will be colored
 581         '''
 582         if self.params.get('logger') is not None:
 583             self.params['logger'].warning(message)
 584         else:
 585             if self.params.get('no_warnings'):
 586                 return
 587             if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 588                 _msg_header = '\033[0;33mWARNING:\033[0m'
 589             else:
 590                 _msg_header = 'WARNING:'
 591             warning_message = '%s %s' % (_msg_header, message)
 592             self.to_stderr(warning_message)
 593
 594     def report_error(self, message, tb=None):
 595         '''
 596         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 597         in red if stderr is a tty file.
 598         '''
 599         if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 600             _msg_header = '\033[0;31mERROR:\033[0m'
 601         else:
 602             _msg_header = 'ERROR:'
 603         error_message = '%s %s' % (_msg_header, message)
 604         self.trouble(error_message, tb)
 605
 606     def report_file_already_downloaded(self, file_name):
 607         """Report file has already been fully downloaded."""
 608         try:
 609             self.to_screen('[download] %s has already been downloaded' % file_name)
 610         except UnicodeEncodeError:
 611             self.to_screen('[download] The file has already been downloaded')
 612
 613     def prepare_filename(self, info_dict):
 614         """Generate the output filename."""
 615         try:
 616             template_dict = dict(info_dict)
 617
 618             template_dict['epoch'] = int(time.time())
 619             autonumber_size = self.params.get('autonumber_size')
 620             if autonumber_size is None:
 621                 autonumber_size = 5
 622             template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
 623             if template_dict.get('resolution') is None:
 624                 if template_dict.get('width') and template_dict.get('height'):
 625                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 626                 elif template_dict.get('height'):
 627                     template_dict['resolution'] = '%sp' % template_dict['height']
 628                 elif template_dict.get('width'):
 629                     template_dict['resolution'] = '%dx?' % template_dict['width']
 630
 631             sanitize = lambda k, v: sanitize_filename(
 632                 compat_str(v),
 633                 restricted=self.params.get('restrictfilenames'),
 634                 is_id=(k == 'id' or k.endswith('_id')))
 635             template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
 636                                  for k, v in template_dict.items()
 637                                  if v is not None and not isinstance(v, (list, tuple, dict)))
 638             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 639
 640             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 641
 642             # For fields playlist_index and autonumber convert all occurrences
 643             # of %(field)s to %(field)0Nd for backward compatibility
 644             field_size_compat_map = {
 645                 'playlist_index': len(str(template_dict['n_entries'])),
 646                 'autonumber': autonumber_size,
 647             }
 648             FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
 649             mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl)
 650             if mobj:
 651                 outtmpl = re.sub(
 652                     FIELD_SIZE_COMPAT_RE,
 653                     r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
 654                     outtmpl)
 655
 656             # Missing numeric fields used together with integer presentation types
 657             # in format specification will break the argument substitution since
 658             # string 'NA' is returned for missing fields. We will patch output
 659             # template for missing fields to meet string presentation type.
 660             for numeric_field in self._NUMERIC_FIELDS:
 661                 if numeric_field not in template_dict:
 662                     # As of [1] format syntax is:
 663                     #  %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
 664                     # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
 665                     FORMAT_RE = r'''(?x)
 666                         (?<!%)
 667                         %
 668                         \({0}\)  # mapping key
 669                         (?:[#0\-+ ]+)?  # conversion flags (optional)
 670                         (?:\d+)?  # minimum field width (optional)
 671                         (?:\.\d+)?  # precision (optional)
 672                         [hlL]?  # length modifier (optional)
 673                         [diouxXeEfFgGcrs%]  # conversion type
 674                     '''
 675                     outtmpl = re.sub(
 676                         FORMAT_RE.format(numeric_field),
 677                         r'%({0})s'.format(numeric_field), outtmpl)
 678
 679             # expand_path translates '%%' into '%' and '$$' into '$'
 680             # correspondingly that is not what we want since we need to keep
 681             # '%%' intact for template dict substitution step. Working around
 682             # with boundary-alike separator hack.
 683             sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
 684             outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
 685
 686             # outtmpl should be expand_path'ed before template dict substitution
 687             # because meta fields may contain env variables we don't want to
 688             # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
 689             # title "Hello $PATH", we don't want `$PATH` to be expanded.
 690             filename = expand_path(outtmpl).replace(sep, '') % template_dict
 691
 692             # Temporary fix for #4787
 693             # 'Treat' all problem characters by passing filename through preferredencoding
 694             # to workaround encoding issues with subprocess on python2 @ Windows
 695             if sys.version_info < (3, 0) and sys.platform == 'win32':
 696                 filename = encodeFilename(filename, True).decode(preferredencoding())
 697             return sanitize_path(filename)
 698         except ValueError as err:
 699             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 700             return None
 701
 702     def _match_entry(self, info_dict, incomplete):
 703         """ Returns None iff the file should be downloaded """
 704
 705         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 706         if 'title' in info_dict:
 707             # This can happen when we're just evaluating the playlist
 708             title = info_dict['title']
 709             matchtitle = self.params.get('matchtitle', False)
 710             if matchtitle:
 711                 if not re.search(matchtitle, title, re.IGNORECASE):
 712                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 713             rejecttitle = self.params.get('rejecttitle', False)
 714             if rejecttitle:
 715                 if re.search(rejecttitle, title, re.IGNORECASE):
 716                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 717         date = info_dict.get('upload_date')
 718         if date is not None:
 719             dateRange = self.params.get('daterange', DateRange())
 720             if date not in dateRange:
 721                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 722         view_count = info_dict.get('view_count')
 723         if view_count is not None:
 724             min_views = self.params.get('min_views')
 725             if min_views is not None and view_count < min_views:
 726                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 727             max_views = self.params.get('max_views')
 728             if max_views is not None and view_count > max_views:
 729                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 730         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 731             return 'Skipping "%s" because it is age restricted' % video_title
 732         if self.in_download_archive(info_dict):
 733             return '%s has already been recorded in archive' % video_title
 734
 735         if not incomplete:
 736             match_filter = self.params.get('match_filter')
 737             if match_filter is not None:
 738                 ret = match_filter(info_dict)
 739                 if ret is not None:
 740                     return ret
 741
 742         return None
 743
 744     @staticmethod
 745     def add_extra_info(info_dict, extra_info):
 746         '''Set the keys from extra_info in info dict if they are missing'''
 747         for key, value in extra_info.items():
 748             info_dict.setdefault(key, value)
 749
 750     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 751                      process=True, force_generic_extractor=False):
 752         '''
 753         Returns a list with a dictionary for each video we find.
 754         If 'download', also downloads the videos.
 755         extra_info is a dict containing the extra values to add to each result
 756         '''
 757
 758         if not ie_key and force_generic_extractor:
 759             ie_key = 'Generic'
 760
 761         if ie_key:
 762             ies = [self.get_info_extractor(ie_key)]
 763         else:
 764             ies = self._ies
 765
 766         for ie in ies:
 767             if not ie.suitable(url):
 768                 continue
 769
 770             ie = self.get_info_extractor(ie.ie_key())
 771             if not ie.working():
 772                 self.report_warning('The program functionality for this site has been marked as broken, '
 773                                     'and will probably not work.')
 774
 775             try:
 776                 ie_result = ie.extract(url)
 777                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 778                     break
 779                 if isinstance(ie_result, list):
 780                     # Backwards compatibility: old IE result format
 781                     ie_result = {
 782                         '_type': 'compat_list',
 783                         'entries': ie_result,
 784                     }
 785                 self.add_default_extra_info(ie_result, ie, url)
 786                 if process:
 787                     return self.process_ie_result(ie_result, download, extra_info)
 788                 else:
 789                     return ie_result
 790             except GeoRestrictedError as e:
 791                 msg = e.msg
 792                 if e.countries:
 793                     msg += '\nThis video is available in %s.' % ', '.join(
 794                         map(ISO3166Utils.short2full, e.countries))
 795                 msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
 796                 self.report_error(msg)
 797                 break
 798             except ExtractorError as e:  # An error we somewhat expected
 799                 self.report_error(compat_str(e), e.format_traceback())
 800                 break
 801             except MaxDownloadsReached:
 802                 raise
 803             except Exception as e:
 804                 if self.params.get('ignoreerrors', False):
 805                     self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
 806                     break
 807                 else:
 808                     raise
 809         else:
 810             self.report_error('no suitable InfoExtractor for URL %s' % url)
 811
 812     def add_default_extra_info(self, ie_result, ie, url):
 813         self.add_extra_info(ie_result, {
 814             'extractor': ie.IE_NAME,
 815             'webpage_url': url,
 816             'webpage_url_basename': url_basename(url),
 817             'extractor_key': ie.ie_key(),
 818         })
 819
 820     def process_ie_result(self, ie_result, download=True, extra_info={}):
 821         """
 822         Take the result of the ie(may be modified) and resolve all unresolved
 823         references (URLs, playlist items).
 824
 825         It will also download the videos if 'download'.
 826         Returns the resolved ie_result.
 827         """
 828         result_type = ie_result.get('_type', 'video')
 829
 830         if result_type in ('url', 'url_transparent'):
 831             ie_result['url'] = sanitize_url(ie_result['url'])
 832             extract_flat = self.params.get('extract_flat', False)
 833             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 834                     extract_flat is True):
 835                 if self.params.get('forcejson', False):
 836                     self.to_stdout(json.dumps(ie_result))
 837                 return ie_result
 838
 839         if result_type == 'video':
 840             self.add_extra_info(ie_result, extra_info)
 841             return self.process_video_result(ie_result, download=download)
 842         elif result_type == 'url':
 843             # We have to add extra_info to the results because it may be
 844             # contained in a playlist
 845             return self.extract_info(ie_result['url'],
 846                                      download,
 847                                      ie_key=ie_result.get('ie_key'),
 848                                      extra_info=extra_info)
 849         elif result_type == 'url_transparent':
 850             # Use the information from the embedding page
 851             info = self.extract_info(
 852                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 853                 extra_info=extra_info, download=False, process=False)
 854
 855             # extract_info may return None when ignoreerrors is enabled and
 856             # extraction failed with an error, don't crash and return early
 857             # in this case
 858             if not info:
 859                 return info
 860
 861             force_properties = dict(
 862                 (k, v) for k, v in ie_result.items() if v is not None)
 863             for f in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
 864                 if f in force_properties:
 865                     del force_properties[f]
 866             new_result = info.copy()
 867             new_result.update(force_properties)
 868
 869             # Extracted info may not be a video result (i.e.
 870             # info.get('_type', 'video') != video) but rather an url or
 871             # url_transparent. In such cases outer metadata (from ie_result)
 872             # should be propagated to inner one (info). For this to happen
 873             # _type of info should be overridden with url_transparent. This
 874             # fixes issue from https://github.com/rg3/youtube-dl/pull/11163.
 875             if new_result.get('_type') == 'url':
 876                 new_result['_type'] = 'url_transparent'
 877
 878             return self.process_ie_result(
 879                 new_result, download=download, extra_info=extra_info)
 880         elif result_type in ('playlist', 'multi_video'):
 881             # We process each entry in the playlist
 882             playlist = ie_result.get('title') or ie_result.get('id')
 883             self.to_screen('[download] Downloading playlist: %s' % playlist)
 884
 885             playlist_results = []
 886
 887             playliststart = self.params.get('playliststart', 1) - 1
 888             playlistend = self.params.get('playlistend')
 889             # For backwards compatibility, interpret -1 as whole list
 890             if playlistend == -1:
 891                 playlistend = None
 892
 893             playlistitems_str = self.params.get('playlist_items')
 894             playlistitems = None
 895             if playlistitems_str is not None:
 896                 def iter_playlistitems(format):
 897                     for string_segment in format.split(','):
 898                         if '-' in string_segment:
 899                             start, end = string_segment.split('-')
 900                             for item in range(int(start), int(end) + 1):
 901                                 yield int(item)
 902                         else:
 903                             yield int(string_segment)
 904                 playlistitems = iter_playlistitems(playlistitems_str)
 905
 906             ie_entries = ie_result['entries']
 907             if isinstance(ie_entries, list):
 908                 n_all_entries = len(ie_entries)
 909                 if playlistitems:
 910                     entries = [
 911                         ie_entries[i - 1] for i in playlistitems
 912                         if -n_all_entries <= i - 1 < n_all_entries]
 913                 else:
 914                     entries = ie_entries[playliststart:playlistend]
 915                 n_entries = len(entries)
 916                 self.to_screen(
 917                     '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
 918                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 919             elif isinstance(ie_entries, PagedList):
 920                 if playlistitems:
 921                     entries = []
 922                     for item in playlistitems:
 923                         entries.extend(ie_entries.getslice(
 924                             item - 1, item
 925                         ))
 926                 else:
 927                     entries = ie_entries.getslice(
 928                         playliststart, playlistend)
 929                 n_entries = len(entries)
 930                 self.to_screen(
 931                     '[%s] playlist %s: Downloading %d videos' %
 932                     (ie_result['extractor'], playlist, n_entries))
 933             else:  # iterable
 934                 if playlistitems:
 935                     entry_list = list(ie_entries)
 936                     entries = [entry_list[i - 1] for i in playlistitems]
 937                 else:
 938                     entries = list(itertools.islice(
 939                         ie_entries, playliststart, playlistend))
 940                 n_entries = len(entries)
 941                 self.to_screen(
 942                     '[%s] playlist %s: Downloading %d videos' %
 943                     (ie_result['extractor'], playlist, n_entries))
 944
 945             if self.params.get('playlistreverse', False):
 946                 entries = entries[::-1]
 947
 948             if self.params.get('playlistrandom', False):
 949                 random.shuffle(entries)
 950
 951             x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
 952
 953             for i, entry in enumerate(entries, 1):
 954                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 955                 # This __x_forwarded_for_ip thing is a bit ugly but requires
 956                 # minimal changes
 957                 if x_forwarded_for:
 958                     entry['__x_forwarded_for_ip'] = x_forwarded_for
 959                 extra = {
 960                     'n_entries': n_entries,
 961                     'playlist': playlist,
 962                     'playlist_id': ie_result.get('id'),
 963                     'playlist_title': ie_result.get('title'),
 964                     'playlist_index': i + playliststart,
 965                     'extractor': ie_result['extractor'],
 966                     'webpage_url': ie_result['webpage_url'],
 967                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 968                     'extractor_key': ie_result['extractor_key'],
 969                 }
 970
 971                 reason = self._match_entry(entry, incomplete=True)
 972                 if reason is not None:
 973                     self.to_screen('[download] ' + reason)
 974                     continue
 975
 976                 entry_result = self.process_ie_result(entry,
 977                                                       download=download,
 978                                                       extra_info=extra)
 979                 playlist_results.append(entry_result)
 980             ie_result['entries'] = playlist_results
 981             self.to_screen('[download] Finished downloading playlist: %s' % playlist)
 982             return ie_result
 983         elif result_type == 'compat_list':
 984             self.report_warning(
 985                 'Extractor %s returned a compat_list result. '
 986                 'It needs to be updated.' % ie_result.get('extractor'))
 987
 988             def _fixup(r):
 989                 self.add_extra_info(
 990                     r,
 991                     {
 992                         'extractor': ie_result['extractor'],
 993                         'webpage_url': ie_result['webpage_url'],
 994                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 995                         'extractor_key': ie_result['extractor_key'],
 996                     }
 997                 )
 998                 return r
 999             ie_result['entries'] = [
1000                 self.process_ie_result(_fixup(r), download, extra_info)
1001                 for r in ie_result['entries']
1002             ]
1003             return ie_result
1004         else:
1005             raise Exception('Invalid result type: %s' % result_type)
1006
1007     def _build_format_filter(self, filter_spec):
1008         " Returns a function to filter the formats according to the filter_spec "
1009
1010         OPERATORS = {
1011             '<': operator.lt,
1012             '<=': operator.le,
1013             '>': operator.gt,
1014             '>=': operator.ge,
1015             '=': operator.eq,
1016             '!=': operator.ne,
1017         }
1018         operator_rex = re.compile(r'''(?x)\s*
1019             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
1020             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1021             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
1022             $
1023             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
1024         m = operator_rex.search(filter_spec)
1025         if m:
1026             try:
1027                 comparison_value = int(m.group('value'))
1028             except ValueError:
1029                 comparison_value = parse_filesize(m.group('value'))
1030                 if comparison_value is None:
1031                     comparison_value = parse_filesize(m.group('value') + 'B')
1032                 if comparison_value is None:
1033                     raise ValueError(
1034                         'Invalid value %r in format specification %r' % (
1035                             m.group('value'), filter_spec))
1036             op = OPERATORS[m.group('op')]
1037
1038         if not m:
1039             STR_OPERATORS = {
1040                 '=': operator.eq,
1041                 '!=': operator.ne,
1042                 '^=': lambda attr, value: attr.startswith(value),
1043                 '$=': lambda attr, value: attr.endswith(value),
1044                 '*=': lambda attr, value: value in attr,
1045             }
1046             str_operator_rex = re.compile(r'''(?x)
1047                 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
1048                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
1049                 \s*(?P<value>[a-zA-Z0-9._-]+)
1050                 \s*$
1051                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
1052             m = str_operator_rex.search(filter_spec)
1053             if m:
1054                 comparison_value = m.group('value')
1055                 op = STR_OPERATORS[m.group('op')]
1056
1057         if not m:
1058             raise ValueError('Invalid filter specification %r' % filter_spec)
1059
1060         def _filter(f):
1061             actual_value = f.get(m.group('key'))
1062             if actual_value is None:
1063                 return m.group('none_inclusive')
1064             return op(actual_value, comparison_value)
1065         return _filter
1066
1067     def _default_format_spec(self, info_dict, download=True):
1068         req_format_list = []
1069
1070         def can_have_partial_formats():
1071             if self.params.get('simulate', False):
1072                 return True
1073             if not download:
1074                 return True
1075             if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
1076                 return False
1077             if info_dict.get('is_live'):
1078                 return False
1079             merger = FFmpegMergerPP(self)
1080             return merger.available and merger.can_merge()
1081         if can_have_partial_formats():
1082             req_format_list.append('bestvideo+bestaudio')
1083         req_format_list.append('best')
1084         return '/'.join(req_format_list)
1085
1086     def build_format_selector(self, format_spec):
1087         def syntax_error(note, start):
1088             message = (
1089                 'Invalid format specification: '
1090                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
1091             return SyntaxError(message)
1092
1093         PICKFIRST = 'PICKFIRST'
1094         MERGE = 'MERGE'
1095         SINGLE = 'SINGLE'
1096         GROUP = 'GROUP'
1097         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1098
1099         def _parse_filter(tokens):
1100             filter_parts = []
1101             for type, string, start, _, _ in tokens:
1102                 if type == tokenize.OP and string == ']':
1103                     return ''.join(filter_parts)
1104                 else:
1105                     filter_parts.append(string)
1106
1107         def _remove_unused_ops(tokens):
1108             # Remove operators that we don't use and join them with the surrounding strings
1109             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1110             ALLOWED_OPS = ('/', '+', ',', '(', ')')
1111             last_string, last_start, last_end, last_line = None, None, None, None
1112             for type, string, start, end, line in tokens:
1113                 if type == tokenize.OP and string == '[':
1114                     if last_string:
1115                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1116                         last_string = None
1117                     yield type, string, start, end, line
1118                     # everything inside brackets will be handled by _parse_filter
1119                     for type, string, start, end, line in tokens:
1120                         yield type, string, start, end, line
1121                         if type == tokenize.OP and string == ']':
1122                             break
1123                 elif type == tokenize.OP and string in ALLOWED_OPS:
1124                     if last_string:
1125                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1126                         last_string = None
1127                     yield type, string, start, end, line
1128                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
1129                     if not last_string:
1130                         last_string = string
1131                         last_start = start
1132                         last_end = end
1133                     else:
1134                         last_string += string
1135             if last_string:
1136                 yield tokenize.NAME, last_string, last_start, last_end, last_line
1137
1138         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
1139             selectors = []
1140             current_selector = None
1141             for type, string, start, _, _ in tokens:
1142                 # ENCODING is only defined in python 3.x
1143                 if type == getattr(tokenize, 'ENCODING', None):
1144                     continue
1145                 elif type in [tokenize.NAME, tokenize.NUMBER]:
1146                     current_selector = FormatSelector(SINGLE, string, [])
1147                 elif type == tokenize.OP:
1148                     if string == ')':
1149                         if not inside_group:
1150                             # ')' will be handled by the parentheses group
1151                             tokens.restore_last_token()
1152                         break
1153                     elif inside_merge and string in ['/', ',']:
1154                         tokens.restore_last_token()
1155                         break
1156                     elif inside_choice and string == ',':
1157                         tokens.restore_last_token()
1158                         break
1159                     elif string == ',':
1160                         if not current_selector:
1161                             raise syntax_error('"," must follow a format selector', start)
1162                         selectors.append(current_selector)
1163                         current_selector = None
1164                     elif string == '/':
1165                         if not current_selector:
1166                             raise syntax_error('"/" must follow a format selector', start)
1167                         first_choice = current_selector
1168                         second_choice = _parse_format_selection(tokens, inside_choice=True)
1169                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1170                     elif string == '[':
1171                         if not current_selector:
1172                             current_selector = FormatSelector(SINGLE, 'best', [])
1173                         format_filter = _parse_filter(tokens)
1174                         current_selector.filters.append(format_filter)
1175                     elif string == '(':
1176                         if current_selector:
1177                             raise syntax_error('Unexpected "("', start)
1178                         group = _parse_format_selection(tokens, inside_group=True)
1179                         current_selector = FormatSelector(GROUP, group, [])
1180                     elif string == '+':
1181                         video_selector = current_selector
1182                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
1183                         if not video_selector or not audio_selector:
1184                             raise syntax_error('"+" must be between two format selectors', start)
1185                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1186                     else:
1187                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1188                 elif type == tokenize.ENDMARKER:
1189                     break
1190             if current_selector:
1191                 selectors.append(current_selector)
1192             return selectors
1193
1194         def _build_selector_function(selector):
1195             if isinstance(selector, list):
1196                 fs = [_build_selector_function(s) for s in selector]
1197
1198                 def selector_function(ctx):
1199                     for f in fs:
1200                         for format in f(ctx):
1201                             yield format
1202                 return selector_function
1203             elif selector.type == GROUP:
1204                 selector_function = _build_selector_function(selector.selector)
1205             elif selector.type == PICKFIRST:
1206                 fs = [_build_selector_function(s) for s in selector.selector]
1207
1208                 def selector_function(ctx):
1209                     for f in fs:
1210                         picked_formats = list(f(ctx))
1211                         if picked_formats:
1212                             return picked_formats
1213                     return []
1214             elif selector.type == SINGLE:
1215                 format_spec = selector.selector
1216
1217                 def selector_function(ctx):
1218                     formats = list(ctx['formats'])
1219                     if not formats:
1220                         return
1221                     if format_spec == 'all':
1222                         for f in formats:
1223                             yield f
1224                     elif format_spec in ['best', 'worst', None]:
1225                         format_idx = 0 if format_spec == 'worst' else -1
1226                         audiovideo_formats = [
1227                             f for f in formats
1228                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1229                         if audiovideo_formats:
1230                             yield audiovideo_formats[format_idx]
1231                         # for extractors with incomplete formats (audio only (soundcloud)
1232                         # or video only (imgur)) we will fallback to best/worst
1233                         # {video,audio}-only format
1234                         elif ctx['incomplete_formats']:
1235                             yield formats[format_idx]
1236                     elif format_spec == 'bestaudio':
1237                         audio_formats = [
1238                             f for f in formats
1239                             if f.get('vcodec') == 'none']
1240                         if audio_formats:
1241                             yield audio_formats[-1]
1242                     elif format_spec == 'worstaudio':
1243                         audio_formats = [
1244                             f for f in formats
1245                             if f.get('vcodec') == 'none']
1246                         if audio_formats:
1247                             yield audio_formats[0]
1248                     elif format_spec == 'bestvideo':
1249                         video_formats = [
1250                             f for f in formats
1251                             if f.get('acodec') == 'none']
1252                         if video_formats:
1253                             yield video_formats[-1]
1254                     elif format_spec == 'worstvideo':
1255                         video_formats = [
1256                             f for f in formats
1257                             if f.get('acodec') == 'none']
1258                         if video_formats:
1259                             yield video_formats[0]
1260                     else:
1261                         extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1262                         if format_spec in extensions:
1263                             filter_f = lambda f: f['ext'] == format_spec
1264                         else:
1265                             filter_f = lambda f: f['format_id'] == format_spec
1266                         matches = list(filter(filter_f, formats))
1267                         if matches:
1268                             yield matches[-1]
1269             elif selector.type == MERGE:
1270                 def _merge(formats_info):
1271                     format_1, format_2 = [f['format_id'] for f in formats_info]
1272                     # The first format must contain the video and the
1273                     # second the audio
1274                     if formats_info[0].get('vcodec') == 'none':
1275                         self.report_error('The first format must '
1276                                           'contain the video, try using '
1277                                           '"-f %s+%s"' % (format_2, format_1))
1278                         return
1279                     # Formats must be opposite (video+audio)
1280                     if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1281                         self.report_error(
1282                             'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1283                             % (format_1, format_2))
1284                         return
1285                     output_ext = (
1286                         formats_info[0]['ext']
1287                         if self.params.get('merge_output_format') is None
1288                         else self.params['merge_output_format'])
1289                     return {
1290                         'requested_formats': formats_info,
1291                         'format': '%s+%s' % (formats_info[0].get('format'),
1292                                              formats_info[1].get('format')),
1293                         'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1294                                                 formats_info[1].get('format_id')),
1295                         'width': formats_info[0].get('width'),
1296                         'height': formats_info[0].get('height'),
1297                         'resolution': formats_info[0].get('resolution'),
1298                         'fps': formats_info[0].get('fps'),
1299                         'vcodec': formats_info[0].get('vcodec'),
1300                         'vbr': formats_info[0].get('vbr'),
1301                         'stretched_ratio': formats_info[0].get('stretched_ratio'),
1302                         'acodec': formats_info[1].get('acodec'),
1303                         'abr': formats_info[1].get('abr'),
1304                         'ext': output_ext,
1305                     }
1306                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1307
1308                 def selector_function(ctx):
1309                     for pair in itertools.product(
1310                             video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
1311                         yield _merge(pair)
1312
1313             filters = [self._build_format_filter(f) for f in selector.filters]
1314
1315             def final_selector(ctx):
1316                 ctx_copy = copy.deepcopy(ctx)
1317                 for _filter in filters:
1318                     ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
1319                 return selector_function(ctx_copy)
1320             return final_selector
1321
1322         stream = io.BytesIO(format_spec.encode('utf-8'))
1323         try:
1324             tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1325         except tokenize.TokenError:
1326             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1327
1328         class TokenIterator(object):
1329             def __init__(self, tokens):
1330                 self.tokens = tokens
1331                 self.counter = 0
1332
1333             def __iter__(self):
1334                 return self
1335
1336             def __next__(self):
1337                 if self.counter >= len(self.tokens):
1338                     raise StopIteration()
1339                 value = self.tokens[self.counter]
1340                 self.counter += 1
1341                 return value
1342
1343             next = __next__
1344
1345             def restore_last_token(self):
1346                 self.counter -= 1
1347
1348         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1349         return _build_selector_function(parsed_selector)
1350
1351     def _calc_headers(self, info_dict):
1352         res = std_headers.copy()
1353
1354         add_headers = info_dict.get('http_headers')
1355         if add_headers:
1356             res.update(add_headers)
1357
1358         cookies = self._calc_cookies(info_dict)
1359         if cookies:
1360             res['Cookie'] = cookies
1361
1362         if 'X-Forwarded-For' not in res:
1363             x_forwarded_for_ip = info_dict.get('__x_forwarded_for_ip')
1364             if x_forwarded_for_ip:
1365                 res['X-Forwarded-For'] = x_forwarded_for_ip
1366
1367         return res
1368
1369     def _calc_cookies(self, info_dict):
1370         pr = sanitized_Request(info_dict['url'])
1371         self.cookiejar.add_cookie_header(pr)
1372         return pr.get_header('Cookie')
1373
1374     def process_video_result(self, info_dict, download=True):
1375         assert info_dict.get('_type', 'video') == 'video'
1376
1377         if 'id' not in info_dict:
1378             raise ExtractorError('Missing "id" field in extractor result')
1379         if 'title' not in info_dict:
1380             raise ExtractorError('Missing "title" field in extractor result')
1381
1382         def report_force_conversion(field, field_not, conversion):
1383             self.report_warning(
1384                 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1385                 % (field, field_not, conversion))
1386
1387         def sanitize_string_field(info, string_field):
1388             field = info.get(string_field)
1389             if field is None or isinstance(field, compat_str):
1390                 return
1391             report_force_conversion(string_field, 'a string', 'string')
1392             info[string_field] = compat_str(field)
1393
1394         def sanitize_numeric_fields(info):
1395             for numeric_field in self._NUMERIC_FIELDS:
1396                 field = info.get(numeric_field)
1397                 if field is None or isinstance(field, compat_numeric_types):
1398                     continue
1399                 report_force_conversion(numeric_field, 'numeric', 'int')
1400                 info[numeric_field] = int_or_none(field)
1401
1402         sanitize_string_field(info_dict, 'id')
1403         sanitize_numeric_fields(info_dict)
1404
1405         if 'playlist' not in info_dict:
1406             # It isn't part of a playlist
1407             info_dict['playlist'] = None
1408             info_dict['playlist_index'] = None
1409
1410         thumbnails = info_dict.get('thumbnails')
1411         if thumbnails is None:
1412             thumbnail = info_dict.get('thumbnail')
1413             if thumbnail:
1414                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1415         if thumbnails:
1416             thumbnails.sort(key=lambda t: (
1417                 t.get('preference') if t.get('preference') is not None else -1,
1418                 t.get('width') if t.get('width') is not None else -1,
1419                 t.get('height') if t.get('height') is not None else -1,
1420                 t.get('id') if t.get('id') is not None else '', t.get('url')))
1421             for i, t in enumerate(thumbnails):
1422                 t['url'] = sanitize_url(t['url'])
1423                 if t.get('width') and t.get('height'):
1424                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1425                 if t.get('id') is None:
1426                     t['id'] = '%d' % i
1427
1428         if self.params.get('list_thumbnails'):
1429             self.list_thumbnails(info_dict)
1430             return
1431
1432         thumbnail = info_dict.get('thumbnail')
1433         if thumbnail:
1434             info_dict['thumbnail'] = sanitize_url(thumbnail)
1435         elif thumbnails:
1436             info_dict['thumbnail'] = thumbnails[-1]['url']
1437
1438         if 'display_id' not in info_dict and 'id' in info_dict:
1439             info_dict['display_id'] = info_dict['id']
1440
1441         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1442             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1443             # see http://bugs.python.org/issue1646728)
1444             try:
1445                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1446                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1447             except (ValueError, OverflowError, OSError):
1448                 pass
1449
1450         # Auto generate title fields corresponding to the *_number fields when missing
1451         # in order to always have clean titles. This is very common for TV series.
1452         for field in ('chapter', 'season', 'episode'):
1453             if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1454                 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1455
1456         subtitles = info_dict.get('subtitles')
1457         if subtitles:
1458             for _, subtitle in subtitles.items():
1459                 for subtitle_format in subtitle:
1460                     if subtitle_format.get('url'):
1461                         subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1462                     if subtitle_format.get('ext') is None:
1463                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1464
1465         if self.params.get('listsubtitles', False):
1466             if 'automatic_captions' in info_dict:
1467                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1468             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1469             return
1470         info_dict['requested_subtitles'] = self.process_subtitles(
1471             info_dict['id'], subtitles,
1472             info_dict.get('automatic_captions'))
1473
1474         # We now pick which formats have to be downloaded
1475         if info_dict.get('formats') is None:
1476             # There's only one format available
1477             formats = [info_dict]
1478         else:
1479             formats = info_dict['formats']
1480
1481         if not formats:
1482             raise ExtractorError('No video formats found!')
1483
1484         def is_wellformed(f):
1485             url = f.get('url')
1486             if not url:
1487                 self.report_warning(
1488                     '"url" field is missing or empty - skipping format, '
1489                     'there is an error in extractor')
1490                 return False
1491             if isinstance(url, bytes):
1492                 sanitize_string_field(f, 'url')
1493             return True
1494
1495         # Filter out malformed formats for better extraction robustness
1496         formats = list(filter(is_wellformed, formats))
1497
1498         formats_dict = {}
1499
1500         # We check that all the formats have the format and format_id fields
1501         for i, format in enumerate(formats):
1502             sanitize_string_field(format, 'format_id')
1503             sanitize_numeric_fields(format)
1504             format['url'] = sanitize_url(format['url'])
1505             if not format.get('format_id'):
1506                 format['format_id'] = compat_str(i)
1507             else:
1508                 # Sanitize format_id from characters used in format selector expression
1509                 format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
1510             format_id = format['format_id']
1511             if format_id not in formats_dict:
1512                 formats_dict[format_id] = []
1513             formats_dict[format_id].append(format)
1514
1515         # Make sure all formats have unique format_id
1516         for format_id, ambiguous_formats in formats_dict.items():
1517             if len(ambiguous_formats) > 1:
1518                 for i, format in enumerate(ambiguous_formats):
1519                     format['format_id'] = '%s-%d' % (format_id, i)
1520
1521         for i, format in enumerate(formats):
1522             if format.get('format') is None:
1523                 format['format'] = '{id} - {res}{note}'.format(
1524                     id=format['format_id'],
1525                     res=self.format_resolution(format),
1526                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1527                 )
1528             # Automatically determine file extension if missing
1529             if format.get('ext') is None:
1530                 format['ext'] = determine_ext(format['url']).lower()
1531             # Automatically determine protocol if missing (useful for format
1532             # selection purposes)
1533             if format.get('protocol') is None:
1534                 format['protocol'] = determine_protocol(format)
1535             # Add HTTP headers, so that external programs can use them from the
1536             # json output
1537             full_format_info = info_dict.copy()
1538             full_format_info.update(format)
1539             format['http_headers'] = self._calc_headers(full_format_info)
1540         # Remove private housekeeping stuff
1541         if '__x_forwarded_for_ip' in info_dict:
1542             del info_dict['__x_forwarded_for_ip']
1543
1544         # TODO Central sorting goes here
1545
1546         if formats[0] is not info_dict:
1547             # only set the 'formats' fields if the original info_dict list them
1548             # otherwise we end up with a circular reference, the first (and unique)
1549             # element in the 'formats' field in info_dict is info_dict itself,
1550             # which can't be exported to json
1551             info_dict['formats'] = formats
1552         if self.params.get('listformats'):
1553             self.list_formats(info_dict)
1554             return
1555
1556         req_format = self.params.get('format')
1557         if req_format is None:
1558             req_format = self._default_format_spec(info_dict, download=download)
1559             if self.params.get('verbose'):
1560                 self.to_stdout('[debug] Default format spec: %s' % req_format)
1561
1562         format_selector = self.build_format_selector(req_format)
1563
1564         # While in format selection we may need to have an access to the original
1565         # format set in order to calculate some metrics or do some processing.
1566         # For now we need to be able to guess whether original formats provided
1567         # by extractor are incomplete or not (i.e. whether extractor provides only
1568         # video-only or audio-only formats) for proper formats selection for
1569         # extractors with such incomplete formats (see
1570         # https://github.com/rg3/youtube-dl/pull/5556).
1571         # Since formats may be filtered during format selection and may not match
1572         # the original formats the results may be incorrect. Thus original formats
1573         # or pre-calculated metrics should be passed to format selection routines
1574         # as well.
1575         # We will pass a context object containing all necessary additional data
1576         # instead of just formats.
1577         # This fixes incorrect format selection issue (see
1578         # https://github.com/rg3/youtube-dl/issues/10083).
1579         incomplete_formats = (
1580             # All formats are video-only or
1581             all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
1582             # all formats are audio-only
1583             all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
1584
1585         ctx = {
1586             'formats': formats,
1587             'incomplete_formats': incomplete_formats,
1588         }
1589
1590         formats_to_download = list(format_selector(ctx))
1591         if not formats_to_download:
1592             raise ExtractorError('requested format not available',
1593                                  expected=True)
1594
1595         if download:
1596             if len(formats_to_download) > 1:
1597                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1598             for format in formats_to_download:
1599                 new_info = dict(info_dict)
1600                 new_info.update(format)
1601                 self.process_info(new_info)
1602         # We update the info dict with the best quality format (backwards compatibility)
1603         info_dict.update(formats_to_download[-1])
1604         return info_dict
1605
1606     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1607         """Select the requested subtitles and their format"""
1608         available_subs = {}
1609         if normal_subtitles and self.params.get('writesubtitles'):
1610             available_subs.update(normal_subtitles)
1611         if automatic_captions and self.params.get('writeautomaticsub'):
1612             for lang, cap_info in automatic_captions.items():
1613                 if lang not in available_subs:
1614                     available_subs[lang] = cap_info
1615
1616         if (not self.params.get('writesubtitles') and not
1617                 self.params.get('writeautomaticsub') or not
1618                 available_subs):
1619             return None
1620
1621         if self.params.get('allsubtitles', False):
1622             requested_langs = available_subs.keys()
1623         else:
1624             if self.params.get('subtitleslangs', False):
1625                 requested_langs = self.params.get('subtitleslangs')
1626             elif 'en' in available_subs:
1627                 requested_langs = ['en']
1628             else:
1629                 requested_langs = [list(available_subs.keys())[0]]
1630
1631         formats_query = self.params.get('subtitlesformat', 'best')
1632         formats_preference = formats_query.split('/') if formats_query else []
1633         subs = {}
1634         for lang in requested_langs:
1635             formats = available_subs.get(lang)
1636             if formats is None:
1637                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1638                 continue
1639             for ext in formats_preference:
1640                 if ext == 'best':
1641                     f = formats[-1]
1642                     break
1643                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1644                 if matches:
1645                     f = matches[-1]
1646                     break
1647             else:
1648                 f = formats[-1]
1649                 self.report_warning(
1650                     'No subtitle format found matching "%s" for language %s, '
1651                     'using %s' % (formats_query, lang, f['ext']))
1652             subs[lang] = f
1653         return subs
1654
1655     def process_info(self, info_dict):
1656         """Process a single resolved IE result."""
1657
1658         assert info_dict.get('_type', 'video') == 'video'
1659
1660         max_downloads = self.params.get('max_downloads')
1661         if max_downloads is not None:
1662             if self._num_downloads >= int(max_downloads):
1663                 raise MaxDownloadsReached()
1664
1665         info_dict['fulltitle'] = info_dict['title']
1666         if len(info_dict['title']) > 200:
1667             info_dict['title'] = info_dict['title'][:197] + '...'
1668
1669         if 'format' not in info_dict:
1670             info_dict['format'] = info_dict['ext']
1671
1672         reason = self._match_entry(info_dict, incomplete=False)
1673         if reason is not None:
1674             self.to_screen('[download] ' + reason)
1675             return
1676
1677         self._num_downloads += 1
1678
1679         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1680
1681         # Forced printings
1682         if self.params.get('forcetitle', False):
1683             self.to_stdout(info_dict['fulltitle'])
1684         if self.params.get('forceid', False):
1685             self.to_stdout(info_dict['id'])
1686         if self.params.get('forceurl', False):
1687             if info_dict.get('requested_formats') is not None:
1688                 for f in info_dict['requested_formats']:
1689                     self.to_stdout(f['url'] + f.get('play_path', ''))
1690             else:
1691                 # For RTMP URLs, also include the playpath
1692                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1693         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1694             self.to_stdout(info_dict['thumbnail'])
1695         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1696             self.to_stdout(info_dict['description'])
1697         if self.params.get('forcefilename', False) and filename is not None:
1698             self.to_stdout(filename)
1699         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1700             self.to_stdout(formatSeconds(info_dict['duration']))
1701         if self.params.get('forceformat', False):
1702             self.to_stdout(info_dict['format'])
1703         if self.params.get('forcejson', False):
1704             self.to_stdout(json.dumps(info_dict))
1705
1706         # Do nothing else if in simulate mode
1707         if self.params.get('simulate', False):
1708             return
1709
1710         if filename is None:
1711             return
1712
1713         def ensure_dir_exists(path):
1714             try:
1715                 dn = os.path.dirname(path)
1716                 if dn and not os.path.exists(dn):
1717                     os.makedirs(dn)
1718                 return True
1719             except (OSError, IOError) as err:
1720                 self.report_error('unable to create directory ' + error_to_compat_str(err))
1721                 return False
1722
1723         if not ensure_dir_exists(sanitize_path(encodeFilename(filename))):
1724             return
1725
1726         if self.params.get('writedescription', False):
1727             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1728             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1729                 self.to_screen('[info] Video description is already present')
1730             elif info_dict.get('description') is None:
1731                 self.report_warning('There\'s no description to write.')
1732             else:
1733                 try:
1734                     self.to_screen('[info] Writing video description to: ' + descfn)
1735                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1736                         descfile.write(info_dict['description'])
1737                 except (OSError, IOError):
1738                     self.report_error('Cannot write description file ' + descfn)
1739                     return
1740
1741         if self.params.get('writeannotations', False):
1742             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1743             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1744                 self.to_screen('[info] Video annotations are already present')
1745             else:
1746                 try:
1747                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1748                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1749                         annofile.write(info_dict['annotations'])
1750                 except (KeyError, TypeError):
1751                     self.report_warning('There are no annotations to write.')
1752                 except (OSError, IOError):
1753                     self.report_error('Cannot write annotations file: ' + annofn)
1754                     return
1755
1756         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1757                                        self.params.get('writeautomaticsub')])
1758
1759         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1760             # subtitles download errors are already managed as troubles in relevant IE
1761             # that way it will silently go on when used with unsupporting IE
1762             subtitles = info_dict['requested_subtitles']
1763             ie = self.get_info_extractor(info_dict['extractor_key'])
1764             for sub_lang, sub_info in subtitles.items():
1765                 sub_format = sub_info['ext']
1766                 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1767                 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1768                     self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
1769                 else:
1770                     self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1771                     if sub_info.get('data') is not None:
1772                         try:
1773                             # Use newline='' to prevent conversion of newline characters
1774                             # See https://github.com/rg3/youtube-dl/issues/10268
1775                             with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
1776                                 subfile.write(sub_info['data'])
1777                         except (OSError, IOError):
1778                             self.report_error('Cannot write subtitles file ' + sub_filename)
1779                             return
1780                     else:
1781                         try:
1782                             sub_data = ie._request_webpage(
1783                                 sub_info['url'], info_dict['id'], note=False).read()
1784                             with io.open(encodeFilename(sub_filename), 'wb') as subfile:
1785                                 subfile.write(sub_data)
1786                         except (ExtractorError, IOError, OSError, ValueError) as err:
1787                             self.report_warning('Unable to download subtitle for "%s": %s' %
1788                                                 (sub_lang, error_to_compat_str(err)))
1789                             continue
1790
1791         if self.params.get('writeinfojson', False):
1792             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1793             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1794                 self.to_screen('[info] Video description metadata is already present')
1795             else:
1796                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1797                 try:
1798                     write_json_file(self.filter_requested_info(info_dict), infofn)
1799                 except (OSError, IOError):
1800                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1801                     return
1802
1803         self._write_thumbnails(info_dict, filename)
1804
1805         if not self.params.get('skip_download', False):
1806             try:
1807                 def dl(name, info):
1808                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1809                     for ph in self._progress_hooks:
1810                         fd.add_progress_hook(ph)
1811                     if self.params.get('verbose'):
1812                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1813                     return fd.download(name, info)
1814
1815                 if info_dict.get('requested_formats') is not None:
1816                     downloaded = []
1817                     success = True
1818                     merger = FFmpegMergerPP(self)
1819                     if not merger.available:
1820                         postprocessors = []
1821                         self.report_warning('You have requested multiple '
1822                                             'formats but ffmpeg or avconv are not installed.'
1823                                             ' The formats won\'t be merged.')
1824                     else:
1825                         postprocessors = [merger]
1826
1827                     def compatible_formats(formats):
1828                         video, audio = formats
1829                         # Check extension
1830                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1831                         if video_ext and audio_ext:
1832                             COMPATIBLE_EXTS = (
1833                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'),
1834                                 ('webm')
1835                             )
1836                             for exts in COMPATIBLE_EXTS:
1837                                 if video_ext in exts and audio_ext in exts:
1838                                     return True
1839                         # TODO: Check acodec/vcodec
1840                         return False
1841
1842                     filename_real_ext = os.path.splitext(filename)[1][1:]
1843                     filename_wo_ext = (
1844                         os.path.splitext(filename)[0]
1845                         if filename_real_ext == info_dict['ext']
1846                         else filename)
1847                     requested_formats = info_dict['requested_formats']
1848                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1849                         info_dict['ext'] = 'mkv'
1850                         self.report_warning(
1851                             'Requested formats are incompatible for merge and will be merged into mkv.')
1852                     # Ensure filename always has a correct extension for successful merge
1853                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1854                     if os.path.exists(encodeFilename(filename)):
1855                         self.to_screen(
1856                             '[download] %s has already been downloaded and '
1857                             'merged' % filename)
1858                     else:
1859                         for f in requested_formats:
1860                             new_info = dict(info_dict)
1861                             new_info.update(f)
1862                             fname = prepend_extension(
1863                                 self.prepare_filename(new_info),
1864                                 'f%s' % f['format_id'], new_info['ext'])
1865                             if not ensure_dir_exists(fname):
1866                                 return
1867                             downloaded.append(fname)
1868                             partial_success = dl(fname, new_info)
1869                             success = success and partial_success
1870                         info_dict['__postprocessors'] = postprocessors
1871                         info_dict['__files_to_merge'] = downloaded
1872                 else:
1873                     # Just a single file
1874                     success = dl(filename, info_dict)
1875             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1876                 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1877                 return
1878             except (OSError, IOError) as err:
1879                 raise UnavailableVideoError(err)
1880             except (ContentTooShortError, ) as err:
1881                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1882                 return
1883
1884             if success and filename != '-':
1885                 # Fixup content
1886                 fixup_policy = self.params.get('fixup')
1887                 if fixup_policy is None:
1888                     fixup_policy = 'detect_or_warn'
1889
1890                 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1891
1892                 stretched_ratio = info_dict.get('stretched_ratio')
1893                 if stretched_ratio is not None and stretched_ratio != 1:
1894                     if fixup_policy == 'warn':
1895                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1896                             info_dict['id'], stretched_ratio))
1897                     elif fixup_policy == 'detect_or_warn':
1898                         stretched_pp = FFmpegFixupStretchedPP(self)
1899                         if stretched_pp.available:
1900                             info_dict.setdefault('__postprocessors', [])
1901                             info_dict['__postprocessors'].append(stretched_pp)
1902                         else:
1903                             self.report_warning(
1904                                 '%s: Non-uniform pixel ratio (%s). %s'
1905                                 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1906                     else:
1907                         assert fixup_policy in ('ignore', 'never')
1908
1909                 if (info_dict.get('requested_formats') is None and
1910                         info_dict.get('container') == 'm4a_dash'):
1911                     if fixup_policy == 'warn':
1912                         self.report_warning(
1913                             '%s: writing DASH m4a. '
1914                             'Only some players support this container.'
1915                             % info_dict['id'])
1916                     elif fixup_policy == 'detect_or_warn':
1917                         fixup_pp = FFmpegFixupM4aPP(self)
1918                         if fixup_pp.available:
1919                             info_dict.setdefault('__postprocessors', [])
1920                             info_dict['__postprocessors'].append(fixup_pp)
1921                         else:
1922                             self.report_warning(
1923                                 '%s: writing DASH m4a. '
1924                                 'Only some players support this container. %s'
1925                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1926                     else:
1927                         assert fixup_policy in ('ignore', 'never')
1928
1929                 if (info_dict.get('protocol') == 'm3u8_native' or
1930                         info_dict.get('protocol') == 'm3u8' and
1931                         self.params.get('hls_prefer_native')):
1932                     if fixup_policy == 'warn':
1933                         self.report_warning('%s: malformed AAC bitstream detected.' % (
1934                             info_dict['id']))
1935                     elif fixup_policy == 'detect_or_warn':
1936                         fixup_pp = FFmpegFixupM3u8PP(self)
1937                         if fixup_pp.available:
1938                             info_dict.setdefault('__postprocessors', [])
1939                             info_dict['__postprocessors'].append(fixup_pp)
1940                         else:
1941                             self.report_warning(
1942                                 '%s: malformed AAC bitstream detected. %s'
1943                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1944                     else:
1945                         assert fixup_policy in ('ignore', 'never')
1946
1947                 try:
1948                     self.post_process(filename, info_dict)
1949                 except (PostProcessingError) as err:
1950                     self.report_error('postprocessing: %s' % str(err))
1951                     return
1952                 self.record_download_archive(info_dict)
1953
1954     def download(self, url_list):
1955         """Download a given list of URLs."""
1956         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1957         if (len(url_list) > 1 and
1958                 outtmpl != '-' and
1959                 '%' not in outtmpl and
1960                 self.params.get('max_downloads') != 1):
1961             raise SameFileError(outtmpl)
1962
1963         for url in url_list:
1964             try:
1965                 # It also downloads the videos
1966                 res = self.extract_info(
1967                     url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1968             except UnavailableVideoError:
1969                 self.report_error('unable to download video')
1970             except MaxDownloadsReached:
1971                 self.to_screen('[info] Maximum number of downloaded files reached.')
1972                 raise
1973             else:
1974                 if self.params.get('dump_single_json', False):
1975                     self.to_stdout(json.dumps(res))
1976
1977         return self._download_retcode
1978
1979     def download_with_info_file(self, info_filename):
1980         with contextlib.closing(fileinput.FileInput(
1981                 [info_filename], mode='r',
1982                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1983             # FileInput doesn't have a read method, we can't call json.load
1984             info = self.filter_requested_info(json.loads('\n'.join(f)))
1985         try:
1986             self.process_ie_result(info, download=True)
1987         except DownloadError:
1988             webpage_url = info.get('webpage_url')
1989             if webpage_url is not None:
1990                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1991                 return self.download([webpage_url])
1992             else:
1993                 raise
1994         return self._download_retcode
1995
1996     @staticmethod
1997     def filter_requested_info(info_dict):
1998         return dict(
1999             (k, v) for k, v in info_dict.items()
2000             if k not in ['requested_formats', 'requested_subtitles'])
2001
2002     def post_process(self, filename, ie_info):
2003         """Run all the postprocessors on the given file."""
2004         info = dict(ie_info)
2005         info['filepath'] = filename
2006         pps_chain = []
2007         if ie_info.get('__postprocessors') is not None:
2008             pps_chain.extend(ie_info['__postprocessors'])
2009         pps_chain.extend(self._pps)
2010         for pp in pps_chain:
2011             files_to_delete = []
2012             try:
2013                 files_to_delete, info = pp.run(info)
2014             except PostProcessingError as e:
2015                 self.report_error(e.msg)
2016             if files_to_delete and not self.params.get('keepvideo', False):
2017                 for old_filename in files_to_delete:
2018                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
2019                     try:
2020                         os.remove(encodeFilename(old_filename))
2021                     except (IOError, OSError):
2022                         self.report_warning('Unable to remove downloaded original file')
2023
2024     def _make_archive_id(self, info_dict):
2025         # Future-proof against any change in case
2026         # and backwards compatibility with prior versions
2027         extractor = info_dict.get('extractor_key')
2028         if extractor is None:
2029             if 'id' in info_dict:
2030                 extractor = info_dict.get('ie_key')  # key in a playlist
2031         if extractor is None:
2032             return None  # Incomplete video information
2033         return extractor.lower() + ' ' + info_dict['id']
2034
2035     def in_download_archive(self, info_dict):
2036         fn = self.params.get('download_archive')
2037         if fn is None:
2038             return False
2039
2040         vid_id = self._make_archive_id(info_dict)
2041         if vid_id is None:
2042             return False  # Incomplete video information
2043
2044         try:
2045             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
2046                 for line in archive_file:
2047                     if line.strip() == vid_id:
2048                         return True
2049         except IOError as ioe:
2050             if ioe.errno != errno.ENOENT:
2051                 raise
2052         return False
2053
2054     def record_download_archive(self, info_dict):
2055         fn = self.params.get('download_archive')
2056         if fn is None:
2057             return
2058         vid_id = self._make_archive_id(info_dict)
2059         assert vid_id
2060         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
2061             archive_file.write(vid_id + '\n')
2062
2063     @staticmethod
2064     def format_resolution(format, default='unknown'):
2065         if format.get('vcodec') == 'none':
2066             return 'audio only'
2067         if format.get('resolution') is not None:
2068             return format['resolution']
2069         if format.get('height') is not None:
2070             if format.get('width') is not None:
2071                 res = '%sx%s' % (format['width'], format['height'])
2072             else:
2073                 res = '%sp' % format['height']
2074         elif format.get('width') is not None:
2075             res = '%dx?' % format['width']
2076         else:
2077             res = default
2078         return res
2079
2080     def _format_note(self, fdict):
2081         res = ''
2082         if fdict.get('ext') in ['f4f', 'f4m']:
2083             res += '(unsupported) '
2084         if fdict.get('language'):
2085             if res:
2086                 res += ' '
2087             res += '[%s] ' % fdict['language']
2088         if fdict.get('format_note') is not None:
2089             res += fdict['format_note'] + ' '
2090         if fdict.get('tbr') is not None:
2091             res += '%4dk ' % fdict['tbr']
2092         if fdict.get('container') is not None:
2093             if res:
2094                 res += ', '
2095             res += '%s container' % fdict['container']
2096         if (fdict.get('vcodec') is not None and
2097                 fdict.get('vcodec') != 'none'):
2098             if res:
2099                 res += ', '
2100             res += fdict['vcodec']
2101             if fdict.get('vbr') is not None:
2102                 res += '@'
2103         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
2104             res += 'video@'
2105         if fdict.get('vbr') is not None:
2106             res += '%4dk' % fdict['vbr']
2107         if fdict.get('fps') is not None:
2108             if res:
2109                 res += ', '
2110             res += '%sfps' % fdict['fps']
2111         if fdict.get('acodec') is not None:
2112             if res:
2113                 res += ', '
2114             if fdict['acodec'] == 'none':
2115                 res += 'video only'
2116             else:
2117                 res += '%-5s' % fdict['acodec']
2118         elif fdict.get('abr') is not None:
2119             if res:
2120                 res += ', '
2121             res += 'audio'
2122         if fdict.get('abr') is not None:
2123             res += '@%3dk' % fdict['abr']
2124         if fdict.get('asr') is not None:
2125             res += ' (%5dHz)' % fdict['asr']
2126         if fdict.get('filesize') is not None:
2127             if res:
2128                 res += ', '
2129             res += format_bytes(fdict['filesize'])
2130         elif fdict.get('filesize_approx') is not None:
2131             if res:
2132                 res += ', '
2133             res += '~' + format_bytes(fdict['filesize_approx'])
2134         return res
2135
2136     def list_formats(self, info_dict):
2137         formats = info_dict.get('formats', [info_dict])
2138         table = [
2139             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
2140             for f in formats
2141             if f.get('preference') is None or f['preference'] >= -1000]
2142         if len(formats) > 1:
2143             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
2144
2145         header_line = ['format code', 'extension', 'resolution', 'note']
2146         self.to_screen(
2147             '[info] Available formats for %s:\n%s' %
2148             (info_dict['id'], render_table(header_line, table)))
2149
2150     def list_thumbnails(self, info_dict):
2151         thumbnails = info_dict.get('thumbnails')
2152         if not thumbnails:
2153             self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
2154             return
2155
2156         self.to_screen(
2157             '[info] Thumbnails for %s:' % info_dict['id'])
2158         self.to_screen(render_table(
2159             ['ID', 'width', 'height', 'URL'],
2160             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
2161
2162     def list_subtitles(self, video_id, subtitles, name='subtitles'):
2163         if not subtitles:
2164             self.to_screen('%s has no %s' % (video_id, name))
2165             return
2166         self.to_screen(
2167             'Available %s for %s:' % (name, video_id))
2168         self.to_screen(render_table(
2169             ['Language', 'formats'],
2170             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
2171                 for lang, formats in subtitles.items()]))
2172
2173     def urlopen(self, req):
2174         """ Start an HTTP download """
2175         if isinstance(req, compat_basestring):
2176             req = sanitized_Request(req)
2177         return self._opener.open(req, timeout=self._socket_timeout)
2178
2179     def print_debug_header(self):
2180         if not self.params.get('verbose'):
2181             return
2182
2183         if type('') is not compat_str:
2184             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
2185             self.report_warning(
2186                 'Your Python is broken! Update to a newer and supported version')
2187
2188         stdout_encoding = getattr(
2189             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
2190         encoding_str = (
2191             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2192                 locale.getpreferredencoding(),
2193                 sys.getfilesystemencoding(),
2194                 stdout_encoding,
2195                 self.get_encoding()))
2196         write_string(encoding_str, encoding=None)
2197
2198         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
2199         if _LAZY_LOADER:
2200             self._write_string('[debug] Lazy loading extractors enabled' + '\n')
2201         try:
2202             sp = subprocess.Popen(
2203                 ['git', 'rev-parse', '--short', 'HEAD'],
2204                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2205                 cwd=os.path.dirname(os.path.abspath(__file__)))
2206             out, err = sp.communicate()
2207             out = out.decode().strip()
2208             if re.match('[0-9a-f]+', out):
2209                 self._write_string('[debug] Git HEAD: ' + out + '\n')
2210         except Exception:
2211             try:
2212                 sys.exc_clear()
2213             except Exception:
2214                 pass
2215         self._write_string('[debug] Python version %s - %s\n' % (
2216             platform.python_version(), platform_name()))
2217
2218         exe_versions = FFmpegPostProcessor.get_versions(self)
2219         exe_versions['rtmpdump'] = rtmpdump_version()
2220         exe_str = ', '.join(
2221             '%s %s' % (exe, v)
2222             for exe, v in sorted(exe_versions.items())
2223             if v
2224         )
2225         if not exe_str:
2226             exe_str = 'none'
2227         self._write_string('[debug] exe versions: %s\n' % exe_str)
2228
2229         proxy_map = {}
2230         for handler in self._opener.handlers:
2231             if hasattr(handler, 'proxies'):
2232                 proxy_map.update(handler.proxies)
2233         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2234
2235         if self.params.get('call_home', False):
2236             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2237             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2238             latest_version = self.urlopen(
2239                 'https://yt-dl.org/latest/version').read().decode('utf-8')
2240             if version_tuple(latest_version) > version_tuple(__version__):
2241                 self.report_warning(
2242                     'You are using an outdated version (newest version: %s)! '
2243                     'See https://yt-dl.org/update if you need help updating.' %
2244                     latest_version)
2245
2246     def _setup_opener(self):
2247         timeout_val = self.params.get('socket_timeout')
2248         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2249
2250         opts_cookiefile = self.params.get('cookiefile')
2251         opts_proxy = self.params.get('proxy')
2252
2253         if opts_cookiefile is None:
2254             self.cookiejar = compat_cookiejar.CookieJar()
2255         else:
2256             opts_cookiefile = expand_path(opts_cookiefile)
2257             self.cookiejar = compat_cookiejar.MozillaCookieJar(
2258                 opts_cookiefile)
2259             if os.access(opts_cookiefile, os.R_OK):
2260                 self.cookiejar.load()
2261
2262         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2263         if opts_proxy is not None:
2264             if opts_proxy == '':
2265                 proxies = {}
2266             else:
2267                 proxies = {'http': opts_proxy, 'https': opts_proxy}
2268         else:
2269             proxies = compat_urllib_request.getproxies()
2270             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2271             if 'http' in proxies and 'https' not in proxies:
2272                 proxies['https'] = proxies['http']
2273         proxy_handler = PerRequestProxyHandler(proxies)
2274
2275         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2276         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2277         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2278         data_handler = compat_urllib_request_DataHandler()
2279
2280         # When passing our own FileHandler instance, build_opener won't add the
2281         # default FileHandler and allows us to disable the file protocol, which
2282         # can be used for malicious purposes (see
2283         # https://github.com/rg3/youtube-dl/issues/8227)
2284         file_handler = compat_urllib_request.FileHandler()
2285
2286         def file_open(*args, **kwargs):
2287             raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2288         file_handler.file_open = file_open
2289
2290         opener = compat_urllib_request.build_opener(
2291             proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2292
2293         # Delete the default user-agent header, which would otherwise apply in
2294         # cases where our custom HTTP handler doesn't come into play
2295         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2296         opener.addheaders = []
2297         self._opener = opener
2298
2299     def encode(self, s):
2300         if isinstance(s, bytes):
2301             return s  # Already encoded
2302
2303         try:
2304             return s.encode(self.get_encoding())
2305         except UnicodeEncodeError as err:
2306             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2307             raise
2308
2309     def get_encoding(self):
2310         encoding = self.params.get('encoding')
2311         if encoding is None:
2312             encoding = preferredencoding()
2313         return encoding
2314
2315     def _write_thumbnails(self, info_dict, filename):
2316         if self.params.get('writethumbnail', False):
2317             thumbnails = info_dict.get('thumbnails')
2318             if thumbnails:
2319                 thumbnails = [thumbnails[-1]]
2320         elif self.params.get('write_all_thumbnails', False):
2321             thumbnails = info_dict.get('thumbnails')
2322         else:
2323             return
2324
2325         if not thumbnails:
2326             # No thumbnails present, so return immediately
2327             return
2328
2329         for t in thumbnails:
2330             thumb_ext = determine_ext(t['url'], 'jpg')
2331             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2332             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2333             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2334
2335             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2336                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2337                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2338             else:
2339                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2340                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2341                 try:
2342                     uf = self.urlopen(t['url'])
2343                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2344                         shutil.copyfileobj(uf, thumbf)
2345                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2346                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2347                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2348                     self.report_warning('Unable to download thumbnail "%s": %s' %
2349                                         (t['url'], error_to_compat_str(err)))