_ Git - youtube-dl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # coding: utf-8
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import contextlib
   8 import copy
   9 import datetime
  10 import errno
  11 import fileinput
  12 import io
  13 import itertools
  14 import json
  15 import locale
  16 import operator
  17 import os
  18 import platform
  19 import re
  20 import shutil
  21 import subprocess
  22 import socket
  23 import sys
  24 import time
  25 import tokenize
  26 import traceback
  27 import random
  28
  29 from string import ascii_letters
  30
  31 from .compat import (
  32     compat_basestring,
  33     compat_cookiejar,
  34     compat_get_terminal_size,
  35     compat_http_client,
  36     compat_kwargs,
  37     compat_numeric_types,
  38     compat_os_name,
  39     compat_str,
  40     compat_tokenize_tokenize,
  41     compat_urllib_error,
  42     compat_urllib_request,
  43     compat_urllib_request_DataHandler,
  44 )
  45 from .utils import (
  46     age_restricted,
  47     args_to_str,
  48     ContentTooShortError,
  49     date_from_str,
  50     DateRange,
  51     DEFAULT_OUTTMPL,
  52     determine_ext,
  53     determine_protocol,
  54     DownloadError,
  55     encode_compat_str,
  56     encodeFilename,
  57     error_to_compat_str,
  58     expand_path,
  59     ExtractorError,
  60     format_bytes,
  61     formatSeconds,
  62     GeoRestrictedError,
  63     int_or_none,
  64     ISO3166Utils,
  65     locked_file,
  66     make_HTTPS_handler,
  67     MaxDownloadsReached,
  68     PagedList,
  69     parse_filesize,
  70     PerRequestProxyHandler,
  71     platform_name,
  72     PostProcessingError,
  73     preferredencoding,
  74     prepend_extension,
  75     register_socks_protocols,
  76     render_table,
  77     replace_extension,
  78     SameFileError,
  79     sanitize_filename,
  80     sanitize_path,
  81     sanitize_url,
  82     sanitized_Request,
  83     std_headers,
  84     subtitles_filename,
  85     UnavailableVideoError,
  86     url_basename,
  87     version_tuple,
  88     write_json_file,
  89     write_string,
  90     YoutubeDLCookieProcessor,
  91     YoutubeDLHandler,
  92 )
  93 from .cache import Cache
  94 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
  95 from .downloader import get_suitable_downloader
  96 from .downloader.rtmp import rtmpdump_version
  97 from .postprocessor import (
  98     FFmpegFixupM3u8PP,
  99     FFmpegFixupM4aPP,
 100     FFmpegFixupStretchedPP,
 101     FFmpegMergerPP,
 102     FFmpegPostProcessor,
 103     get_postprocessor,
 104 )
 105 from .version import __version__
 106
 107 if compat_os_name == 'nt':
 108     import ctypes
 109
 110
 111 class YoutubeDL(object):
 112     """YoutubeDL class.
 113
 114     YoutubeDL objects are the ones responsible of downloading the
 115     actual video file and writing it to disk if the user has requested
 116     it, among some other tasks. In most cases there should be one per
 117     program. As, given a video URL, the downloader doesn't know how to
 118     extract all the needed information, task that InfoExtractors do, it
 119     has to pass the URL to one of them.
 120
 121     For this, YoutubeDL objects have a method that allows
 122     InfoExtractors to be registered in a given order. When it is passed
 123     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 124     finds that reports being able to handle it. The InfoExtractor extracts
 125     all the information about the video or videos the URL refers to, and
 126     YoutubeDL process the extracted information, possibly using a File
 127     Downloader to download the video.
 128
 129     YoutubeDL objects accept a lot of parameters. In order not to saturate
 130     the object constructor with arguments, it receives a dictionary of
 131     options instead. These options are available through the params
 132     attribute for the InfoExtractors to use. The YoutubeDL also
 133     registers itself as the downloader in charge for the InfoExtractors
 134     that are added to it, so this is a "mutual registration".
 135
 136     Available options:
 137
 138     username:          Username for authentication purposes.
 139     password:          Password for authentication purposes.
 140     videopassword:     Password for accessing a video.
 141     ap_mso:            Adobe Pass multiple-system operator identifier.
 142     ap_username:       Multiple-system operator account username.
 143     ap_password:       Multiple-system operator account password.
 144     usenetrc:          Use netrc for authentication instead.
 145     verbose:           Print additional info to stdout.
 146     quiet:             Do not print messages to stdout.
 147     no_warnings:       Do not print out anything for warnings.
 148     forceurl:          Force printing final URL.
 149     forcetitle:        Force printing title.
 150     forceid:           Force printing ID.
 151     forcethumbnail:    Force printing thumbnail URL.
 152     forcedescription:  Force printing description.
 153     forcefilename:     Force printing final filename.
 154     forceduration:     Force printing duration.
 155     forcejson:         Force printing info_dict as JSON.
 156     dump_single_json:  Force printing the info_dict of the whole playlist
 157                        (or video) as a single JSON line.
 158     simulate:          Do not download the video files.
 159     format:            Video format code. See options.py for more information.
 160     outtmpl:           Template for output names.
 161     restrictfilenames: Do not allow "&" and spaces in file names
 162     ignoreerrors:      Do not stop on download errors.
 163     force_generic_extractor: Force downloader to use the generic extractor
 164     nooverwrites:      Prevent overwriting files.
 165     playliststart:     Playlist item to start at.
 166     playlistend:       Playlist item to end at.
 167     playlist_items:    Specific indices of playlist to download.
 168     playlistreverse:   Download playlist items in reverse order.
 169     playlistrandom:    Download playlist items in random order.
 170     matchtitle:        Download only matching titles.
 171     rejecttitle:       Reject downloads for matching titles.
 172     logger:            Log messages to a logging.Logger instance.
 173     logtostderr:       Log messages to stderr instead of stdout.
 174     writedescription:  Write the video description to a .description file
 175     writeinfojson:     Write the video description to a .info.json file
 176     writeannotations:  Write the video annotations to a .annotations.xml file
 177     writethumbnail:    Write the thumbnail image to a file
 178     write_all_thumbnails:  Write all thumbnail formats to files
 179     writesubtitles:    Write the video subtitles to a file
 180     writeautomaticsub: Write the automatically generated subtitles to a file
 181     allsubtitles:      Downloads all the subtitles of the video
 182                        (requires writesubtitles or writeautomaticsub)
 183     listsubtitles:     Lists all available subtitles for the video
 184     subtitlesformat:   The format code for subtitles
 185     subtitleslangs:    List of languages of the subtitles to download
 186     keepvideo:         Keep the video file after post-processing
 187     daterange:         A DateRange object, download only if the upload_date is in the range.
 188     skip_download:     Skip the actual download of the video file
 189     cachedir:          Location of the cache files in the filesystem.
 190                        False to disable filesystem cache.
 191     noplaylist:        Download single video instead of a playlist if in doubt.
 192     age_limit:         An integer representing the user's age in years.
 193                        Unsuitable videos for the given age are skipped.
 194     min_views:         An integer representing the minimum view count the video
 195                        must have in order to not be skipped.
 196                        Videos without view count information are always
 197                        downloaded. None for no limit.
 198     max_views:         An integer representing the maximum view count.
 199                        Videos that are more popular than that are not
 200                        downloaded.
 201                        Videos without view count information are always
 202                        downloaded. None for no limit.
 203     download_archive:  File name of a file where all downloads are recorded.
 204                        Videos already present in the file are not downloaded
 205                        again.
 206     cookiefile:        File name where cookies should be read from and dumped to.
 207     nocheckcertificate:Do not verify SSL certificates
 208     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 209                        At the moment, this is only supported by YouTube.
 210     proxy:             URL of the proxy server to use
 211     geo_verification_proxy:  URL of the proxy to use for IP address verification
 212                        on geo-restricted sites. (Experimental)
 213     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 214     bidi_workaround:   Work around buggy terminals without bidirectional text
 215                        support, using fridibi
 216     debug_printtraffic:Print out sent and received HTTP traffic
 217     include_ads:       Download ads as well
 218     default_search:    Prepend this string if an input url is not valid.
 219                        'auto' for elaborate guessing
 220     encoding:          Use this encoding instead of the system-specified.
 221     extract_flat:      Do not resolve URLs, return the immediate result.
 222                        Pass in 'in_playlist' to only show this behavior for
 223                        playlist items.
 224     postprocessors:    A list of dictionaries, each with an entry
 225                        * key:  The name of the postprocessor. See
 226                                youtube_dl/postprocessor/__init__.py for a list.
 227                        as well as any further keyword arguments for the
 228                        postprocessor.
 229     progress_hooks:    A list of functions that get called on download
 230                        progress, with a dictionary with the entries
 231                        * status: One of "downloading", "error", or "finished".
 232                                  Check this first and ignore unknown values.
 233
 234                        If status is one of "downloading", or "finished", the
 235                        following properties may also be present:
 236                        * filename: The final filename (always present)
 237                        * tmpfilename: The filename we're currently writing to
 238                        * downloaded_bytes: Bytes on disk
 239                        * total_bytes: Size of the whole file, None if unknown
 240                        * total_bytes_estimate: Guess of the eventual file size,
 241                                                None if unavailable.
 242                        * elapsed: The number of seconds since download started.
 243                        * eta: The estimated time in seconds, None if unknown
 244                        * speed: The download speed in bytes/second, None if
 245                                 unknown
 246                        * fragment_index: The counter of the currently
 247                                          downloaded video fragment.
 248                        * fragment_count: The number of fragments (= individual
 249                                          files that will be merged)
 250
 251                        Progress hooks are guaranteed to be called at least once
 252                        (with status "finished") if the download is successful.
 253     merge_output_format: Extension to use when merging formats.
 254     fixup:             Automatically correct known faults of the file.
 255                        One of:
 256                        - "never": do nothing
 257                        - "warn": only emit a warning
 258                        - "detect_or_warn": check whether we can do anything
 259                                            about it, warn otherwise (default)
 260     source_address:    (Experimental) Client-side IP address to bind to.
 261     call_home:         Boolean, true iff we are allowed to contact the
 262                        youtube-dl servers for debugging.
 263     sleep_interval:    Number of seconds to sleep before each download when
 264                        used alone or a lower bound of a range for randomized
 265                        sleep before each download (minimum possible number
 266                        of seconds to sleep) when used along with
 267                        max_sleep_interval.
 268     max_sleep_interval:Upper bound of a range for randomized sleep before each
 269                        download (maximum possible number of seconds to sleep).
 270                        Must only be used along with sleep_interval.
 271                        Actual sleep time will be a random float from range
 272                        [sleep_interval; max_sleep_interval].
 273     listformats:       Print an overview of available video formats and exit.
 274     list_thumbnails:   Print a table of all thumbnails and exit.
 275     match_filter:      A function that gets called with the info_dict of
 276                        every video.
 277                        If it returns a message, the video is ignored.
 278                        If it returns None, the video is downloaded.
 279                        match_filter_func in utils.py is one example for this.
 280     no_color:          Do not emit color codes in output.
 281     geo_bypass:        Bypass geographic restriction via faking X-Forwarded-For
 282                        HTTP header (experimental)
 283     geo_bypass_country:
 284                        Two-letter ISO 3166-2 country code that will be used for
 285                        explicit geographic restriction bypassing via faking
 286                        X-Forwarded-For HTTP header (experimental)
 287
 288     The following options determine which downloader is picked:
 289     external_downloader: Executable of the external downloader to call.
 290                        None or unset for standard (built-in) downloader.
 291     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
 292                        if True, otherwise use ffmpeg/avconv if False, otherwise
 293                        use downloader suggested by extractor if None.
 294
 295     The following parameters are not used by YoutubeDL itself, they are used by
 296     the downloader (see youtube_dl/downloader/common.py):
 297     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 298     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 299     xattr_set_filesize, external_downloader_args, hls_use_mpegts.
 300
 301     The following options are used by the post processors:
 302     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 303                        otherwise prefer avconv.
 304     postprocessor_args: A list of additional command-line arguments for the
 305                         postprocessor.
 306     """
 307
 308     _NUMERIC_FIELDS = set((
 309         'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
 310         'timestamp', 'upload_year', 'upload_month', 'upload_day',
 311         'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
 312         'average_rating', 'comment_count', 'age_limit',
 313         'start_time', 'end_time',
 314         'chapter_number', 'season_number', 'episode_number',
 315         'track_number', 'disc_number', 'release_year',
 316         'playlist_index',
 317     ))
 318
 319     params = None
 320     _ies = []
 321     _pps = []
 322     _download_retcode = None
 323     _num_downloads = None
 324     _screen_file = None
 325
 326     def __init__(self, params=None, auto_init=True):
 327         """Create a FileDownloader object with the given options."""
 328         if params is None:
 329             params = {}
 330         self._ies = []
 331         self._ies_instances = {}
 332         self._pps = []
 333         self._progress_hooks = []
 334         self._download_retcode = 0
 335         self._num_downloads = 0
 336         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 337         self._err_file = sys.stderr
 338         self.params = {
 339             # Default parameters
 340             'nocheckcertificate': False,
 341         }
 342         self.params.update(params)
 343         self.cache = Cache(self)
 344
 345         def check_deprecated(param, option, suggestion):
 346             if self.params.get(param) is not None:
 347                 self.report_warning(
 348                     '%s is deprecated. Use %s instead.' % (option, suggestion))
 349                 return True
 350             return False
 351
 352         if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
 353             if self.params.get('geo_verification_proxy') is None:
 354                 self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
 355
 356         check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
 357         check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
 358         check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
 359
 360         if params.get('bidi_workaround', False):
 361             try:
 362                 import pty
 363                 master, slave = pty.openpty()
 364                 width = compat_get_terminal_size().columns
 365                 if width is None:
 366                     width_args = []
 367                 else:
 368                     width_args = ['-w', str(width)]
 369                 sp_kwargs = dict(
 370                     stdin=subprocess.PIPE,
 371                     stdout=slave,
 372                     stderr=self._err_file)
 373                 try:
 374                     self._output_process = subprocess.Popen(
 375                         ['bidiv'] + width_args, **sp_kwargs
 376                     )
 377                 except OSError:
 378                     self._output_process = subprocess.Popen(
 379                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 380                 self._output_channel = os.fdopen(master, 'rb')
 381             except OSError as ose:
 382                 if ose.errno == errno.ENOENT:
 383                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 384                 else:
 385                     raise
 386
 387         if (sys.platform != 'win32' and
 388                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
 389                 not params.get('restrictfilenames', False)):
 390             # Unicode filesystem API will throw errors (#1474, #13027)
 391             self.report_warning(
 392                 'Assuming --restrict-filenames since file system encoding '
 393                 'cannot encode all characters. '
 394                 'Set the LC_ALL environment variable to fix this.')
 395             self.params['restrictfilenames'] = True
 396
 397         if isinstance(params.get('outtmpl'), bytes):
 398             self.report_warning(
 399                 'Parameter outtmpl is bytes, but should be a unicode string. '
 400                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 401
 402         self._setup_opener()
 403
 404         if auto_init:
 405             self.print_debug_header()
 406             self.add_default_info_extractors()
 407
 408         for pp_def_raw in self.params.get('postprocessors', []):
 409             pp_class = get_postprocessor(pp_def_raw['key'])
 410             pp_def = dict(pp_def_raw)
 411             del pp_def['key']
 412             pp = pp_class(self, **compat_kwargs(pp_def))
 413             self.add_post_processor(pp)
 414
 415         for ph in self.params.get('progress_hooks', []):
 416             self.add_progress_hook(ph)
 417
 418         register_socks_protocols()
 419
 420     def warn_if_short_id(self, argv):
 421         # short YouTube ID starting with dash?
 422         idxs = [
 423             i for i, a in enumerate(argv)
 424             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 425         if idxs:
 426             correct_argv = (
 427                 ['youtube-dl'] +
 428                 [a for i, a in enumerate(argv) if i not in idxs] +
 429                 ['--'] + [argv[i] for i in idxs]
 430             )
 431             self.report_warning(
 432                 'Long argument string detected. '
 433                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 434                 args_to_str(correct_argv))
 435
 436     def add_info_extractor(self, ie):
 437         """Add an InfoExtractor object to the end of the list."""
 438         self._ies.append(ie)
 439         if not isinstance(ie, type):
 440             self._ies_instances[ie.ie_key()] = ie
 441             ie.set_downloader(self)
 442
 443     def get_info_extractor(self, ie_key):
 444         """
 445         Get an instance of an IE with name ie_key, it will try to get one from
 446         the _ies list, if there's no instance it will create a new one and add
 447         it to the extractor list.
 448         """
 449         ie = self._ies_instances.get(ie_key)
 450         if ie is None:
 451             ie = get_info_extractor(ie_key)()
 452             self.add_info_extractor(ie)
 453         return ie
 454
 455     def add_default_info_extractors(self):
 456         """
 457         Add the InfoExtractors returned by gen_extractors to the end of the list
 458         """
 459         for ie in gen_extractor_classes():
 460             self.add_info_extractor(ie)
 461
 462     def add_post_processor(self, pp):
 463         """Add a PostProcessor object to the end of the chain."""
 464         self._pps.append(pp)
 465         pp.set_downloader(self)
 466
 467     def add_progress_hook(self, ph):
 468         """Add the progress hook (currently only for the file downloader)"""
 469         self._progress_hooks.append(ph)
 470
 471     def _bidi_workaround(self, message):
 472         if not hasattr(self, '_output_channel'):
 473             return message
 474
 475         assert hasattr(self, '_output_process')
 476         assert isinstance(message, compat_str)
 477         line_count = message.count('\n') + 1
 478         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 479         self._output_process.stdin.flush()
 480         res = ''.join(self._output_channel.readline().decode('utf-8')
 481                       for _ in range(line_count))
 482         return res[:-len('\n')]
 483
 484     def to_screen(self, message, skip_eol=False):
 485         """Print message to stdout if not in quiet mode."""
 486         return self.to_stdout(message, skip_eol, check_quiet=True)
 487
 488     def _write_string(self, s, out=None):
 489         write_string(s, out=out, encoding=self.params.get('encoding'))
 490
 491     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 492         """Print message to stdout if not in quiet mode."""
 493         if self.params.get('logger'):
 494             self.params['logger'].debug(message)
 495         elif not check_quiet or not self.params.get('quiet', False):
 496             message = self._bidi_workaround(message)
 497             terminator = ['\n', ''][skip_eol]
 498             output = message + terminator
 499
 500             self._write_string(output, self._screen_file)
 501
 502     def to_stderr(self, message):
 503         """Print message to stderr."""
 504         assert isinstance(message, compat_str)
 505         if self.params.get('logger'):
 506             self.params['logger'].error(message)
 507         else:
 508             message = self._bidi_workaround(message)
 509             output = message + '\n'
 510             self._write_string(output, self._err_file)
 511
 512     def to_console_title(self, message):
 513         if not self.params.get('consoletitle', False):
 514             return
 515         if compat_os_name == 'nt':
 516             if ctypes.windll.kernel32.GetConsoleWindow():
 517                 # c_wchar_p() might not be necessary if `message` is
 518                 # already of type unicode()
 519                 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 520         elif 'TERM' in os.environ:
 521             self._write_string('\033]0;%s\007' % message, self._screen_file)
 522
 523     def save_console_title(self):
 524         if not self.params.get('consoletitle', False):
 525             return
 526         if compat_os_name != 'nt' and 'TERM' in os.environ:
 527             # Save the title on stack
 528             self._write_string('\033[22;0t', self._screen_file)
 529
 530     def restore_console_title(self):
 531         if not self.params.get('consoletitle', False):
 532             return
 533         if compat_os_name != 'nt' and 'TERM' in os.environ:
 534             # Restore the title from stack
 535             self._write_string('\033[23;0t', self._screen_file)
 536
 537     def __enter__(self):
 538         self.save_console_title()
 539         return self
 540
 541     def __exit__(self, *args):
 542         self.restore_console_title()
 543
 544         if self.params.get('cookiefile') is not None:
 545             self.cookiejar.save()
 546
 547     def trouble(self, message=None, tb=None):
 548         """Determine action to take when a download problem appears.
 549
 550         Depending on if the downloader has been configured to ignore
 551         download errors or not, this method may throw an exception or
 552         not when errors are found, after printing the message.
 553
 554         tb, if given, is additional traceback information.
 555         """
 556         if message is not None:
 557             self.to_stderr(message)
 558         if self.params.get('verbose'):
 559             if tb is None:
 560                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 561                     tb = ''
 562                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 563                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 564                     tb += encode_compat_str(traceback.format_exc())
 565                 else:
 566                     tb_data = traceback.format_list(traceback.extract_stack())
 567                     tb = ''.join(tb_data)
 568             self.to_stderr(tb)
 569         if not self.params.get('ignoreerrors', False):
 570             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 571                 exc_info = sys.exc_info()[1].exc_info
 572             else:
 573                 exc_info = sys.exc_info()
 574             raise DownloadError(message, exc_info)
 575         self._download_retcode = 1
 576
 577     def report_warning(self, message):
 578         '''
 579         Print the message to stderr, it will be prefixed with 'WARNING:'
 580         If stderr is a tty file the 'WARNING:' will be colored
 581         '''
 582         if self.params.get('logger') is not None:
 583             self.params['logger'].warning(message)
 584         else:
 585             if self.params.get('no_warnings'):
 586                 return
 587             if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 588                 _msg_header = '\033[0;33mWARNING:\033[0m'
 589             else:
 590                 _msg_header = 'WARNING:'
 591             warning_message = '%s %s' % (_msg_header, message)
 592             self.to_stderr(warning_message)
 593
 594     def report_error(self, message, tb=None):
 595         '''
 596         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 597         in red if stderr is a tty file.
 598         '''
 599         if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 600             _msg_header = '\033[0;31mERROR:\033[0m'
 601         else:
 602             _msg_header = 'ERROR:'
 603         error_message = '%s %s' % (_msg_header, message)
 604         self.trouble(error_message, tb)
 605
 606     def report_file_already_downloaded(self, file_name):
 607         """Report file has already been fully downloaded."""
 608         try:
 609             self.to_screen('[download] %s has already been downloaded' % file_name)
 610         except UnicodeEncodeError:
 611             self.to_screen('[download] The file has already been downloaded')
 612
 613     def prepare_filename(self, info_dict):
 614         """Generate the output filename."""
 615         try:
 616             template_dict = dict(info_dict)
 617
 618             template_dict['epoch'] = int(time.time())
 619             autonumber_size = self.params.get('autonumber_size')
 620             if autonumber_size is None:
 621                 autonumber_size = 5
 622             template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
 623             if template_dict.get('resolution') is None:
 624                 if template_dict.get('width') and template_dict.get('height'):
 625                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 626                 elif template_dict.get('height'):
 627                     template_dict['resolution'] = '%sp' % template_dict['height']
 628                 elif template_dict.get('width'):
 629                     template_dict['resolution'] = '%dx?' % template_dict['width']
 630
 631             sanitize = lambda k, v: sanitize_filename(
 632                 compat_str(v),
 633                 restricted=self.params.get('restrictfilenames'),
 634                 is_id=(k == 'id' or k.endswith('_id')))
 635             template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
 636                                  for k, v in template_dict.items()
 637                                  if v is not None and not isinstance(v, (list, tuple, dict)))
 638             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 639
 640             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 641
 642             # For fields playlist_index and autonumber convert all occurrences
 643             # of %(field)s to %(field)0Nd for backward compatibility
 644             field_size_compat_map = {
 645                 'playlist_index': len(str(template_dict['n_entries'])),
 646                 'autonumber': autonumber_size,
 647             }
 648             FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
 649             mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl)
 650             if mobj:
 651                 outtmpl = re.sub(
 652                     FIELD_SIZE_COMPAT_RE,
 653                     r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
 654                     outtmpl)
 655
 656             # Missing numeric fields used together with integer presentation types
 657             # in format specification will break the argument substitution since
 658             # string 'NA' is returned for missing fields. We will patch output
 659             # template for missing fields to meet string presentation type.
 660             for numeric_field in self._NUMERIC_FIELDS:
 661                 if numeric_field not in template_dict:
 662                     # As of [1] format syntax is:
 663                     #  %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
 664                     # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
 665                     FORMAT_RE = r'''(?x)
 666                         (?<!%)
 667                         %
 668                         \({0}\)  # mapping key
 669                         (?:[#0\-+ ]+)?  # conversion flags (optional)
 670                         (?:\d+)?  # minimum field width (optional)
 671                         (?:\.\d+)?  # precision (optional)
 672                         [hlL]?  # length modifier (optional)
 673                         [diouxXeEfFgGcrs%]  # conversion type
 674                     '''
 675                     outtmpl = re.sub(
 676                         FORMAT_RE.format(numeric_field),
 677                         r'%({0})s'.format(numeric_field), outtmpl)
 678
 679             # expand_path translates '%%' into '%' and '$$' into '$'
 680             # correspondingly that is not what we want since we need to keep
 681             # '%%' intact for template dict substitution step. Working around
 682             # with boundary-alike separator hack.
 683             sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
 684             outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
 685
 686             # outtmpl should be expand_path'ed before template dict substitution
 687             # because meta fields may contain env variables we don't want to
 688             # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
 689             # title "Hello $PATH", we don't want `$PATH` to be expanded.
 690             filename = expand_path(outtmpl).replace(sep, '') % template_dict
 691
 692             # Temporary fix for #4787
 693             # 'Treat' all problem characters by passing filename through preferredencoding
 694             # to workaround encoding issues with subprocess on python2 @ Windows
 695             if sys.version_info < (3, 0) and sys.platform == 'win32':
 696                 filename = encodeFilename(filename, True).decode(preferredencoding())
 697             return sanitize_path(filename)
 698         except ValueError as err:
 699             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 700             return None
 701
 702     def _match_entry(self, info_dict, incomplete):
 703         """ Returns None iff the file should be downloaded """
 704
 705         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 706         if 'title' in info_dict:
 707             # This can happen when we're just evaluating the playlist
 708             title = info_dict['title']
 709             matchtitle = self.params.get('matchtitle', False)
 710             if matchtitle:
 711                 if not re.search(matchtitle, title, re.IGNORECASE):
 712                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 713             rejecttitle = self.params.get('rejecttitle', False)
 714             if rejecttitle:
 715                 if re.search(rejecttitle, title, re.IGNORECASE):
 716                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 717         date = info_dict.get('upload_date')
 718         if date is not None:
 719             dateRange = self.params.get('daterange', DateRange())
 720             if date not in dateRange:
 721                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 722         view_count = info_dict.get('view_count')
 723         if view_count is not None:
 724             min_views = self.params.get('min_views')
 725             if min_views is not None and view_count < min_views:
 726                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 727             max_views = self.params.get('max_views')
 728             if max_views is not None and view_count > max_views:
 729                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 730         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 731             return 'Skipping "%s" because it is age restricted' % video_title
 732         if self.in_download_archive(info_dict):
 733             return '%s has already been recorded in archive' % video_title
 734
 735         if not incomplete:
 736             match_filter = self.params.get('match_filter')
 737             if match_filter is not None:
 738                 ret = match_filter(info_dict)
 739                 if ret is not None:
 740                     return ret
 741
 742         return None
 743
 744     @staticmethod
 745     def add_extra_info(info_dict, extra_info):
 746         '''Set the keys from extra_info in info dict if they are missing'''
 747         for key, value in extra_info.items():
 748             info_dict.setdefault(key, value)
 749
 750     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 751                      process=True, force_generic_extractor=False):
 752         '''
 753         Returns a list with a dictionary for each video we find.
 754         If 'download', also downloads the videos.
 755         extra_info is a dict containing the extra values to add to each result
 756         '''
 757
 758         if not ie_key and force_generic_extractor:
 759             ie_key = 'Generic'
 760
 761         if ie_key:
 762             ies = [self.get_info_extractor(ie_key)]
 763         else:
 764             ies = self._ies
 765
 766         for ie in ies:
 767             if not ie.suitable(url):
 768                 continue
 769
 770             ie = self.get_info_extractor(ie.ie_key())
 771             if not ie.working():
 772                 self.report_warning('The program functionality for this site has been marked as broken, '
 773                                     'and will probably not work.')
 774
 775             try:
 776                 ie_result = ie.extract(url)
 777                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 778                     break
 779                 if isinstance(ie_result, list):
 780                     # Backwards compatibility: old IE result format
 781                     ie_result = {
 782                         '_type': 'compat_list',
 783                         'entries': ie_result,
 784                     }
 785                 self.add_default_extra_info(ie_result, ie, url)
 786                 if process:
 787                     return self.process_ie_result(ie_result, download, extra_info)
 788                 else:
 789                     return ie_result
 790             except GeoRestrictedError as e:
 791                 msg = e.msg
 792                 if e.countries:
 793                     msg += '\nThis video is available in %s.' % ', '.join(
 794                         map(ISO3166Utils.short2full, e.countries))
 795                 msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
 796                 self.report_error(msg)
 797                 break
 798             except ExtractorError as e:  # An error we somewhat expected
 799                 self.report_error(compat_str(e), e.format_traceback())
 800                 break
 801             except MaxDownloadsReached:
 802                 raise
 803             except Exception as e:
 804                 if self.params.get('ignoreerrors', False):
 805                     self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
 806                     break
 807                 else:
 808                     raise
 809         else:
 810             self.report_error('no suitable InfoExtractor for URL %s' % url)
 811
 812     def add_default_extra_info(self, ie_result, ie, url):
 813         self.add_extra_info(ie_result, {
 814             'extractor': ie.IE_NAME,
 815             'webpage_url': url,
 816             'webpage_url_basename': url_basename(url),
 817             'extractor_key': ie.ie_key(),
 818         })
 819
 820     def process_ie_result(self, ie_result, download=True, extra_info={}):
 821         """
 822         Take the result of the ie(may be modified) and resolve all unresolved
 823         references (URLs, playlist items).
 824
 825         It will also download the videos if 'download'.
 826         Returns the resolved ie_result.
 827         """
 828         result_type = ie_result.get('_type', 'video')
 829
 830         if result_type in ('url', 'url_transparent'):
 831             ie_result['url'] = sanitize_url(ie_result['url'])
 832             extract_flat = self.params.get('extract_flat', False)
 833             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 834                     extract_flat is True):
 835                 if self.params.get('forcejson', False):
 836                     self.to_stdout(json.dumps(ie_result))
 837                 return ie_result
 838
 839         if result_type == 'video':
 840             self.add_extra_info(ie_result, extra_info)
 841             return self.process_video_result(ie_result, download=download)
 842         elif result_type == 'url':
 843             # We have to add extra_info to the results because it may be
 844             # contained in a playlist
 845             return self.extract_info(ie_result['url'],
 846                                      download,
 847                                      ie_key=ie_result.get('ie_key'),
 848                                      extra_info=extra_info)
 849         elif result_type == 'url_transparent':
 850             # Use the information from the embedding page
 851             info = self.extract_info(
 852                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 853                 extra_info=extra_info, download=False, process=False)
 854
 855             # extract_info may return None when ignoreerrors is enabled and
 856             # extraction failed with an error, don't crash and return early
 857             # in this case
 858             if not info:
 859                 return info
 860
 861             force_properties = dict(
 862                 (k, v) for k, v in ie_result.items() if v is not None)
 863             for f in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
 864                 if f in force_properties:
 865                     del force_properties[f]
 866             new_result = info.copy()
 867             new_result.update(force_properties)
 868
 869             # Extracted info may not be a video result (i.e.
 870             # info.get('_type', 'video') != video) but rather an url or
 871             # url_transparent. In such cases outer metadata (from ie_result)
 872             # should be propagated to inner one (info). For this to happen
 873             # _type of info should be overridden with url_transparent. This
 874             # fixes issue from https://github.com/rg3/youtube-dl/pull/11163.
 875             if new_result.get('_type') == 'url':
 876                 new_result['_type'] = 'url_transparent'
 877
 878             return self.process_ie_result(
 879                 new_result, download=download, extra_info=extra_info)
 880         elif result_type in ('playlist', 'multi_video'):
 881             # We process each entry in the playlist
 882             playlist = ie_result.get('title') or ie_result.get('id')
 883             self.to_screen('[download] Downloading playlist: %s' % playlist)
 884
 885             playlist_results = []
 886
 887             playliststart = self.params.get('playliststart', 1) - 1
 888             playlistend = self.params.get('playlistend')
 889             # For backwards compatibility, interpret -1 as whole list
 890             if playlistend == -1:
 891                 playlistend = None
 892
 893             playlistitems_str = self.params.get('playlist_items')
 894             playlistitems = None
 895             if playlistitems_str is not None:
 896                 def iter_playlistitems(format):
 897                     for string_segment in format.split(','):
 898                         if '-' in string_segment:
 899                             start, end = string_segment.split('-')
 900                             for item in range(int(start), int(end) + 1):
 901                                 yield int(item)
 902                         else:
 903                             yield int(string_segment)
 904                 playlistitems = iter_playlistitems(playlistitems_str)
 905
 906             ie_entries = ie_result['entries']
 907             if isinstance(ie_entries, list):
 908                 n_all_entries = len(ie_entries)
 909                 if playlistitems:
 910                     entries = [
 911                         ie_entries[i - 1] for i in playlistitems
 912                         if -n_all_entries <= i - 1 < n_all_entries]
 913                 else:
 914                     entries = ie_entries[playliststart:playlistend]
 915                 n_entries = len(entries)
 916                 self.to_screen(
 917                     '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
 918                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 919             elif isinstance(ie_entries, PagedList):
 920                 if playlistitems:
 921                     entries = []
 922                     for item in playlistitems:
 923                         entries.extend(ie_entries.getslice(
 924                             item - 1, item
 925                         ))
 926                 else:
 927                     entries = ie_entries.getslice(
 928                         playliststart, playlistend)
 929                 n_entries = len(entries)
 930                 self.to_screen(
 931                     '[%s] playlist %s: Downloading %d videos' %
 932                     (ie_result['extractor'], playlist, n_entries))
 933             else:  # iterable
 934                 if playlistitems:
 935                     entry_list = list(ie_entries)
 936                     entries = [entry_list[i - 1] for i in playlistitems]
 937                 else:
 938                     entries = list(itertools.islice(
 939                         ie_entries, playliststart, playlistend))
 940                 n_entries = len(entries)
 941                 self.to_screen(
 942                     '[%s] playlist %s: Downloading %d videos' %
 943                     (ie_result['extractor'], playlist, n_entries))
 944
 945             if self.params.get('playlistreverse', False):
 946                 entries = entries[::-1]
 947
 948             if self.params.get('playlistrandom', False):
 949                 random.shuffle(entries)
 950
 951             x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
 952
 953             for i, entry in enumerate(entries, 1):
 954                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 955                 # This __x_forwarded_for_ip thing is a bit ugly but requires
 956                 # minimal changes
 957                 if x_forwarded_for:
 958                     entry['__x_forwarded_for_ip'] = x_forwarded_for
 959                 extra = {
 960                     'n_entries': n_entries,
 961                     'playlist': playlist,
 962                     'playlist_id': ie_result.get('id'),
 963                     'playlist_title': ie_result.get('title'),
 964                     'playlist_index': i + playliststart,
 965                     'extractor': ie_result['extractor'],
 966                     'webpage_url': ie_result['webpage_url'],
 967                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 968                     'extractor_key': ie_result['extractor_key'],
 969                 }
 970
 971                 reason = self._match_entry(entry, incomplete=True)
 972                 if reason is not None:
 973                     self.to_screen('[download] ' + reason)
 974                     continue
 975
 976                 entry_result = self.process_ie_result(entry,
 977                                                       download=download,
 978                                                       extra_info=extra)
 979                 playlist_results.append(entry_result)
 980             ie_result['entries'] = playlist_results
 981             self.to_screen('[download] Finished downloading playlist: %s' % playlist)
 982             return ie_result
 983         elif result_type == 'compat_list':
 984             self.report_warning(
 985                 'Extractor %s returned a compat_list result. '
 986                 'It needs to be updated.' % ie_result.get('extractor'))
 987
 988             def _fixup(r):
 989                 self.add_extra_info(
 990                     r,
 991                     {
 992                         'extractor': ie_result['extractor'],
 993                         'webpage_url': ie_result['webpage_url'],
 994                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 995                         'extractor_key': ie_result['extractor_key'],
 996                     }
 997                 )
 998                 return r
 999             ie_result['entries'] = [
1000                 self.process_ie_result(_fixup(r), download, extra_info)
1001                 for r in ie_result['entries']
1002             ]
1003             return ie_result
1004         else:
1005             raise Exception('Invalid result type: %s' % result_type)
1006
1007     def _build_format_filter(self, filter_spec):
1008         " Returns a function to filter the formats according to the filter_spec "
1009
1010         OPERATORS = {
1011             '<': operator.lt,
1012             '<=': operator.le,
1013             '>': operator.gt,
1014             '>=': operator.ge,
1015             '=': operator.eq,
1016             '!=': operator.ne,
1017         }
1018         operator_rex = re.compile(r'''(?x)\s*
1019             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
1020             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1021             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
1022             $
1023             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
1024         m = operator_rex.search(filter_spec)
1025         if m:
1026             try:
1027                 comparison_value = int(m.group('value'))
1028             except ValueError:
1029                 comparison_value = parse_filesize(m.group('value'))
1030                 if comparison_value is None:
1031                     comparison_value = parse_filesize(m.group('value') + 'B')
1032                 if comparison_value is None:
1033                     raise ValueError(
1034                         'Invalid value %r in format specification %r' % (
1035                             m.group('value'), filter_spec))
1036             op = OPERATORS[m.group('op')]
1037
1038         if not m:
1039             STR_OPERATORS = {
1040                 '=': operator.eq,
1041                 '!=': operator.ne,
1042                 '^=': lambda attr, value: attr.startswith(value),
1043                 '$=': lambda attr, value: attr.endswith(value),
1044                 '*=': lambda attr, value: value in attr,
1045             }
1046             str_operator_rex = re.compile(r'''(?x)
1047                 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
1048                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
1049                 \s*(?P<value>[a-zA-Z0-9._-]+)
1050                 \s*$
1051                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
1052             m = str_operator_rex.search(filter_spec)
1053             if m:
1054                 comparison_value = m.group('value')
1055                 op = STR_OPERATORS[m.group('op')]
1056
1057         if not m:
1058             raise ValueError('Invalid filter specification %r' % filter_spec)
1059
1060         def _filter(f):
1061             actual_value = f.get(m.group('key'))
1062             if actual_value is None:
1063                 return m.group('none_inclusive')
1064             return op(actual_value, comparison_value)
1065         return _filter
1066
1067     def _default_format_spec(self, info_dict, download=True):
1068         req_format_list = []
1069
1070         def can_have_partial_formats():
1071             if self.params.get('simulate', False):
1072                 return True
1073             if not download:
1074                 return True
1075             if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
1076                 return False
1077             if info_dict.get('is_live'):
1078                 return False
1079             merger = FFmpegMergerPP(self)
1080             return merger.available and merger.can_merge()
1081         if can_have_partial_formats():
1082             req_format_list.append('bestvideo+bestaudio')
1083         req_format_list.append('best')
1084         return '/'.join(req_format_list)
1085
1086     def build_format_selector(self, format_spec):
1087         def syntax_error(note, start):
1088             message = (
1089                 'Invalid format specification: '
1090                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
1091             return SyntaxError(message)
1092
1093         PICKFIRST = 'PICKFIRST'
1094         MERGE = 'MERGE'
1095         SINGLE = 'SINGLE'
1096         GROUP = 'GROUP'
1097         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1098
1099         def _parse_filter(tokens):
1100             filter_parts = []
1101             for type, string, start, _, _ in tokens:
1102                 if type == tokenize.OP and string == ']':
1103                     return ''.join(filter_parts)
1104                 else:
1105                     filter_parts.append(string)
1106
1107         def _remove_unused_ops(tokens):
1108             # Remove operators that we don't use and join them with the surrounding strings
1109             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1110             ALLOWED_OPS = ('/', '+', ',', '(', ')')
1111             last_string, last_start, last_end, last_line = None, None, None, None
1112             for type, string, start, end, line in tokens:
1113                 if type == tokenize.OP and string == '[':
1114                     if last_string:
1115                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1116                         last_string = None
1117                     yield type, string, start, end, line
1118                     # everything inside brackets will be handled by _parse_filter
1119                     for type, string, start, end, line in tokens:
1120                         yield type, string, start, end, line
1121                         if type == tokenize.OP and string == ']':
1122                             break
1123                 elif type == tokenize.OP and string in ALLOWED_OPS:
1124                     if last_string:
1125                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1126                         last_string = None
1127                     yield type, string, start, end, line
1128                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
1129                     if not last_string:
1130                         last_string = string
1131                         last_start = start
1132                         last_end = end
1133                     else:
1134                         last_string += string
1135             if last_string:
1136                 yield tokenize.NAME, last_string, last_start, last_end, last_line
1137
1138         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
1139             selectors = []
1140             current_selector = None
1141             for type, string, start, _, _ in tokens:
1142                 # ENCODING is only defined in python 3.x
1143                 if type == getattr(tokenize, 'ENCODING', None):
1144                     continue
1145                 elif type in [tokenize.NAME, tokenize.NUMBER]:
1146                     current_selector = FormatSelector(SINGLE, string, [])
1147                 elif type == tokenize.OP:
1148                     if string == ')':
1149                         if not inside_group:
1150                             # ')' will be handled by the parentheses group
1151                             tokens.restore_last_token()
1152                         break
1153                     elif inside_merge and string in ['/', ',']:
1154                         tokens.restore_last_token()
1155                         break
1156                     elif inside_choice and string == ',':
1157                         tokens.restore_last_token()
1158                         break
1159                     elif string == ',':
1160                         if not current_selector:
1161                             raise syntax_error('"," must follow a format selector', start)
1162                         selectors.append(current_selector)
1163                         current_selector = None
1164                     elif string == '/':
1165                         if not current_selector:
1166                             raise syntax_error('"/" must follow a format selector', start)
1167                         first_choice = current_selector
1168                         second_choice = _parse_format_selection(tokens, inside_choice=True)
1169                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1170                     elif string == '[':
1171                         if not current_selector:
1172                             current_selector = FormatSelector(SINGLE, 'best', [])
1173                         format_filter = _parse_filter(tokens)
1174                         current_selector.filters.append(format_filter)
1175                     elif string == '(':
1176                         if current_selector:
1177                             raise syntax_error('Unexpected "("', start)
1178                         group = _parse_format_selection(tokens, inside_group=True)
1179                         current_selector = FormatSelector(GROUP, group, [])
1180                     elif string == '+':
1181                         video_selector = current_selector
1182                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
1183                         if not video_selector or not audio_selector:
1184                             raise syntax_error('"+" must be between two format selectors', start)
1185                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1186                     else:
1187                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1188                 elif type == tokenize.ENDMARKER:
1189                     break
1190             if current_selector:
1191                 selectors.append(current_selector)
1192             return selectors
1193
1194         def _build_selector_function(selector):
1195             if isinstance(selector, list):
1196                 fs = [_build_selector_function(s) for s in selector]
1197
1198                 def selector_function(ctx):
1199                     for f in fs:
1200                         for format in f(ctx):
1201                             yield format
1202                 return selector_function
1203             elif selector.type == GROUP:
1204                 selector_function = _build_selector_function(selector.selector)
1205             elif selector.type == PICKFIRST:
1206                 fs = [_build_selector_function(s) for s in selector.selector]
1207
1208                 def selector_function(ctx):
1209                     for f in fs:
1210                         picked_formats = list(f(ctx))
1211                         if picked_formats:
1212                             return picked_formats
1213                     return []
1214             elif selector.type == SINGLE:
1215                 format_spec = selector.selector
1216
1217                 def selector_function(ctx):
1218                     formats = list(ctx['formats'])
1219                     if not formats:
1220                         return
1221                     if format_spec == 'all':
1222                         for f in formats:
1223                             yield f
1224                     elif format_spec in ['best', 'worst', None]:
1225                         format_idx = 0 if format_spec == 'worst' else -1
1226                         audiovideo_formats = [
1227                             f for f in formats
1228                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1229                         if audiovideo_formats:
1230                             yield audiovideo_formats[format_idx]
1231                         # for extractors with incomplete formats (audio only (soundcloud)
1232                         # or video only (imgur)) we will fallback to best/worst
1233                         # {video,audio}-only format
1234                         elif ctx['incomplete_formats']:
1235                             yield formats[format_idx]
1236                     elif format_spec == 'bestaudio':
1237                         audio_formats = [
1238                             f for f in formats
1239                             if f.get('vcodec') == 'none']
1240                         if audio_formats:
1241                             yield audio_formats[-1]
1242                     elif format_spec == 'worstaudio':
1243                         audio_formats = [
1244                             f for f in formats
1245                             if f.get('vcodec') == 'none']
1246                         if audio_formats:
1247                             yield audio_formats[0]
1248                     elif format_spec == 'bestvideo':
1249                         video_formats = [
1250                             f for f in formats
1251                             if f.get('acodec') == 'none']
1252                         if video_formats:
1253                             yield video_formats[-1]
1254                     elif format_spec == 'worstvideo':
1255                         video_formats = [
1256                             f for f in formats
1257                             if f.get('acodec') == 'none']
1258                         if video_formats:
1259                             yield video_formats[0]
1260                     else:
1261                         extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1262                         if format_spec in extensions:
1263                             filter_f = lambda f: f['ext'] == format_spec
1264                         else:
1265                             filter_f = lambda f: f['format_id'] == format_spec
1266                         matches = list(filter(filter_f, formats))
1267                         if matches:
1268                             yield matches[-1]
1269             elif selector.type == MERGE:
1270                 def _merge(formats_info):
1271                     format_1, format_2 = [f['format_id'] for f in formats_info]
1272                     # The first format must contain the video and the
1273                     # second the audio
1274                     if formats_info[0].get('vcodec') == 'none':
1275                         self.report_error('The first format must '
1276                                           'contain the video, try using '
1277                                           '"-f %s+%s"' % (format_2, format_1))
1278                         return
1279                     # Formats must be opposite (video+audio)
1280                     if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1281                         self.report_error(
1282                             'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1283                             % (format_1, format_2))
1284                         return
1285                     output_ext = (
1286                         formats_info[0]['ext']
1287                         if self.params.get('merge_output_format') is None
1288                         else self.params['merge_output_format'])
1289                     return {
1290                         'requested_formats': formats_info,
1291                         'format': '%s+%s' % (formats_info[0].get('format'),
1292                                              formats_info[1].get('format')),
1293                         'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1294                                                 formats_info[1].get('format_id')),
1295                         'width': formats_info[0].get('width'),
1296                         'height': formats_info[0].get('height'),
1297                         'resolution': formats_info[0].get('resolution'),
1298                         'fps': formats_info[0].get('fps'),
1299                         'vcodec': formats_info[0].get('vcodec'),
1300                         'vbr': formats_info[0].get('vbr'),
1301                         'stretched_ratio': formats_info[0].get('stretched_ratio'),
1302                         'acodec': formats_info[1].get('acodec'),
1303                         'abr': formats_info[1].get('abr'),
1304                         'ext': output_ext,
1305                     }
1306                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1307
1308                 def selector_function(ctx):
1309                     for pair in itertools.product(
1310                             video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
1311                         yield _merge(pair)
1312
1313             filters = [self._build_format_filter(f) for f in selector.filters]
1314
1315             def final_selector(ctx):
1316                 ctx_copy = copy.deepcopy(ctx)
1317                 for _filter in filters:
1318                     ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
1319                 return selector_function(ctx_copy)
1320             return final_selector
1321
1322         stream = io.BytesIO(format_spec.encode('utf-8'))
1323         try:
1324             tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1325         except tokenize.TokenError:
1326             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1327
1328         class TokenIterator(object):
1329             def __init__(self, tokens):
1330                 self.tokens = tokens
1331                 self.counter = 0
1332
1333             def __iter__(self):
1334                 return self
1335
1336             def __next__(self):
1337                 if self.counter >= len(self.tokens):
1338                     raise StopIteration()
1339                 value = self.tokens[self.counter]
1340                 self.counter += 1
1341                 return value
1342
1343             next = __next__
1344
1345             def restore_last_token(self):
1346                 self.counter -= 1
1347
1348         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1349         return _build_selector_function(parsed_selector)
1350
1351     def _calc_headers(self, info_dict):
1352         res = std_headers.copy()
1353
1354         add_headers = info_dict.get('http_headers')
1355         if add_headers:
1356             res.update(add_headers)
1357
1358         cookies = self._calc_cookies(info_dict)
1359         if cookies:
1360             res['Cookie'] = cookies
1361
1362         if 'X-Forwarded-For' not in res:
1363             x_forwarded_for_ip = info_dict.get('__x_forwarded_for_ip')
1364             if x_forwarded_for_ip:
1365                 res['X-Forwarded-For'] = x_forwarded_for_ip
1366
1367         return res
1368
1369     def _calc_cookies(self, info_dict):
1370         pr = sanitized_Request(info_dict['url'])
1371         self.cookiejar.add_cookie_header(pr)
1372         return pr.get_header('Cookie')
1373
1374     def process_video_result(self, info_dict, download=True):
1375         assert info_dict.get('_type', 'video') == 'video'
1376
1377         if 'id' not in info_dict:
1378             raise ExtractorError('Missing "id" field in extractor result')
1379         if 'title' not in info_dict:
1380             raise ExtractorError('Missing "title" field in extractor result')
1381
1382         def report_force_conversion(field, field_not, conversion):
1383             self.report_warning(
1384                 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1385                 % (field, field_not, conversion))
1386
1387         def sanitize_string_field(info, string_field):
1388             field = info.get(string_field)
1389             if field is None or isinstance(field, compat_str):
1390                 return
1391             report_force_conversion(string_field, 'a string', 'string')
1392             info[string_field] = compat_str(field)
1393
1394         def sanitize_numeric_fields(info):
1395             for numeric_field in self._NUMERIC_FIELDS:
1396                 field = info.get(numeric_field)
1397                 if field is None or isinstance(field, compat_numeric_types):
1398                     continue
1399                 report_force_conversion(numeric_field, 'numeric', 'int')
1400                 info[numeric_field] = int_or_none(field)
1401
1402         sanitize_string_field(info_dict, 'id')
1403         sanitize_numeric_fields(info_dict)
1404
1405         if 'playlist' not in info_dict:
1406             # It isn't part of a playlist
1407             info_dict['playlist'] = None
1408             info_dict['playlist_index'] = None
1409
1410         thumbnails = info_dict.get('thumbnails')
1411         if thumbnails is None:
1412             thumbnail = info_dict.get('thumbnail')
1413             if thumbnail:
1414                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1415         if thumbnails:
1416             thumbnails.sort(key=lambda t: (
1417                 t.get('preference') if t.get('preference') is not None else -1,
1418                 t.get('width') if t.get('width') is not None else -1,
1419                 t.get('height') if t.get('height') is not None else -1,
1420                 t.get('id') if t.get('id') is not None else '', t.get('url')))
1421             for i, t in enumerate(thumbnails):
1422                 t['url'] = sanitize_url(t['url'])
1423                 if t.get('width') and t.get('height'):
1424                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1425                 if t.get('id') is None:
1426                     t['id'] = '%d' % i
1427
1428         if self.params.get('list_thumbnails'):
1429             self.list_thumbnails(info_dict)
1430             return
1431
1432         thumbnail = info_dict.get('thumbnail')
1433         if thumbnail:
1434             info_dict['thumbnail'] = sanitize_url(thumbnail)
1435         elif thumbnails:
1436             info_dict['thumbnail'] = thumbnails[-1]['url']
1437
1438         if 'display_id' not in info_dict and 'id' in info_dict:
1439             info_dict['display_id'] = info_dict['id']
1440
1441         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1442             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1443             # see http://bugs.python.org/issue1646728)
1444             try:
1445                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1446                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1447             except (ValueError, OverflowError, OSError):
1448                 pass
1449
1450         # Auto generate title fields corresponding to the *_number fields when missing
1451         # in order to always have clean titles. This is very common for TV series.
1452         for field in ('chapter', 'season', 'episode'):
1453             if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1454                 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1455
1456         subtitles = info_dict.get('subtitles')
1457         if subtitles:
1458             for _, subtitle in subtitles.items():
1459                 for subtitle_format in subtitle:
1460                     if subtitle_format.get('url'):
1461                         subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1462                     if subtitle_format.get('ext') is None:
1463                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1464
1465         if self.params.get('listsubtitles', False):
1466             if 'automatic_captions' in info_dict:
1467                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1468             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1469             return
1470         info_dict['requested_subtitles'] = self.process_subtitles(
1471             info_dict['id'], subtitles,
1472             info_dict.get('automatic_captions'))
1473
1474         # We now pick which formats have to be downloaded
1475         if info_dict.get('formats') is None:
1476             # There's only one format available
1477             formats = [info_dict]
1478         else:
1479             formats = info_dict['formats']
1480
1481         if not formats:
1482             raise ExtractorError('No video formats found!')
1483
1484         def is_wellformed(f):
1485             url = f.get('url')
1486             valid_url = url and isinstance(url, compat_str)
1487             if not valid_url:
1488                 self.report_warning(
1489                     '"url" field is missing or empty - skipping format, '
1490                     'there is an error in extractor')
1491             return valid_url
1492
1493         # Filter out malformed formats for better extraction robustness
1494         formats = list(filter(is_wellformed, formats))
1495
1496         formats_dict = {}
1497
1498         # We check that all the formats have the format and format_id fields
1499         for i, format in enumerate(formats):
1500             sanitize_string_field(format, 'format_id')
1501             sanitize_numeric_fields(format)
1502             format['url'] = sanitize_url(format['url'])
1503             if format.get('format_id') is None:
1504                 format['format_id'] = compat_str(i)
1505             else:
1506                 # Sanitize format_id from characters used in format selector expression
1507                 format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
1508             format_id = format['format_id']
1509             if format_id not in formats_dict:
1510                 formats_dict[format_id] = []
1511             formats_dict[format_id].append(format)
1512
1513         # Make sure all formats have unique format_id
1514         for format_id, ambiguous_formats in formats_dict.items():
1515             if len(ambiguous_formats) > 1:
1516                 for i, format in enumerate(ambiguous_formats):
1517                     format['format_id'] = '%s-%d' % (format_id, i)
1518
1519         for i, format in enumerate(formats):
1520             if format.get('format') is None:
1521                 format['format'] = '{id} - {res}{note}'.format(
1522                     id=format['format_id'],
1523                     res=self.format_resolution(format),
1524                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1525                 )
1526             # Automatically determine file extension if missing
1527             if format.get('ext') is None:
1528                 format['ext'] = determine_ext(format['url']).lower()
1529             # Automatically determine protocol if missing (useful for format
1530             # selection purposes)
1531             if format.get('protocol') is None:
1532                 format['protocol'] = determine_protocol(format)
1533             # Add HTTP headers, so that external programs can use them from the
1534             # json output
1535             full_format_info = info_dict.copy()
1536             full_format_info.update(format)
1537             format['http_headers'] = self._calc_headers(full_format_info)
1538         # Remove private housekeeping stuff
1539         if '__x_forwarded_for_ip' in info_dict:
1540             del info_dict['__x_forwarded_for_ip']
1541
1542         # TODO Central sorting goes here
1543
1544         if formats[0] is not info_dict:
1545             # only set the 'formats' fields if the original info_dict list them
1546             # otherwise we end up with a circular reference, the first (and unique)
1547             # element in the 'formats' field in info_dict is info_dict itself,
1548             # which can't be exported to json
1549             info_dict['formats'] = formats
1550         if self.params.get('listformats'):
1551             self.list_formats(info_dict)
1552             return
1553
1554         req_format = self.params.get('format')
1555         if req_format is None:
1556             req_format = self._default_format_spec(info_dict, download=download)
1557             if self.params.get('verbose'):
1558                 self.to_stdout('[debug] Default format spec: %s' % req_format)
1559
1560         format_selector = self.build_format_selector(req_format)
1561
1562         # While in format selection we may need to have an access to the original
1563         # format set in order to calculate some metrics or do some processing.
1564         # For now we need to be able to guess whether original formats provided
1565         # by extractor are incomplete or not (i.e. whether extractor provides only
1566         # video-only or audio-only formats) for proper formats selection for
1567         # extractors with such incomplete formats (see
1568         # https://github.com/rg3/youtube-dl/pull/5556).
1569         # Since formats may be filtered during format selection and may not match
1570         # the original formats the results may be incorrect. Thus original formats
1571         # or pre-calculated metrics should be passed to format selection routines
1572         # as well.
1573         # We will pass a context object containing all necessary additional data
1574         # instead of just formats.
1575         # This fixes incorrect format selection issue (see
1576         # https://github.com/rg3/youtube-dl/issues/10083).
1577         incomplete_formats = (
1578             # All formats are video-only or
1579             all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
1580             # all formats are audio-only
1581             all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
1582
1583         ctx = {
1584             'formats': formats,
1585             'incomplete_formats': incomplete_formats,
1586         }
1587
1588         formats_to_download = list(format_selector(ctx))
1589         if not formats_to_download:
1590             raise ExtractorError('requested format not available',
1591                                  expected=True)
1592
1593         if download:
1594             if len(formats_to_download) > 1:
1595                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1596             for format in formats_to_download:
1597                 new_info = dict(info_dict)
1598                 new_info.update(format)
1599                 self.process_info(new_info)
1600         # We update the info dict with the best quality format (backwards compatibility)
1601         info_dict.update(formats_to_download[-1])
1602         return info_dict
1603
1604     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1605         """Select the requested subtitles and their format"""
1606         available_subs = {}
1607         if normal_subtitles and self.params.get('writesubtitles'):
1608             available_subs.update(normal_subtitles)
1609         if automatic_captions and self.params.get('writeautomaticsub'):
1610             for lang, cap_info in automatic_captions.items():
1611                 if lang not in available_subs:
1612                     available_subs[lang] = cap_info
1613
1614         if (not self.params.get('writesubtitles') and not
1615                 self.params.get('writeautomaticsub') or not
1616                 available_subs):
1617             return None
1618
1619         if self.params.get('allsubtitles', False):
1620             requested_langs = available_subs.keys()
1621         else:
1622             if self.params.get('subtitleslangs', False):
1623                 requested_langs = self.params.get('subtitleslangs')
1624             elif 'en' in available_subs:
1625                 requested_langs = ['en']
1626             else:
1627                 requested_langs = [list(available_subs.keys())[0]]
1628
1629         formats_query = self.params.get('subtitlesformat', 'best')
1630         formats_preference = formats_query.split('/') if formats_query else []
1631         subs = {}
1632         for lang in requested_langs:
1633             formats = available_subs.get(lang)
1634             if formats is None:
1635                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1636                 continue
1637             for ext in formats_preference:
1638                 if ext == 'best':
1639                     f = formats[-1]
1640                     break
1641                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1642                 if matches:
1643                     f = matches[-1]
1644                     break
1645             else:
1646                 f = formats[-1]
1647                 self.report_warning(
1648                     'No subtitle format found matching "%s" for language %s, '
1649                     'using %s' % (formats_query, lang, f['ext']))
1650             subs[lang] = f
1651         return subs
1652
1653     def process_info(self, info_dict):
1654         """Process a single resolved IE result."""
1655
1656         assert info_dict.get('_type', 'video') == 'video'
1657
1658         max_downloads = self.params.get('max_downloads')
1659         if max_downloads is not None:
1660             if self._num_downloads >= int(max_downloads):
1661                 raise MaxDownloadsReached()
1662
1663         info_dict['fulltitle'] = info_dict['title']
1664         if len(info_dict['title']) > 200:
1665             info_dict['title'] = info_dict['title'][:197] + '...'
1666
1667         if 'format' not in info_dict:
1668             info_dict['format'] = info_dict['ext']
1669
1670         reason = self._match_entry(info_dict, incomplete=False)
1671         if reason is not None:
1672             self.to_screen('[download] ' + reason)
1673             return
1674
1675         self._num_downloads += 1
1676
1677         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1678
1679         # Forced printings
1680         if self.params.get('forcetitle', False):
1681             self.to_stdout(info_dict['fulltitle'])
1682         if self.params.get('forceid', False):
1683             self.to_stdout(info_dict['id'])
1684         if self.params.get('forceurl', False):
1685             if info_dict.get('requested_formats') is not None:
1686                 for f in info_dict['requested_formats']:
1687                     self.to_stdout(f['url'] + f.get('play_path', ''))
1688             else:
1689                 # For RTMP URLs, also include the playpath
1690                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1691         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1692             self.to_stdout(info_dict['thumbnail'])
1693         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1694             self.to_stdout(info_dict['description'])
1695         if self.params.get('forcefilename', False) and filename is not None:
1696             self.to_stdout(filename)
1697         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1698             self.to_stdout(formatSeconds(info_dict['duration']))
1699         if self.params.get('forceformat', False):
1700             self.to_stdout(info_dict['format'])
1701         if self.params.get('forcejson', False):
1702             self.to_stdout(json.dumps(info_dict))
1703
1704         # Do nothing else if in simulate mode
1705         if self.params.get('simulate', False):
1706             return
1707
1708         if filename is None:
1709             return
1710
1711         try:
1712             dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1713             if dn and not os.path.exists(dn):
1714                 os.makedirs(dn)
1715         except (OSError, IOError) as err:
1716             self.report_error('unable to create directory ' + error_to_compat_str(err))
1717             return
1718
1719         if self.params.get('writedescription', False):
1720             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1721             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1722                 self.to_screen('[info] Video description is already present')
1723             elif info_dict.get('description') is None:
1724                 self.report_warning('There\'s no description to write.')
1725             else:
1726                 try:
1727                     self.to_screen('[info] Writing video description to: ' + descfn)
1728                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1729                         descfile.write(info_dict['description'])
1730                 except (OSError, IOError):
1731                     self.report_error('Cannot write description file ' + descfn)
1732                     return
1733
1734         if self.params.get('writeannotations', False):
1735             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1736             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1737                 self.to_screen('[info] Video annotations are already present')
1738             else:
1739                 try:
1740                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1741                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1742                         annofile.write(info_dict['annotations'])
1743                 except (KeyError, TypeError):
1744                     self.report_warning('There are no annotations to write.')
1745                 except (OSError, IOError):
1746                     self.report_error('Cannot write annotations file: ' + annofn)
1747                     return
1748
1749         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1750                                        self.params.get('writeautomaticsub')])
1751
1752         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1753             # subtitles download errors are already managed as troubles in relevant IE
1754             # that way it will silently go on when used with unsupporting IE
1755             subtitles = info_dict['requested_subtitles']
1756             ie = self.get_info_extractor(info_dict['extractor_key'])
1757             for sub_lang, sub_info in subtitles.items():
1758                 sub_format = sub_info['ext']
1759                 if sub_info.get('data') is not None:
1760                     sub_data = sub_info['data']
1761                 else:
1762                     try:
1763                         sub_data = ie._download_webpage(
1764                             sub_info['url'], info_dict['id'], note=False)
1765                     except ExtractorError as err:
1766                         self.report_warning('Unable to download subtitle for "%s": %s' %
1767                                             (sub_lang, error_to_compat_str(err.cause)))
1768                         continue
1769                 try:
1770                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1771                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1772                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1773                     else:
1774                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1775                         # Use newline='' to prevent conversion of newline characters
1776                         # See https://github.com/rg3/youtube-dl/issues/10268
1777                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
1778                             subfile.write(sub_data)
1779                 except (OSError, IOError):
1780                     self.report_error('Cannot write subtitles file ' + sub_filename)
1781                     return
1782
1783         if self.params.get('writeinfojson', False):
1784             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1785             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1786                 self.to_screen('[info] Video description metadata is already present')
1787             else:
1788                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1789                 try:
1790                     write_json_file(self.filter_requested_info(info_dict), infofn)
1791                 except (OSError, IOError):
1792                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1793                     return
1794
1795         self._write_thumbnails(info_dict, filename)
1796
1797         if not self.params.get('skip_download', False):
1798             try:
1799                 def dl(name, info):
1800                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1801                     for ph in self._progress_hooks:
1802                         fd.add_progress_hook(ph)
1803                     if self.params.get('verbose'):
1804                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1805                     return fd.download(name, info)
1806
1807                 if info_dict.get('requested_formats') is not None:
1808                     downloaded = []
1809                     success = True
1810                     merger = FFmpegMergerPP(self)
1811                     if not merger.available:
1812                         postprocessors = []
1813                         self.report_warning('You have requested multiple '
1814                                             'formats but ffmpeg or avconv are not installed.'
1815                                             ' The formats won\'t be merged.')
1816                     else:
1817                         postprocessors = [merger]
1818
1819                     def compatible_formats(formats):
1820                         video, audio = formats
1821                         # Check extension
1822                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1823                         if video_ext and audio_ext:
1824                             COMPATIBLE_EXTS = (
1825                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'),
1826                                 ('webm')
1827                             )
1828                             for exts in COMPATIBLE_EXTS:
1829                                 if video_ext in exts and audio_ext in exts:
1830                                     return True
1831                         # TODO: Check acodec/vcodec
1832                         return False
1833
1834                     filename_real_ext = os.path.splitext(filename)[1][1:]
1835                     filename_wo_ext = (
1836                         os.path.splitext(filename)[0]
1837                         if filename_real_ext == info_dict['ext']
1838                         else filename)
1839                     requested_formats = info_dict['requested_formats']
1840                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1841                         info_dict['ext'] = 'mkv'
1842                         self.report_warning(
1843                             'Requested formats are incompatible for merge and will be merged into mkv.')
1844                     # Ensure filename always has a correct extension for successful merge
1845                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1846                     if os.path.exists(encodeFilename(filename)):
1847                         self.to_screen(
1848                             '[download] %s has already been downloaded and '
1849                             'merged' % filename)
1850                     else:
1851                         for f in requested_formats:
1852                             new_info = dict(info_dict)
1853                             new_info.update(f)
1854                             fname = self.prepare_filename(new_info)
1855                             fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1856                             downloaded.append(fname)
1857                             partial_success = dl(fname, new_info)
1858                             success = success and partial_success
1859                         info_dict['__postprocessors'] = postprocessors
1860                         info_dict['__files_to_merge'] = downloaded
1861                 else:
1862                     # Just a single file
1863                     success = dl(filename, info_dict)
1864             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1865                 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1866                 return
1867             except (OSError, IOError) as err:
1868                 raise UnavailableVideoError(err)
1869             except (ContentTooShortError, ) as err:
1870                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1871                 return
1872
1873             if success and filename != '-':
1874                 # Fixup content
1875                 fixup_policy = self.params.get('fixup')
1876                 if fixup_policy is None:
1877                     fixup_policy = 'detect_or_warn'
1878
1879                 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1880
1881                 stretched_ratio = info_dict.get('stretched_ratio')
1882                 if stretched_ratio is not None and stretched_ratio != 1:
1883                     if fixup_policy == 'warn':
1884                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1885                             info_dict['id'], stretched_ratio))
1886                     elif fixup_policy == 'detect_or_warn':
1887                         stretched_pp = FFmpegFixupStretchedPP(self)
1888                         if stretched_pp.available:
1889                             info_dict.setdefault('__postprocessors', [])
1890                             info_dict['__postprocessors'].append(stretched_pp)
1891                         else:
1892                             self.report_warning(
1893                                 '%s: Non-uniform pixel ratio (%s). %s'
1894                                 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1895                     else:
1896                         assert fixup_policy in ('ignore', 'never')
1897
1898                 if (info_dict.get('requested_formats') is None and
1899                         info_dict.get('container') == 'm4a_dash'):
1900                     if fixup_policy == 'warn':
1901                         self.report_warning(
1902                             '%s: writing DASH m4a. '
1903                             'Only some players support this container.'
1904                             % info_dict['id'])
1905                     elif fixup_policy == 'detect_or_warn':
1906                         fixup_pp = FFmpegFixupM4aPP(self)
1907                         if fixup_pp.available:
1908                             info_dict.setdefault('__postprocessors', [])
1909                             info_dict['__postprocessors'].append(fixup_pp)
1910                         else:
1911                             self.report_warning(
1912                                 '%s: writing DASH m4a. '
1913                                 'Only some players support this container. %s'
1914                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1915                     else:
1916                         assert fixup_policy in ('ignore', 'never')
1917
1918                 if (info_dict.get('protocol') == 'm3u8_native' or
1919                         info_dict.get('protocol') == 'm3u8' and
1920                         self.params.get('hls_prefer_native')):
1921                     if fixup_policy == 'warn':
1922                         self.report_warning('%s: malformed AAC bitstream detected.' % (
1923                             info_dict['id']))
1924                     elif fixup_policy == 'detect_or_warn':
1925                         fixup_pp = FFmpegFixupM3u8PP(self)
1926                         if fixup_pp.available:
1927                             info_dict.setdefault('__postprocessors', [])
1928                             info_dict['__postprocessors'].append(fixup_pp)
1929                         else:
1930                             self.report_warning(
1931                                 '%s: malformed AAC bitstream detected. %s'
1932                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1933                     else:
1934                         assert fixup_policy in ('ignore', 'never')
1935
1936                 try:
1937                     self.post_process(filename, info_dict)
1938                 except (PostProcessingError) as err:
1939                     self.report_error('postprocessing: %s' % str(err))
1940                     return
1941                 self.record_download_archive(info_dict)
1942
1943     def download(self, url_list):
1944         """Download a given list of URLs."""
1945         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1946         if (len(url_list) > 1 and
1947                 outtmpl != '-' and
1948                 '%' not in outtmpl and
1949                 self.params.get('max_downloads') != 1):
1950             raise SameFileError(outtmpl)
1951
1952         for url in url_list:
1953             try:
1954                 # It also downloads the videos
1955                 res = self.extract_info(
1956                     url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1957             except UnavailableVideoError:
1958                 self.report_error('unable to download video')
1959             except MaxDownloadsReached:
1960                 self.to_screen('[info] Maximum number of downloaded files reached.')
1961                 raise
1962             else:
1963                 if self.params.get('dump_single_json', False):
1964                     self.to_stdout(json.dumps(res))
1965
1966         return self._download_retcode
1967
1968     def download_with_info_file(self, info_filename):
1969         with contextlib.closing(fileinput.FileInput(
1970                 [info_filename], mode='r',
1971                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1972             # FileInput doesn't have a read method, we can't call json.load
1973             info = self.filter_requested_info(json.loads('\n'.join(f)))
1974         try:
1975             self.process_ie_result(info, download=True)
1976         except DownloadError:
1977             webpage_url = info.get('webpage_url')
1978             if webpage_url is not None:
1979                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1980                 return self.download([webpage_url])
1981             else:
1982                 raise
1983         return self._download_retcode
1984
1985     @staticmethod
1986     def filter_requested_info(info_dict):
1987         return dict(
1988             (k, v) for k, v in info_dict.items()
1989             if k not in ['requested_formats', 'requested_subtitles'])
1990
1991     def post_process(self, filename, ie_info):
1992         """Run all the postprocessors on the given file."""
1993         info = dict(ie_info)
1994         info['filepath'] = filename
1995         pps_chain = []
1996         if ie_info.get('__postprocessors') is not None:
1997             pps_chain.extend(ie_info['__postprocessors'])
1998         pps_chain.extend(self._pps)
1999         for pp in pps_chain:
2000             files_to_delete = []
2001             try:
2002                 files_to_delete, info = pp.run(info)
2003             except PostProcessingError as e:
2004                 self.report_error(e.msg)
2005             if files_to_delete and not self.params.get('keepvideo', False):
2006                 for old_filename in files_to_delete:
2007                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
2008                     try:
2009                         os.remove(encodeFilename(old_filename))
2010                     except (IOError, OSError):
2011                         self.report_warning('Unable to remove downloaded original file')
2012
2013     def _make_archive_id(self, info_dict):
2014         # Future-proof against any change in case
2015         # and backwards compatibility with prior versions
2016         extractor = info_dict.get('extractor_key')
2017         if extractor is None:
2018             if 'id' in info_dict:
2019                 extractor = info_dict.get('ie_key')  # key in a playlist
2020         if extractor is None:
2021             return None  # Incomplete video information
2022         return extractor.lower() + ' ' + info_dict['id']
2023
2024     def in_download_archive(self, info_dict):
2025         fn = self.params.get('download_archive')
2026         if fn is None:
2027             return False
2028
2029         vid_id = self._make_archive_id(info_dict)
2030         if vid_id is None:
2031             return False  # Incomplete video information
2032
2033         try:
2034             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
2035                 for line in archive_file:
2036                     if line.strip() == vid_id:
2037                         return True
2038         except IOError as ioe:
2039             if ioe.errno != errno.ENOENT:
2040                 raise
2041         return False
2042
2043     def record_download_archive(self, info_dict):
2044         fn = self.params.get('download_archive')
2045         if fn is None:
2046             return
2047         vid_id = self._make_archive_id(info_dict)
2048         assert vid_id
2049         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
2050             archive_file.write(vid_id + '\n')
2051
2052     @staticmethod
2053     def format_resolution(format, default='unknown'):
2054         if format.get('vcodec') == 'none':
2055             return 'audio only'
2056         if format.get('resolution') is not None:
2057             return format['resolution']
2058         if format.get('height') is not None:
2059             if format.get('width') is not None:
2060                 res = '%sx%s' % (format['width'], format['height'])
2061             else:
2062                 res = '%sp' % format['height']
2063         elif format.get('width') is not None:
2064             res = '%dx?' % format['width']
2065         else:
2066             res = default
2067         return res
2068
2069     def _format_note(self, fdict):
2070         res = ''
2071         if fdict.get('ext') in ['f4f', 'f4m']:
2072             res += '(unsupported) '
2073         if fdict.get('language'):
2074             if res:
2075                 res += ' '
2076             res += '[%s] ' % fdict['language']
2077         if fdict.get('format_note') is not None:
2078             res += fdict['format_note'] + ' '
2079         if fdict.get('tbr') is not None:
2080             res += '%4dk ' % fdict['tbr']
2081         if fdict.get('container') is not None:
2082             if res:
2083                 res += ', '
2084             res += '%s container' % fdict['container']
2085         if (fdict.get('vcodec') is not None and
2086                 fdict.get('vcodec') != 'none'):
2087             if res:
2088                 res += ', '
2089             res += fdict['vcodec']
2090             if fdict.get('vbr') is not None:
2091                 res += '@'
2092         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
2093             res += 'video@'
2094         if fdict.get('vbr') is not None:
2095             res += '%4dk' % fdict['vbr']
2096         if fdict.get('fps') is not None:
2097             if res:
2098                 res += ', '
2099             res += '%sfps' % fdict['fps']
2100         if fdict.get('acodec') is not None:
2101             if res:
2102                 res += ', '
2103             if fdict['acodec'] == 'none':
2104                 res += 'video only'
2105             else:
2106                 res += '%-5s' % fdict['acodec']
2107         elif fdict.get('abr') is not None:
2108             if res:
2109                 res += ', '
2110             res += 'audio'
2111         if fdict.get('abr') is not None:
2112             res += '@%3dk' % fdict['abr']
2113         if fdict.get('asr') is not None:
2114             res += ' (%5dHz)' % fdict['asr']
2115         if fdict.get('filesize') is not None:
2116             if res:
2117                 res += ', '
2118             res += format_bytes(fdict['filesize'])
2119         elif fdict.get('filesize_approx') is not None:
2120             if res:
2121                 res += ', '
2122             res += '~' + format_bytes(fdict['filesize_approx'])
2123         return res
2124
2125     def list_formats(self, info_dict):
2126         formats = info_dict.get('formats', [info_dict])
2127         table = [
2128             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
2129             for f in formats
2130             if f.get('preference') is None or f['preference'] >= -1000]
2131         if len(formats) > 1:
2132             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
2133
2134         header_line = ['format code', 'extension', 'resolution', 'note']
2135         self.to_screen(
2136             '[info] Available formats for %s:\n%s' %
2137             (info_dict['id'], render_table(header_line, table)))
2138
2139     def list_thumbnails(self, info_dict):
2140         thumbnails = info_dict.get('thumbnails')
2141         if not thumbnails:
2142             self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
2143             return
2144
2145         self.to_screen(
2146             '[info] Thumbnails for %s:' % info_dict['id'])
2147         self.to_screen(render_table(
2148             ['ID', 'width', 'height', 'URL'],
2149             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
2150
2151     def list_subtitles(self, video_id, subtitles, name='subtitles'):
2152         if not subtitles:
2153             self.to_screen('%s has no %s' % (video_id, name))
2154             return
2155         self.to_screen(
2156             'Available %s for %s:' % (name, video_id))
2157         self.to_screen(render_table(
2158             ['Language', 'formats'],
2159             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
2160                 for lang, formats in subtitles.items()]))
2161
2162     def urlopen(self, req):
2163         """ Start an HTTP download """
2164         if isinstance(req, compat_basestring):
2165             req = sanitized_Request(req)
2166         return self._opener.open(req, timeout=self._socket_timeout)
2167
2168     def print_debug_header(self):
2169         if not self.params.get('verbose'):
2170             return
2171
2172         if type('') is not compat_str:
2173             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
2174             self.report_warning(
2175                 'Your Python is broken! Update to a newer and supported version')
2176
2177         stdout_encoding = getattr(
2178             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
2179         encoding_str = (
2180             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2181                 locale.getpreferredencoding(),
2182                 sys.getfilesystemencoding(),
2183                 stdout_encoding,
2184                 self.get_encoding()))
2185         write_string(encoding_str, encoding=None)
2186
2187         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
2188         if _LAZY_LOADER:
2189             self._write_string('[debug] Lazy loading extractors enabled' + '\n')
2190         try:
2191             sp = subprocess.Popen(
2192                 ['git', 'rev-parse', '--short', 'HEAD'],
2193                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2194                 cwd=os.path.dirname(os.path.abspath(__file__)))
2195             out, err = sp.communicate()
2196             out = out.decode().strip()
2197             if re.match('[0-9a-f]+', out):
2198                 self._write_string('[debug] Git HEAD: ' + out + '\n')
2199         except Exception:
2200             try:
2201                 sys.exc_clear()
2202             except Exception:
2203                 pass
2204         self._write_string('[debug] Python version %s - %s\n' % (
2205             platform.python_version(), platform_name()))
2206
2207         exe_versions = FFmpegPostProcessor.get_versions(self)
2208         exe_versions['rtmpdump'] = rtmpdump_version()
2209         exe_str = ', '.join(
2210             '%s %s' % (exe, v)
2211             for exe, v in sorted(exe_versions.items())
2212             if v
2213         )
2214         if not exe_str:
2215             exe_str = 'none'
2216         self._write_string('[debug] exe versions: %s\n' % exe_str)
2217
2218         proxy_map = {}
2219         for handler in self._opener.handlers:
2220             if hasattr(handler, 'proxies'):
2221                 proxy_map.update(handler.proxies)
2222         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2223
2224         if self.params.get('call_home', False):
2225             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2226             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2227             latest_version = self.urlopen(
2228                 'https://yt-dl.org/latest/version').read().decode('utf-8')
2229             if version_tuple(latest_version) > version_tuple(__version__):
2230                 self.report_warning(
2231                     'You are using an outdated version (newest version: %s)! '
2232                     'See https://yt-dl.org/update if you need help updating.' %
2233                     latest_version)
2234
2235     def _setup_opener(self):
2236         timeout_val = self.params.get('socket_timeout')
2237         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2238
2239         opts_cookiefile = self.params.get('cookiefile')
2240         opts_proxy = self.params.get('proxy')
2241
2242         if opts_cookiefile is None:
2243             self.cookiejar = compat_cookiejar.CookieJar()
2244         else:
2245             opts_cookiefile = expand_path(opts_cookiefile)
2246             self.cookiejar = compat_cookiejar.MozillaCookieJar(
2247                 opts_cookiefile)
2248             if os.access(opts_cookiefile, os.R_OK):
2249                 self.cookiejar.load()
2250
2251         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2252         if opts_proxy is not None:
2253             if opts_proxy == '':
2254                 proxies = {}
2255             else:
2256                 proxies = {'http': opts_proxy, 'https': opts_proxy}
2257         else:
2258             proxies = compat_urllib_request.getproxies()
2259             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2260             if 'http' in proxies and 'https' not in proxies:
2261                 proxies['https'] = proxies['http']
2262         proxy_handler = PerRequestProxyHandler(proxies)
2263
2264         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2265         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2266         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2267         data_handler = compat_urllib_request_DataHandler()
2268
2269         # When passing our own FileHandler instance, build_opener won't add the
2270         # default FileHandler and allows us to disable the file protocol, which
2271         # can be used for malicious purposes (see
2272         # https://github.com/rg3/youtube-dl/issues/8227)
2273         file_handler = compat_urllib_request.FileHandler()
2274
2275         def file_open(*args, **kwargs):
2276             raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2277         file_handler.file_open = file_open
2278
2279         opener = compat_urllib_request.build_opener(
2280             proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2281
2282         # Delete the default user-agent header, which would otherwise apply in
2283         # cases where our custom HTTP handler doesn't come into play
2284         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2285         opener.addheaders = []
2286         self._opener = opener
2287
2288     def encode(self, s):
2289         if isinstance(s, bytes):
2290             return s  # Already encoded
2291
2292         try:
2293             return s.encode(self.get_encoding())
2294         except UnicodeEncodeError as err:
2295             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2296             raise
2297
2298     def get_encoding(self):
2299         encoding = self.params.get('encoding')
2300         if encoding is None:
2301             encoding = preferredencoding()
2302         return encoding
2303
2304     def _write_thumbnails(self, info_dict, filename):
2305         if self.params.get('writethumbnail', False):
2306             thumbnails = info_dict.get('thumbnails')
2307             if thumbnails:
2308                 thumbnails = [thumbnails[-1]]
2309         elif self.params.get('write_all_thumbnails', False):
2310             thumbnails = info_dict.get('thumbnails')
2311         else:
2312             return
2313
2314         if not thumbnails:
2315             # No thumbnails present, so return immediately
2316             return
2317
2318         for t in thumbnails:
2319             thumb_ext = determine_ext(t['url'], 'jpg')
2320             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2321             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2322             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2323
2324             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2325                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2326                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2327             else:
2328                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2329                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2330                 try:
2331                     uf = self.urlopen(t['url'])
2332                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2333                         shutil.copyfileobj(uf, thumbf)
2334                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2335                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2336                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2337                     self.report_warning('Unable to download thumbnail "%s": %s' %
2338                                         (t['url'], error_to_compat_str(err)))