_ Git - youtube-dl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # coding: utf-8
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import contextlib
   8 import copy
   9 import datetime
  10 import errno
  11 import fileinput
  12 import io
  13 import itertools
  14 import json
  15 import locale
  16 import operator
  17 import os
  18 import platform
  19 import re
  20 import shutil
  21 import subprocess
  22 import socket
  23 import sys
  24 import time
  25 import tokenize
  26 import traceback
  27 import random
  28
  29 from string import ascii_letters
  30
  31 from .compat import (
  32     compat_basestring,
  33     compat_cookiejar,
  34     compat_get_terminal_size,
  35     compat_http_client,
  36     compat_kwargs,
  37     compat_numeric_types,
  38     compat_os_name,
  39     compat_str,
  40     compat_tokenize_tokenize,
  41     compat_urllib_error,
  42     compat_urllib_request,
  43     compat_urllib_request_DataHandler,
  44 )
  45 from .utils import (
  46     age_restricted,
  47     args_to_str,
  48     ContentTooShortError,
  49     date_from_str,
  50     DateRange,
  51     DEFAULT_OUTTMPL,
  52     determine_ext,
  53     determine_protocol,
  54     DownloadError,
  55     encode_compat_str,
  56     encodeFilename,
  57     error_to_compat_str,
  58     expand_path,
  59     ExtractorError,
  60     format_bytes,
  61     formatSeconds,
  62     GeoRestrictedError,
  63     int_or_none,
  64     ISO3166Utils,
  65     locked_file,
  66     make_HTTPS_handler,
  67     MaxDownloadsReached,
  68     PagedList,
  69     parse_filesize,
  70     PerRequestProxyHandler,
  71     platform_name,
  72     PostProcessingError,
  73     preferredencoding,
  74     prepend_extension,
  75     register_socks_protocols,
  76     render_table,
  77     replace_extension,
  78     SameFileError,
  79     sanitize_filename,
  80     sanitize_path,
  81     sanitize_url,
  82     sanitized_Request,
  83     std_headers,
  84     subtitles_filename,
  85     UnavailableVideoError,
  86     url_basename,
  87     version_tuple,
  88     write_json_file,
  89     write_string,
  90     YoutubeDLCookieProcessor,
  91     YoutubeDLHandler,
  92 )
  93 from .cache import Cache
  94 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
  95 from .downloader import get_suitable_downloader
  96 from .downloader.rtmp import rtmpdump_version
  97 from .postprocessor import (
  98     FFmpegFixupM3u8PP,
  99     FFmpegFixupM4aPP,
 100     FFmpegFixupStretchedPP,
 101     FFmpegMergerPP,
 102     FFmpegPostProcessor,
 103     get_postprocessor,
 104 )
 105 from .version import __version__
 106
 107 if compat_os_name == 'nt':
 108     import ctypes
 109
 110
 111 class YoutubeDL(object):
 112     """YoutubeDL class.
 113
 114     YoutubeDL objects are the ones responsible of downloading the
 115     actual video file and writing it to disk if the user has requested
 116     it, among some other tasks. In most cases there should be one per
 117     program. As, given a video URL, the downloader doesn't know how to
 118     extract all the needed information, task that InfoExtractors do, it
 119     has to pass the URL to one of them.
 120
 121     For this, YoutubeDL objects have a method that allows
 122     InfoExtractors to be registered in a given order. When it is passed
 123     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 124     finds that reports being able to handle it. The InfoExtractor extracts
 125     all the information about the video or videos the URL refers to, and
 126     YoutubeDL process the extracted information, possibly using a File
 127     Downloader to download the video.
 128
 129     YoutubeDL objects accept a lot of parameters. In order not to saturate
 130     the object constructor with arguments, it receives a dictionary of
 131     options instead. These options are available through the params
 132     attribute for the InfoExtractors to use. The YoutubeDL also
 133     registers itself as the downloader in charge for the InfoExtractors
 134     that are added to it, so this is a "mutual registration".
 135
 136     Available options:
 137
 138     username:          Username for authentication purposes.
 139     password:          Password for authentication purposes.
 140     videopassword:     Password for accessing a video.
 141     ap_mso:            Adobe Pass multiple-system operator identifier.
 142     ap_username:       Multiple-system operator account username.
 143     ap_password:       Multiple-system operator account password.
 144     usenetrc:          Use netrc for authentication instead.
 145     verbose:           Print additional info to stdout.
 146     quiet:             Do not print messages to stdout.
 147     no_warnings:       Do not print out anything for warnings.
 148     forceurl:          Force printing final URL.
 149     forcetitle:        Force printing title.
 150     forceid:           Force printing ID.
 151     forcethumbnail:    Force printing thumbnail URL.
 152     forcedescription:  Force printing description.
 153     forcefilename:     Force printing final filename.
 154     forceduration:     Force printing duration.
 155     forcejson:         Force printing info_dict as JSON.
 156     dump_single_json:  Force printing the info_dict of the whole playlist
 157                        (or video) as a single JSON line.
 158     simulate:          Do not download the video files.
 159     format:            Video format code. See options.py for more information.
 160     outtmpl:           Template for output names.
 161     restrictfilenames: Do not allow "&" and spaces in file names
 162     ignoreerrors:      Do not stop on download errors.
 163     force_generic_extractor: Force downloader to use the generic extractor
 164     nooverwrites:      Prevent overwriting files.
 165     playliststart:     Playlist item to start at.
 166     playlistend:       Playlist item to end at.
 167     playlist_items:    Specific indices of playlist to download.
 168     playlistreverse:   Download playlist items in reverse order.
 169     playlistrandom:    Download playlist items in random order.
 170     matchtitle:        Download only matching titles.
 171     rejecttitle:       Reject downloads for matching titles.
 172     logger:            Log messages to a logging.Logger instance.
 173     logtostderr:       Log messages to stderr instead of stdout.
 174     writedescription:  Write the video description to a .description file
 175     writeinfojson:     Write the video description to a .info.json file
 176     writeannotations:  Write the video annotations to a .annotations.xml file
 177     writethumbnail:    Write the thumbnail image to a file
 178     write_all_thumbnails:  Write all thumbnail formats to files
 179     writesubtitles:    Write the video subtitles to a file
 180     writeautomaticsub: Write the automatically generated subtitles to a file
 181     allsubtitles:      Downloads all the subtitles of the video
 182                        (requires writesubtitles or writeautomaticsub)
 183     listsubtitles:     Lists all available subtitles for the video
 184     subtitlesformat:   The format code for subtitles
 185     subtitleslangs:    List of languages of the subtitles to download
 186     keepvideo:         Keep the video file after post-processing
 187     daterange:         A DateRange object, download only if the upload_date is in the range.
 188     skip_download:     Skip the actual download of the video file
 189     cachedir:          Location of the cache files in the filesystem.
 190                        False to disable filesystem cache.
 191     noplaylist:        Download single video instead of a playlist if in doubt.
 192     age_limit:         An integer representing the user's age in years.
 193                        Unsuitable videos for the given age are skipped.
 194     min_views:         An integer representing the minimum view count the video
 195                        must have in order to not be skipped.
 196                        Videos without view count information are always
 197                        downloaded. None for no limit.
 198     max_views:         An integer representing the maximum view count.
 199                        Videos that are more popular than that are not
 200                        downloaded.
 201                        Videos without view count information are always
 202                        downloaded. None for no limit.
 203     download_archive:  File name of a file where all downloads are recorded.
 204                        Videos already present in the file are not downloaded
 205                        again.
 206     cookiefile:        File name where cookies should be read from and dumped to.
 207     nocheckcertificate:Do not verify SSL certificates
 208     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 209                        At the moment, this is only supported by YouTube.
 210     proxy:             URL of the proxy server to use
 211     geo_verification_proxy:  URL of the proxy to use for IP address verification
 212                        on geo-restricted sites. (Experimental)
 213     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 214     bidi_workaround:   Work around buggy terminals without bidirectional text
 215                        support, using fridibi
 216     debug_printtraffic:Print out sent and received HTTP traffic
 217     include_ads:       Download ads as well
 218     default_search:    Prepend this string if an input url is not valid.
 219                        'auto' for elaborate guessing
 220     encoding:          Use this encoding instead of the system-specified.
 221     extract_flat:      Do not resolve URLs, return the immediate result.
 222                        Pass in 'in_playlist' to only show this behavior for
 223                        playlist items.
 224     postprocessors:    A list of dictionaries, each with an entry
 225                        * key:  The name of the postprocessor. See
 226                                youtube_dl/postprocessor/__init__.py for a list.
 227                        as well as any further keyword arguments for the
 228                        postprocessor.
 229     progress_hooks:    A list of functions that get called on download
 230                        progress, with a dictionary with the entries
 231                        * status: One of "downloading", "error", or "finished".
 232                                  Check this first and ignore unknown values.
 233
 234                        If status is one of "downloading", or "finished", the
 235                        following properties may also be present:
 236                        * filename: The final filename (always present)
 237                        * tmpfilename: The filename we're currently writing to
 238                        * downloaded_bytes: Bytes on disk
 239                        * total_bytes: Size of the whole file, None if unknown
 240                        * total_bytes_estimate: Guess of the eventual file size,
 241                                                None if unavailable.
 242                        * elapsed: The number of seconds since download started.
 243                        * eta: The estimated time in seconds, None if unknown
 244                        * speed: The download speed in bytes/second, None if
 245                                 unknown
 246                        * fragment_index: The counter of the currently
 247                                          downloaded video fragment.
 248                        * fragment_count: The number of fragments (= individual
 249                                          files that will be merged)
 250
 251                        Progress hooks are guaranteed to be called at least once
 252                        (with status "finished") if the download is successful.
 253     merge_output_format: Extension to use when merging formats.
 254     fixup:             Automatically correct known faults of the file.
 255                        One of:
 256                        - "never": do nothing
 257                        - "warn": only emit a warning
 258                        - "detect_or_warn": check whether we can do anything
 259                                            about it, warn otherwise (default)
 260     source_address:    (Experimental) Client-side IP address to bind to.
 261     call_home:         Boolean, true iff we are allowed to contact the
 262                        youtube-dl servers for debugging.
 263     sleep_interval:    Number of seconds to sleep before each download when
 264                        used alone or a lower bound of a range for randomized
 265                        sleep before each download (minimum possible number
 266                        of seconds to sleep) when used along with
 267                        max_sleep_interval.
 268     max_sleep_interval:Upper bound of a range for randomized sleep before each
 269                        download (maximum possible number of seconds to sleep).
 270                        Must only be used along with sleep_interval.
 271                        Actual sleep time will be a random float from range
 272                        [sleep_interval; max_sleep_interval].
 273     listformats:       Print an overview of available video formats and exit.
 274     list_thumbnails:   Print a table of all thumbnails and exit.
 275     match_filter:      A function that gets called with the info_dict of
 276                        every video.
 277                        If it returns a message, the video is ignored.
 278                        If it returns None, the video is downloaded.
 279                        match_filter_func in utils.py is one example for this.
 280     no_color:          Do not emit color codes in output.
 281     geo_bypass:        Bypass geographic restriction via faking X-Forwarded-For
 282                        HTTP header (experimental)
 283     geo_bypass_country:
 284                        Two-letter ISO 3166-2 country code that will be used for
 285                        explicit geographic restriction bypassing via faking
 286                        X-Forwarded-For HTTP header (experimental)
 287
 288     The following options determine which downloader is picked:
 289     external_downloader: Executable of the external downloader to call.
 290                        None or unset for standard (built-in) downloader.
 291     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
 292                        if True, otherwise use ffmpeg/avconv if False, otherwise
 293                        use downloader suggested by extractor if None.
 294
 295     The following parameters are not used by YoutubeDL itself, they are used by
 296     the downloader (see youtube_dl/downloader/common.py):
 297     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 298     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 299     xattr_set_filesize, external_downloader_args, hls_use_mpegts.
 300
 301     The following options are used by the post processors:
 302     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 303                        otherwise prefer avconv.
 304     postprocessor_args: A list of additional command-line arguments for the
 305                         postprocessor.
 306     """
 307
 308     _NUMERIC_FIELDS = set((
 309         'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
 310         'timestamp', 'upload_year', 'upload_month', 'upload_day',
 311         'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
 312         'average_rating', 'comment_count', 'age_limit',
 313         'start_time', 'end_time',
 314         'chapter_number', 'season_number', 'episode_number',
 315         'track_number', 'disc_number', 'release_year',
 316         'playlist_index',
 317     ))
 318
 319     params = None
 320     _ies = []
 321     _pps = []
 322     _download_retcode = None
 323     _num_downloads = None
 324     _screen_file = None
 325
 326     def __init__(self, params=None, auto_init=True):
 327         """Create a FileDownloader object with the given options."""
 328         if params is None:
 329             params = {}
 330         self._ies = []
 331         self._ies_instances = {}
 332         self._pps = []
 333         self._progress_hooks = []
 334         self._download_retcode = 0
 335         self._num_downloads = 0
 336         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 337         self._err_file = sys.stderr
 338         self.params = {
 339             # Default parameters
 340             'nocheckcertificate': False,
 341         }
 342         self.params.update(params)
 343         self.cache = Cache(self)
 344
 345         def check_deprecated(param, option, suggestion):
 346             if self.params.get(param) is not None:
 347                 self.report_warning(
 348                     '%s is deprecated. Use %s instead.' % (option, suggestion))
 349                 return True
 350             return False
 351
 352         if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
 353             if self.params.get('geo_verification_proxy') is None:
 354                 self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
 355
 356         check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
 357         check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
 358         check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
 359
 360         if params.get('bidi_workaround', False):
 361             try:
 362                 import pty
 363                 master, slave = pty.openpty()
 364                 width = compat_get_terminal_size().columns
 365                 if width is None:
 366                     width_args = []
 367                 else:
 368                     width_args = ['-w', str(width)]
 369                 sp_kwargs = dict(
 370                     stdin=subprocess.PIPE,
 371                     stdout=slave,
 372                     stderr=self._err_file)
 373                 try:
 374                     self._output_process = subprocess.Popen(
 375                         ['bidiv'] + width_args, **sp_kwargs
 376                     )
 377                 except OSError:
 378                     self._output_process = subprocess.Popen(
 379                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 380                 self._output_channel = os.fdopen(master, 'rb')
 381             except OSError as ose:
 382                 if ose.errno == errno.ENOENT:
 383                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 384                 else:
 385                     raise
 386
 387         if (sys.platform != 'win32' and
 388                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
 389                 not params.get('restrictfilenames', False)):
 390             # Unicode filesystem API will throw errors (#1474, #13027)
 391             self.report_warning(
 392                 'Assuming --restrict-filenames since file system encoding '
 393                 'cannot encode all characters. '
 394                 'Set the LC_ALL environment variable to fix this.')
 395             self.params['restrictfilenames'] = True
 396
 397         if isinstance(params.get('outtmpl'), bytes):
 398             self.report_warning(
 399                 'Parameter outtmpl is bytes, but should be a unicode string. '
 400                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 401
 402         self._setup_opener()
 403
 404         if auto_init:
 405             self.print_debug_header()
 406             self.add_default_info_extractors()
 407
 408         for pp_def_raw in self.params.get('postprocessors', []):
 409             pp_class = get_postprocessor(pp_def_raw['key'])
 410             pp_def = dict(pp_def_raw)
 411             del pp_def['key']
 412             pp = pp_class(self, **compat_kwargs(pp_def))
 413             self.add_post_processor(pp)
 414
 415         for ph in self.params.get('progress_hooks', []):
 416             self.add_progress_hook(ph)
 417
 418         register_socks_protocols()
 419
 420     def warn_if_short_id(self, argv):
 421         # short YouTube ID starting with dash?
 422         idxs = [
 423             i for i, a in enumerate(argv)
 424             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 425         if idxs:
 426             correct_argv = (
 427                 ['youtube-dl'] +
 428                 [a for i, a in enumerate(argv) if i not in idxs] +
 429                 ['--'] + [argv[i] for i in idxs]
 430             )
 431             self.report_warning(
 432                 'Long argument string detected. '
 433                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 434                 args_to_str(correct_argv))
 435
 436     def add_info_extractor(self, ie):
 437         """Add an InfoExtractor object to the end of the list."""
 438         self._ies.append(ie)
 439         if not isinstance(ie, type):
 440             self._ies_instances[ie.ie_key()] = ie
 441             ie.set_downloader(self)
 442
 443     def get_info_extractor(self, ie_key):
 444         """
 445         Get an instance of an IE with name ie_key, it will try to get one from
 446         the _ies list, if there's no instance it will create a new one and add
 447         it to the extractor list.
 448         """
 449         ie = self._ies_instances.get(ie_key)
 450         if ie is None:
 451             ie = get_info_extractor(ie_key)()
 452             self.add_info_extractor(ie)
 453         return ie
 454
 455     def add_default_info_extractors(self):
 456         """
 457         Add the InfoExtractors returned by gen_extractors to the end of the list
 458         """
 459         for ie in gen_extractor_classes():
 460             self.add_info_extractor(ie)
 461
 462     def add_post_processor(self, pp):
 463         """Add a PostProcessor object to the end of the chain."""
 464         self._pps.append(pp)
 465         pp.set_downloader(self)
 466
 467     def add_progress_hook(self, ph):
 468         """Add the progress hook (currently only for the file downloader)"""
 469         self._progress_hooks.append(ph)
 470
 471     def _bidi_workaround(self, message):
 472         if not hasattr(self, '_output_channel'):
 473             return message
 474
 475         assert hasattr(self, '_output_process')
 476         assert isinstance(message, compat_str)
 477         line_count = message.count('\n') + 1
 478         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 479         self._output_process.stdin.flush()
 480         res = ''.join(self._output_channel.readline().decode('utf-8')
 481                       for _ in range(line_count))
 482         return res[:-len('\n')]
 483
 484     def to_screen(self, message, skip_eol=False):
 485         """Print message to stdout if not in quiet mode."""
 486         return self.to_stdout(message, skip_eol, check_quiet=True)
 487
 488     def _write_string(self, s, out=None):
 489         write_string(s, out=out, encoding=self.params.get('encoding'))
 490
 491     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 492         """Print message to stdout if not in quiet mode."""
 493         if self.params.get('logger'):
 494             self.params['logger'].debug(message)
 495         elif not check_quiet or not self.params.get('quiet', False):
 496             message = self._bidi_workaround(message)
 497             terminator = ['\n', ''][skip_eol]
 498             output = message + terminator
 499
 500             self._write_string(output, self._screen_file)
 501
 502     def to_stderr(self, message):
 503         """Print message to stderr."""
 504         assert isinstance(message, compat_str)
 505         if self.params.get('logger'):
 506             self.params['logger'].error(message)
 507         else:
 508             message = self._bidi_workaround(message)
 509             output = message + '\n'
 510             self._write_string(output, self._err_file)
 511
 512     def to_console_title(self, message):
 513         if not self.params.get('consoletitle', False):
 514             return
 515         if compat_os_name == 'nt':
 516             if ctypes.windll.kernel32.GetConsoleWindow():
 517                 # c_wchar_p() might not be necessary if `message` is
 518                 # already of type unicode()
 519                 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 520         elif 'TERM' in os.environ:
 521             self._write_string('\033]0;%s\007' % message, self._screen_file)
 522
 523     def save_console_title(self):
 524         if not self.params.get('consoletitle', False):
 525             return
 526         if compat_os_name != 'nt' and 'TERM' in os.environ:
 527             # Save the title on stack
 528             self._write_string('\033[22;0t', self._screen_file)
 529
 530     def restore_console_title(self):
 531         if not self.params.get('consoletitle', False):
 532             return
 533         if compat_os_name != 'nt' and 'TERM' in os.environ:
 534             # Restore the title from stack
 535             self._write_string('\033[23;0t', self._screen_file)
 536
 537     def __enter__(self):
 538         self.save_console_title()
 539         return self
 540
 541     def __exit__(self, *args):
 542         self.restore_console_title()
 543
 544         if self.params.get('cookiefile') is not None:
 545             self.cookiejar.save()
 546
 547     def trouble(self, message=None, tb=None):
 548         """Determine action to take when a download problem appears.
 549
 550         Depending on if the downloader has been configured to ignore
 551         download errors or not, this method may throw an exception or
 552         not when errors are found, after printing the message.
 553
 554         tb, if given, is additional traceback information.
 555         """
 556         if message is not None:
 557             self.to_stderr(message)
 558         if self.params.get('verbose'):
 559             if tb is None:
 560                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 561                     tb = ''
 562                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 563                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 564                     tb += encode_compat_str(traceback.format_exc())
 565                 else:
 566                     tb_data = traceback.format_list(traceback.extract_stack())
 567                     tb = ''.join(tb_data)
 568             self.to_stderr(tb)
 569         if not self.params.get('ignoreerrors', False):
 570             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 571                 exc_info = sys.exc_info()[1].exc_info
 572             else:
 573                 exc_info = sys.exc_info()
 574             raise DownloadError(message, exc_info)
 575         self._download_retcode = 1
 576
 577     def report_warning(self, message):
 578         '''
 579         Print the message to stderr, it will be prefixed with 'WARNING:'
 580         If stderr is a tty file the 'WARNING:' will be colored
 581         '''
 582         if self.params.get('logger') is not None:
 583             self.params['logger'].warning(message)
 584         else:
 585             if self.params.get('no_warnings'):
 586                 return
 587             if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 588                 _msg_header = '\033[0;33mWARNING:\033[0m'
 589             else:
 590                 _msg_header = 'WARNING:'
 591             warning_message = '%s %s' % (_msg_header, message)
 592             self.to_stderr(warning_message)
 593
 594     def report_error(self, message, tb=None):
 595         '''
 596         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 597         in red if stderr is a tty file.
 598         '''
 599         if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 600             _msg_header = '\033[0;31mERROR:\033[0m'
 601         else:
 602             _msg_header = 'ERROR:'
 603         error_message = '%s %s' % (_msg_header, message)
 604         self.trouble(error_message, tb)
 605
 606     def report_file_already_downloaded(self, file_name):
 607         """Report file has already been fully downloaded."""
 608         try:
 609             self.to_screen('[download] %s has already been downloaded' % file_name)
 610         except UnicodeEncodeError:
 611             self.to_screen('[download] The file has already been downloaded')
 612
 613     def prepare_filename(self, info_dict):
 614         """Generate the output filename."""
 615         try:
 616             template_dict = dict(info_dict)
 617
 618             template_dict['epoch'] = int(time.time())
 619             autonumber_size = self.params.get('autonumber_size')
 620             if autonumber_size is None:
 621                 autonumber_size = 5
 622             template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
 623             if template_dict.get('resolution') is None:
 624                 if template_dict.get('width') and template_dict.get('height'):
 625                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 626                 elif template_dict.get('height'):
 627                     template_dict['resolution'] = '%sp' % template_dict['height']
 628                 elif template_dict.get('width'):
 629                     template_dict['resolution'] = '%dx?' % template_dict['width']
 630
 631             sanitize = lambda k, v: sanitize_filename(
 632                 compat_str(v),
 633                 restricted=self.params.get('restrictfilenames'),
 634                 is_id=(k == 'id' or k.endswith('_id')))
 635             template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
 636                                  for k, v in template_dict.items()
 637                                  if v is not None and not isinstance(v, (list, tuple, dict)))
 638             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 639
 640             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 641
 642             # For fields playlist_index and autonumber convert all occurrences
 643             # of %(field)s to %(field)0Nd for backward compatibility
 644             field_size_compat_map = {
 645                 'playlist_index': len(str(template_dict['n_entries'])),
 646                 'autonumber': autonumber_size,
 647             }
 648             FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
 649             mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl)
 650             if mobj:
 651                 outtmpl = re.sub(
 652                     FIELD_SIZE_COMPAT_RE,
 653                     r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
 654                     outtmpl)
 655
 656             # Missing numeric fields used together with integer presentation types
 657             # in format specification will break the argument substitution since
 658             # string 'NA' is returned for missing fields. We will patch output
 659             # template for missing fields to meet string presentation type.
 660             for numeric_field in self._NUMERIC_FIELDS:
 661                 if numeric_field not in template_dict:
 662                     # As of [1] format syntax is:
 663                     #  %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
 664                     # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
 665                     FORMAT_RE = r'''(?x)
 666                         (?<!%)
 667                         %
 668                         \({0}\)  # mapping key
 669                         (?:[#0\-+ ]+)?  # conversion flags (optional)
 670                         (?:\d+)?  # minimum field width (optional)
 671                         (?:\.\d+)?  # precision (optional)
 672                         [hlL]?  # length modifier (optional)
 673                         [diouxXeEfFgGcrs%]  # conversion type
 674                     '''
 675                     outtmpl = re.sub(
 676                         FORMAT_RE.format(numeric_field),
 677                         r'%({0})s'.format(numeric_field), outtmpl)
 678
 679             # expand_path translates '%%' into '%' and '$$' into '$'
 680             # correspondingly that is not what we want since we need to keep
 681             # '%%' intact for template dict substitution step. Working around
 682             # with boundary-alike separator hack.
 683             sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
 684             outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
 685
 686             # outtmpl should be expand_path'ed before template dict substitution
 687             # because meta fields may contain env variables we don't want to
 688             # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
 689             # title "Hello $PATH", we don't want `$PATH` to be expanded.
 690             filename = expand_path(outtmpl).replace(sep, '') % template_dict
 691
 692             # Temporary fix for #4787
 693             # 'Treat' all problem characters by passing filename through preferredencoding
 694             # to workaround encoding issues with subprocess on python2 @ Windows
 695             if sys.version_info < (3, 0) and sys.platform == 'win32':
 696                 filename = encodeFilename(filename, True).decode(preferredencoding())
 697             return sanitize_path(filename)
 698         except ValueError as err:
 699             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 700             return None
 701
 702     def _match_entry(self, info_dict, incomplete):
 703         """ Returns None iff the file should be downloaded """
 704
 705         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 706         if 'title' in info_dict:
 707             # This can happen when we're just evaluating the playlist
 708             title = info_dict['title']
 709             matchtitle = self.params.get('matchtitle', False)
 710             if matchtitle:
 711                 if not re.search(matchtitle, title, re.IGNORECASE):
 712                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 713             rejecttitle = self.params.get('rejecttitle', False)
 714             if rejecttitle:
 715                 if re.search(rejecttitle, title, re.IGNORECASE):
 716                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 717         date = info_dict.get('upload_date')
 718         if date is not None:
 719             dateRange = self.params.get('daterange', DateRange())
 720             if date not in dateRange:
 721                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 722         view_count = info_dict.get('view_count')
 723         if view_count is not None:
 724             min_views = self.params.get('min_views')
 725             if min_views is not None and view_count < min_views:
 726                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 727             max_views = self.params.get('max_views')
 728             if max_views is not None and view_count > max_views:
 729                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 730         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 731             return 'Skipping "%s" because it is age restricted' % video_title
 732         if self.in_download_archive(info_dict):
 733             return '%s has already been recorded in archive' % video_title
 734
 735         if not incomplete:
 736             match_filter = self.params.get('match_filter')
 737             if match_filter is not None:
 738                 ret = match_filter(info_dict)
 739                 if ret is not None:
 740                     return ret
 741
 742         return None
 743
 744     @staticmethod
 745     def add_extra_info(info_dict, extra_info):
 746         '''Set the keys from extra_info in info dict if they are missing'''
 747         for key, value in extra_info.items():
 748             info_dict.setdefault(key, value)
 749
 750     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 751                      process=True, force_generic_extractor=False):
 752         '''
 753         Returns a list with a dictionary for each video we find.
 754         If 'download', also downloads the videos.
 755         extra_info is a dict containing the extra values to add to each result
 756         '''
 757
 758         if not ie_key and force_generic_extractor:
 759             ie_key = 'Generic'
 760
 761         if ie_key:
 762             ies = [self.get_info_extractor(ie_key)]
 763         else:
 764             ies = self._ies
 765
 766         for ie in ies:
 767             if not ie.suitable(url):
 768                 continue
 769
 770             ie = self.get_info_extractor(ie.ie_key())
 771             if not ie.working():
 772                 self.report_warning('The program functionality for this site has been marked as broken, '
 773                                     'and will probably not work.')
 774
 775             try:
 776                 ie_result = ie.extract(url)
 777                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 778                     break
 779                 if isinstance(ie_result, list):
 780                     # Backwards compatibility: old IE result format
 781                     ie_result = {
 782                         '_type': 'compat_list',
 783                         'entries': ie_result,
 784                     }
 785                 self.add_default_extra_info(ie_result, ie, url)
 786                 if process:
 787                     return self.process_ie_result(ie_result, download, extra_info)
 788                 else:
 789                     return ie_result
 790             except GeoRestrictedError as e:
 791                 msg = e.msg
 792                 if e.countries:
 793                     msg += '\nThis video is available in %s.' % ', '.join(
 794                         map(ISO3166Utils.short2full, e.countries))
 795                 msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
 796                 self.report_error(msg)
 797                 break
 798             except ExtractorError as e:  # An error we somewhat expected
 799                 self.report_error(compat_str(e), e.format_traceback())
 800                 break
 801             except MaxDownloadsReached:
 802                 raise
 803             except Exception as e:
 804                 if self.params.get('ignoreerrors', False):
 805                     self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
 806                     break
 807                 else:
 808                     raise
 809         else:
 810             self.report_error('no suitable InfoExtractor for URL %s' % url)
 811
 812     def add_default_extra_info(self, ie_result, ie, url):
 813         self.add_extra_info(ie_result, {
 814             'extractor': ie.IE_NAME,
 815             'webpage_url': url,
 816             'webpage_url_basename': url_basename(url),
 817             'extractor_key': ie.ie_key(),
 818         })
 819
 820     def process_ie_result(self, ie_result, download=True, extra_info={}):
 821         """
 822         Take the result of the ie(may be modified) and resolve all unresolved
 823         references (URLs, playlist items).
 824
 825         It will also download the videos if 'download'.
 826         Returns the resolved ie_result.
 827         """
 828         result_type = ie_result.get('_type', 'video')
 829
 830         if result_type in ('url', 'url_transparent'):
 831             ie_result['url'] = sanitize_url(ie_result['url'])
 832             extract_flat = self.params.get('extract_flat', False)
 833             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 834                     extract_flat is True):
 835                 if self.params.get('forcejson', False):
 836                     self.to_stdout(json.dumps(ie_result))
 837                 return ie_result
 838
 839         if result_type == 'video':
 840             self.add_extra_info(ie_result, extra_info)
 841             return self.process_video_result(ie_result, download=download)
 842         elif result_type == 'url':
 843             # We have to add extra_info to the results because it may be
 844             # contained in a playlist
 845             return self.extract_info(ie_result['url'],
 846                                      download,
 847                                      ie_key=ie_result.get('ie_key'),
 848                                      extra_info=extra_info)
 849         elif result_type == 'url_transparent':
 850             # Use the information from the embedding page
 851             info = self.extract_info(
 852                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 853                 extra_info=extra_info, download=False, process=False)
 854
 855             # extract_info may return None when ignoreerrors is enabled and
 856             # extraction failed with an error, don't crash and return early
 857             # in this case
 858             if not info:
 859                 return info
 860
 861             force_properties = dict(
 862                 (k, v) for k, v in ie_result.items() if v is not None)
 863             for f in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
 864                 if f in force_properties:
 865                     del force_properties[f]
 866             new_result = info.copy()
 867             new_result.update(force_properties)
 868
 869             # Extracted info may not be a video result (i.e.
 870             # info.get('_type', 'video') != video) but rather an url or
 871             # url_transparent. In such cases outer metadata (from ie_result)
 872             # should be propagated to inner one (info). For this to happen
 873             # _type of info should be overridden with url_transparent. This
 874             # fixes issue from https://github.com/rg3/youtube-dl/pull/11163.
 875             if new_result.get('_type') == 'url':
 876                 new_result['_type'] = 'url_transparent'
 877
 878             return self.process_ie_result(
 879                 new_result, download=download, extra_info=extra_info)
 880         elif result_type in ('playlist', 'multi_video'):
 881             # We process each entry in the playlist
 882             playlist = ie_result.get('title') or ie_result.get('id')
 883             self.to_screen('[download] Downloading playlist: %s' % playlist)
 884
 885             playlist_results = []
 886
 887             playliststart = self.params.get('playliststart', 1) - 1
 888             playlistend = self.params.get('playlistend')
 889             # For backwards compatibility, interpret -1 as whole list
 890             if playlistend == -1:
 891                 playlistend = None
 892
 893             playlistitems_str = self.params.get('playlist_items')
 894             playlistitems = None
 895             if playlistitems_str is not None:
 896                 def iter_playlistitems(format):
 897                     for string_segment in format.split(','):
 898                         if '-' in string_segment:
 899                             start, end = string_segment.split('-')
 900                             for item in range(int(start), int(end) + 1):
 901                                 yield int(item)
 902                         else:
 903                             yield int(string_segment)
 904                 playlistitems = iter_playlistitems(playlistitems_str)
 905
 906             ie_entries = ie_result['entries']
 907             if isinstance(ie_entries, list):
 908                 n_all_entries = len(ie_entries)
 909                 if playlistitems:
 910                     entries = [
 911                         ie_entries[i - 1] for i in playlistitems
 912                         if -n_all_entries <= i - 1 < n_all_entries]
 913                 else:
 914                     entries = ie_entries[playliststart:playlistend]
 915                 n_entries = len(entries)
 916                 self.to_screen(
 917                     '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
 918                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 919             elif isinstance(ie_entries, PagedList):
 920                 if playlistitems:
 921                     entries = []
 922                     for item in playlistitems:
 923                         entries.extend(ie_entries.getslice(
 924                             item - 1, item
 925                         ))
 926                 else:
 927                     entries = ie_entries.getslice(
 928                         playliststart, playlistend)
 929                 n_entries = len(entries)
 930                 self.to_screen(
 931                     '[%s] playlist %s: Downloading %d videos' %
 932                     (ie_result['extractor'], playlist, n_entries))
 933             else:  # iterable
 934                 if playlistitems:
 935                     entry_list = list(ie_entries)
 936                     entries = [entry_list[i - 1] for i in playlistitems]
 937                 else:
 938                     entries = list(itertools.islice(
 939                         ie_entries, playliststart, playlistend))
 940                 n_entries = len(entries)
 941                 self.to_screen(
 942                     '[%s] playlist %s: Downloading %d videos' %
 943                     (ie_result['extractor'], playlist, n_entries))
 944
 945             if self.params.get('playlistreverse', False):
 946                 entries = entries[::-1]
 947
 948             if self.params.get('playlistrandom', False):
 949                 random.shuffle(entries)
 950
 951             x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
 952
 953             for i, entry in enumerate(entries, 1):
 954                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 955                 # This __x_forwarded_for_ip thing is a bit ugly but requires
 956                 # minimal changes
 957                 if x_forwarded_for:
 958                     entry['__x_forwarded_for_ip'] = x_forwarded_for
 959                 extra = {
 960                     'n_entries': n_entries,
 961                     'playlist': playlist,
 962                     'playlist_id': ie_result.get('id'),
 963                     'playlist_title': ie_result.get('title'),
 964                     'playlist_index': i + playliststart,
 965                     'extractor': ie_result['extractor'],
 966                     'webpage_url': ie_result['webpage_url'],
 967                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 968                     'extractor_key': ie_result['extractor_key'],
 969                 }
 970
 971                 reason = self._match_entry(entry, incomplete=True)
 972                 if reason is not None:
 973                     self.to_screen('[download] ' + reason)
 974                     continue
 975
 976                 entry_result = self.process_ie_result(entry,
 977                                                       download=download,
 978                                                       extra_info=extra)
 979                 playlist_results.append(entry_result)
 980             ie_result['entries'] = playlist_results
 981             self.to_screen('[download] Finished downloading playlist: %s' % playlist)
 982             return ie_result
 983         elif result_type == 'compat_list':
 984             self.report_warning(
 985                 'Extractor %s returned a compat_list result. '
 986                 'It needs to be updated.' % ie_result.get('extractor'))
 987
 988             def _fixup(r):
 989                 self.add_extra_info(
 990                     r,
 991                     {
 992                         'extractor': ie_result['extractor'],
 993                         'webpage_url': ie_result['webpage_url'],
 994                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 995                         'extractor_key': ie_result['extractor_key'],
 996                     }
 997                 )
 998                 return r
 999             ie_result['entries'] = [
1000                 self.process_ie_result(_fixup(r), download, extra_info)
1001                 for r in ie_result['entries']
1002             ]
1003             return ie_result
1004         else:
1005             raise Exception('Invalid result type: %s' % result_type)
1006
1007     def _build_format_filter(self, filter_spec):
1008         " Returns a function to filter the formats according to the filter_spec "
1009
1010         OPERATORS = {
1011             '<': operator.lt,
1012             '<=': operator.le,
1013             '>': operator.gt,
1014             '>=': operator.ge,
1015             '=': operator.eq,
1016             '!=': operator.ne,
1017         }
1018         operator_rex = re.compile(r'''(?x)\s*
1019             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
1020             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1021             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
1022             $
1023             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
1024         m = operator_rex.search(filter_spec)
1025         if m:
1026             try:
1027                 comparison_value = int(m.group('value'))
1028             except ValueError:
1029                 comparison_value = parse_filesize(m.group('value'))
1030                 if comparison_value is None:
1031                     comparison_value = parse_filesize(m.group('value') + 'B')
1032                 if comparison_value is None:
1033                     raise ValueError(
1034                         'Invalid value %r in format specification %r' % (
1035                             m.group('value'), filter_spec))
1036             op = OPERATORS[m.group('op')]
1037
1038         if not m:
1039             STR_OPERATORS = {
1040                 '=': operator.eq,
1041                 '!=': operator.ne,
1042                 '^=': lambda attr, value: attr.startswith(value),
1043                 '$=': lambda attr, value: attr.endswith(value),
1044                 '*=': lambda attr, value: value in attr,
1045             }
1046             str_operator_rex = re.compile(r'''(?x)
1047                 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
1048                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
1049                 \s*(?P<value>[a-zA-Z0-9._-]+)
1050                 \s*$
1051                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
1052             m = str_operator_rex.search(filter_spec)
1053             if m:
1054                 comparison_value = m.group('value')
1055                 op = STR_OPERATORS[m.group('op')]
1056
1057         if not m:
1058             raise ValueError('Invalid filter specification %r' % filter_spec)
1059
1060         def _filter(f):
1061             actual_value = f.get(m.group('key'))
1062             if actual_value is None:
1063                 return m.group('none_inclusive')
1064             return op(actual_value, comparison_value)
1065         return _filter
1066
1067     def build_format_selector(self, format_spec):
1068         def syntax_error(note, start):
1069             message = (
1070                 'Invalid format specification: '
1071                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
1072             return SyntaxError(message)
1073
1074         PICKFIRST = 'PICKFIRST'
1075         MERGE = 'MERGE'
1076         SINGLE = 'SINGLE'
1077         GROUP = 'GROUP'
1078         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1079
1080         def _parse_filter(tokens):
1081             filter_parts = []
1082             for type, string, start, _, _ in tokens:
1083                 if type == tokenize.OP and string == ']':
1084                     return ''.join(filter_parts)
1085                 else:
1086                     filter_parts.append(string)
1087
1088         def _remove_unused_ops(tokens):
1089             # Remove operators that we don't use and join them with the surrounding strings
1090             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1091             ALLOWED_OPS = ('/', '+', ',', '(', ')')
1092             last_string, last_start, last_end, last_line = None, None, None, None
1093             for type, string, start, end, line in tokens:
1094                 if type == tokenize.OP and string == '[':
1095                     if last_string:
1096                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1097                         last_string = None
1098                     yield type, string, start, end, line
1099                     # everything inside brackets will be handled by _parse_filter
1100                     for type, string, start, end, line in tokens:
1101                         yield type, string, start, end, line
1102                         if type == tokenize.OP and string == ']':
1103                             break
1104                 elif type == tokenize.OP and string in ALLOWED_OPS:
1105                     if last_string:
1106                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1107                         last_string = None
1108                     yield type, string, start, end, line
1109                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
1110                     if not last_string:
1111                         last_string = string
1112                         last_start = start
1113                         last_end = end
1114                     else:
1115                         last_string += string
1116             if last_string:
1117                 yield tokenize.NAME, last_string, last_start, last_end, last_line
1118
1119         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
1120             selectors = []
1121             current_selector = None
1122             for type, string, start, _, _ in tokens:
1123                 # ENCODING is only defined in python 3.x
1124                 if type == getattr(tokenize, 'ENCODING', None):
1125                     continue
1126                 elif type in [tokenize.NAME, tokenize.NUMBER]:
1127                     current_selector = FormatSelector(SINGLE, string, [])
1128                 elif type == tokenize.OP:
1129                     if string == ')':
1130                         if not inside_group:
1131                             # ')' will be handled by the parentheses group
1132                             tokens.restore_last_token()
1133                         break
1134                     elif inside_merge and string in ['/', ',']:
1135                         tokens.restore_last_token()
1136                         break
1137                     elif inside_choice and string == ',':
1138                         tokens.restore_last_token()
1139                         break
1140                     elif string == ',':
1141                         if not current_selector:
1142                             raise syntax_error('"," must follow a format selector', start)
1143                         selectors.append(current_selector)
1144                         current_selector = None
1145                     elif string == '/':
1146                         if not current_selector:
1147                             raise syntax_error('"/" must follow a format selector', start)
1148                         first_choice = current_selector
1149                         second_choice = _parse_format_selection(tokens, inside_choice=True)
1150                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1151                     elif string == '[':
1152                         if not current_selector:
1153                             current_selector = FormatSelector(SINGLE, 'best', [])
1154                         format_filter = _parse_filter(tokens)
1155                         current_selector.filters.append(format_filter)
1156                     elif string == '(':
1157                         if current_selector:
1158                             raise syntax_error('Unexpected "("', start)
1159                         group = _parse_format_selection(tokens, inside_group=True)
1160                         current_selector = FormatSelector(GROUP, group, [])
1161                     elif string == '+':
1162                         video_selector = current_selector
1163                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
1164                         if not video_selector or not audio_selector:
1165                             raise syntax_error('"+" must be between two format selectors', start)
1166                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1167                     else:
1168                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1169                 elif type == tokenize.ENDMARKER:
1170                     break
1171             if current_selector:
1172                 selectors.append(current_selector)
1173             return selectors
1174
1175         def _build_selector_function(selector):
1176             if isinstance(selector, list):
1177                 fs = [_build_selector_function(s) for s in selector]
1178
1179                 def selector_function(ctx):
1180                     for f in fs:
1181                         for format in f(ctx):
1182                             yield format
1183                 return selector_function
1184             elif selector.type == GROUP:
1185                 selector_function = _build_selector_function(selector.selector)
1186             elif selector.type == PICKFIRST:
1187                 fs = [_build_selector_function(s) for s in selector.selector]
1188
1189                 def selector_function(ctx):
1190                     for f in fs:
1191                         picked_formats = list(f(ctx))
1192                         if picked_formats:
1193                             return picked_formats
1194                     return []
1195             elif selector.type == SINGLE:
1196                 format_spec = selector.selector
1197
1198                 def selector_function(ctx):
1199                     formats = list(ctx['formats'])
1200                     if not formats:
1201                         return
1202                     if format_spec == 'all':
1203                         for f in formats:
1204                             yield f
1205                     elif format_spec in ['best', 'worst', None]:
1206                         format_idx = 0 if format_spec == 'worst' else -1
1207                         audiovideo_formats = [
1208                             f for f in formats
1209                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1210                         if audiovideo_formats:
1211                             yield audiovideo_formats[format_idx]
1212                         # for extractors with incomplete formats (audio only (soundcloud)
1213                         # or video only (imgur)) we will fallback to best/worst
1214                         # {video,audio}-only format
1215                         elif ctx['incomplete_formats']:
1216                             yield formats[format_idx]
1217                     elif format_spec == 'bestaudio':
1218                         audio_formats = [
1219                             f for f in formats
1220                             if f.get('vcodec') == 'none']
1221                         if audio_formats:
1222                             yield audio_formats[-1]
1223                     elif format_spec == 'worstaudio':
1224                         audio_formats = [
1225                             f for f in formats
1226                             if f.get('vcodec') == 'none']
1227                         if audio_formats:
1228                             yield audio_formats[0]
1229                     elif format_spec == 'bestvideo':
1230                         video_formats = [
1231                             f for f in formats
1232                             if f.get('acodec') == 'none']
1233                         if video_formats:
1234                             yield video_formats[-1]
1235                     elif format_spec == 'worstvideo':
1236                         video_formats = [
1237                             f for f in formats
1238                             if f.get('acodec') == 'none']
1239                         if video_formats:
1240                             yield video_formats[0]
1241                     else:
1242                         extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1243                         if format_spec in extensions:
1244                             filter_f = lambda f: f['ext'] == format_spec
1245                         else:
1246                             filter_f = lambda f: f['format_id'] == format_spec
1247                         matches = list(filter(filter_f, formats))
1248                         if matches:
1249                             yield matches[-1]
1250             elif selector.type == MERGE:
1251                 def _merge(formats_info):
1252                     format_1, format_2 = [f['format_id'] for f in formats_info]
1253                     # The first format must contain the video and the
1254                     # second the audio
1255                     if formats_info[0].get('vcodec') == 'none':
1256                         self.report_error('The first format must '
1257                                           'contain the video, try using '
1258                                           '"-f %s+%s"' % (format_2, format_1))
1259                         return
1260                     # Formats must be opposite (video+audio)
1261                     if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1262                         self.report_error(
1263                             'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1264                             % (format_1, format_2))
1265                         return
1266                     output_ext = (
1267                         formats_info[0]['ext']
1268                         if self.params.get('merge_output_format') is None
1269                         else self.params['merge_output_format'])
1270                     return {
1271                         'requested_formats': formats_info,
1272                         'format': '%s+%s' % (formats_info[0].get('format'),
1273                                              formats_info[1].get('format')),
1274                         'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1275                                                 formats_info[1].get('format_id')),
1276                         'width': formats_info[0].get('width'),
1277                         'height': formats_info[0].get('height'),
1278                         'resolution': formats_info[0].get('resolution'),
1279                         'fps': formats_info[0].get('fps'),
1280                         'vcodec': formats_info[0].get('vcodec'),
1281                         'vbr': formats_info[0].get('vbr'),
1282                         'stretched_ratio': formats_info[0].get('stretched_ratio'),
1283                         'acodec': formats_info[1].get('acodec'),
1284                         'abr': formats_info[1].get('abr'),
1285                         'ext': output_ext,
1286                     }
1287                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1288
1289                 def selector_function(ctx):
1290                     for pair in itertools.product(
1291                             video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
1292                         yield _merge(pair)
1293
1294             filters = [self._build_format_filter(f) for f in selector.filters]
1295
1296             def final_selector(ctx):
1297                 ctx_copy = copy.deepcopy(ctx)
1298                 for _filter in filters:
1299                     ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
1300                 return selector_function(ctx_copy)
1301             return final_selector
1302
1303         stream = io.BytesIO(format_spec.encode('utf-8'))
1304         try:
1305             tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1306         except tokenize.TokenError:
1307             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1308
1309         class TokenIterator(object):
1310             def __init__(self, tokens):
1311                 self.tokens = tokens
1312                 self.counter = 0
1313
1314             def __iter__(self):
1315                 return self
1316
1317             def __next__(self):
1318                 if self.counter >= len(self.tokens):
1319                     raise StopIteration()
1320                 value = self.tokens[self.counter]
1321                 self.counter += 1
1322                 return value
1323
1324             next = __next__
1325
1326             def restore_last_token(self):
1327                 self.counter -= 1
1328
1329         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1330         return _build_selector_function(parsed_selector)
1331
1332     def _calc_headers(self, info_dict):
1333         res = std_headers.copy()
1334
1335         add_headers = info_dict.get('http_headers')
1336         if add_headers:
1337             res.update(add_headers)
1338
1339         cookies = self._calc_cookies(info_dict)
1340         if cookies:
1341             res['Cookie'] = cookies
1342
1343         if 'X-Forwarded-For' not in res:
1344             x_forwarded_for_ip = info_dict.get('__x_forwarded_for_ip')
1345             if x_forwarded_for_ip:
1346                 res['X-Forwarded-For'] = x_forwarded_for_ip
1347
1348         return res
1349
1350     def _calc_cookies(self, info_dict):
1351         pr = sanitized_Request(info_dict['url'])
1352         self.cookiejar.add_cookie_header(pr)
1353         return pr.get_header('Cookie')
1354
1355     def process_video_result(self, info_dict, download=True):
1356         assert info_dict.get('_type', 'video') == 'video'
1357
1358         if 'id' not in info_dict:
1359             raise ExtractorError('Missing "id" field in extractor result')
1360         if 'title' not in info_dict:
1361             raise ExtractorError('Missing "title" field in extractor result')
1362
1363         def report_force_conversion(field, field_not, conversion):
1364             self.report_warning(
1365                 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1366                 % (field, field_not, conversion))
1367
1368         def sanitize_string_field(info, string_field):
1369             field = info.get(string_field)
1370             if field is None or isinstance(field, compat_str):
1371                 return
1372             report_force_conversion(string_field, 'a string', 'string')
1373             info[string_field] = compat_str(field)
1374
1375         def sanitize_numeric_fields(info):
1376             for numeric_field in self._NUMERIC_FIELDS:
1377                 field = info.get(numeric_field)
1378                 if field is None or isinstance(field, compat_numeric_types):
1379                     continue
1380                 report_force_conversion(numeric_field, 'numeric', 'int')
1381                 info[numeric_field] = int_or_none(field)
1382
1383         sanitize_string_field(info_dict, 'id')
1384         sanitize_numeric_fields(info_dict)
1385
1386         if 'playlist' not in info_dict:
1387             # It isn't part of a playlist
1388             info_dict['playlist'] = None
1389             info_dict['playlist_index'] = None
1390
1391         thumbnails = info_dict.get('thumbnails')
1392         if thumbnails is None:
1393             thumbnail = info_dict.get('thumbnail')
1394             if thumbnail:
1395                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1396         if thumbnails:
1397             thumbnails.sort(key=lambda t: (
1398                 t.get('preference') if t.get('preference') is not None else -1,
1399                 t.get('width') if t.get('width') is not None else -1,
1400                 t.get('height') if t.get('height') is not None else -1,
1401                 t.get('id') if t.get('id') is not None else '', t.get('url')))
1402             for i, t in enumerate(thumbnails):
1403                 t['url'] = sanitize_url(t['url'])
1404                 if t.get('width') and t.get('height'):
1405                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1406                 if t.get('id') is None:
1407                     t['id'] = '%d' % i
1408
1409         if self.params.get('list_thumbnails'):
1410             self.list_thumbnails(info_dict)
1411             return
1412
1413         thumbnail = info_dict.get('thumbnail')
1414         if thumbnail:
1415             info_dict['thumbnail'] = sanitize_url(thumbnail)
1416         elif thumbnails:
1417             info_dict['thumbnail'] = thumbnails[-1]['url']
1418
1419         if 'display_id' not in info_dict and 'id' in info_dict:
1420             info_dict['display_id'] = info_dict['id']
1421
1422         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1423             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1424             # see http://bugs.python.org/issue1646728)
1425             try:
1426                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1427                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1428             except (ValueError, OverflowError, OSError):
1429                 pass
1430
1431         # Auto generate title fields corresponding to the *_number fields when missing
1432         # in order to always have clean titles. This is very common for TV series.
1433         for field in ('chapter', 'season', 'episode'):
1434             if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1435                 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1436
1437         subtitles = info_dict.get('subtitles')
1438         if subtitles:
1439             for _, subtitle in subtitles.items():
1440                 for subtitle_format in subtitle:
1441                     if subtitle_format.get('url'):
1442                         subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1443                     if subtitle_format.get('ext') is None:
1444                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1445
1446         if self.params.get('listsubtitles', False):
1447             if 'automatic_captions' in info_dict:
1448                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1449             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1450             return
1451         info_dict['requested_subtitles'] = self.process_subtitles(
1452             info_dict['id'], subtitles,
1453             info_dict.get('automatic_captions'))
1454
1455         # We now pick which formats have to be downloaded
1456         if info_dict.get('formats') is None:
1457             # There's only one format available
1458             formats = [info_dict]
1459         else:
1460             formats = info_dict['formats']
1461
1462         if not formats:
1463             raise ExtractorError('No video formats found!')
1464
1465         def is_wellformed(f):
1466             url = f.get('url')
1467             valid_url = url and isinstance(url, compat_str)
1468             if not valid_url:
1469                 self.report_warning(
1470                     '"url" field is missing or empty - skipping format, '
1471                     'there is an error in extractor')
1472             return valid_url
1473
1474         # Filter out malformed formats for better extraction robustness
1475         formats = list(filter(is_wellformed, formats))
1476
1477         formats_dict = {}
1478
1479         # We check that all the formats have the format and format_id fields
1480         for i, format in enumerate(formats):
1481             sanitize_string_field(format, 'format_id')
1482             sanitize_numeric_fields(format)
1483             format['url'] = sanitize_url(format['url'])
1484             if format.get('format_id') is None:
1485                 format['format_id'] = compat_str(i)
1486             else:
1487                 # Sanitize format_id from characters used in format selector expression
1488                 format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
1489             format_id = format['format_id']
1490             if format_id not in formats_dict:
1491                 formats_dict[format_id] = []
1492             formats_dict[format_id].append(format)
1493
1494         # Make sure all formats have unique format_id
1495         for format_id, ambiguous_formats in formats_dict.items():
1496             if len(ambiguous_formats) > 1:
1497                 for i, format in enumerate(ambiguous_formats):
1498                     format['format_id'] = '%s-%d' % (format_id, i)
1499
1500         for i, format in enumerate(formats):
1501             if format.get('format') is None:
1502                 format['format'] = '{id} - {res}{note}'.format(
1503                     id=format['format_id'],
1504                     res=self.format_resolution(format),
1505                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1506                 )
1507             # Automatically determine file extension if missing
1508             if format.get('ext') is None:
1509                 format['ext'] = determine_ext(format['url']).lower()
1510             # Automatically determine protocol if missing (useful for format
1511             # selection purposes)
1512             if format.get('protocol') is None:
1513                 format['protocol'] = determine_protocol(format)
1514             # Add HTTP headers, so that external programs can use them from the
1515             # json output
1516             full_format_info = info_dict.copy()
1517             full_format_info.update(format)
1518             format['http_headers'] = self._calc_headers(full_format_info)
1519         # Remove private housekeeping stuff
1520         if '__x_forwarded_for_ip' in info_dict:
1521             del info_dict['__x_forwarded_for_ip']
1522
1523         # TODO Central sorting goes here
1524
1525         if formats[0] is not info_dict:
1526             # only set the 'formats' fields if the original info_dict list them
1527             # otherwise we end up with a circular reference, the first (and unique)
1528             # element in the 'formats' field in info_dict is info_dict itself,
1529             # which can't be exported to json
1530             info_dict['formats'] = formats
1531         if self.params.get('listformats'):
1532             self.list_formats(info_dict)
1533             return
1534
1535         req_format = self.params.get('format')
1536         if req_format is None:
1537             req_format_list = []
1538             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1539                     not info_dict.get('is_live')):
1540                 merger = FFmpegMergerPP(self)
1541                 if merger.available and merger.can_merge():
1542                     req_format_list.append('bestvideo+bestaudio')
1543             req_format_list.append('best')
1544             req_format = '/'.join(req_format_list)
1545         format_selector = self.build_format_selector(req_format)
1546
1547         # While in format selection we may need to have an access to the original
1548         # format set in order to calculate some metrics or do some processing.
1549         # For now we need to be able to guess whether original formats provided
1550         # by extractor are incomplete or not (i.e. whether extractor provides only
1551         # video-only or audio-only formats) for proper formats selection for
1552         # extractors with such incomplete formats (see
1553         # https://github.com/rg3/youtube-dl/pull/5556).
1554         # Since formats may be filtered during format selection and may not match
1555         # the original formats the results may be incorrect. Thus original formats
1556         # or pre-calculated metrics should be passed to format selection routines
1557         # as well.
1558         # We will pass a context object containing all necessary additional data
1559         # instead of just formats.
1560         # This fixes incorrect format selection issue (see
1561         # https://github.com/rg3/youtube-dl/issues/10083).
1562         incomplete_formats = (
1563             # All formats are video-only or
1564             all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
1565             # all formats are audio-only
1566             all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
1567
1568         ctx = {
1569             'formats': formats,
1570             'incomplete_formats': incomplete_formats,
1571         }
1572
1573         formats_to_download = list(format_selector(ctx))
1574         if not formats_to_download:
1575             raise ExtractorError('requested format not available',
1576                                  expected=True)
1577
1578         if download:
1579             if len(formats_to_download) > 1:
1580                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1581             for format in formats_to_download:
1582                 new_info = dict(info_dict)
1583                 new_info.update(format)
1584                 self.process_info(new_info)
1585         # We update the info dict with the best quality format (backwards compatibility)
1586         info_dict.update(formats_to_download[-1])
1587         return info_dict
1588
1589     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1590         """Select the requested subtitles and their format"""
1591         available_subs = {}
1592         if normal_subtitles and self.params.get('writesubtitles'):
1593             available_subs.update(normal_subtitles)
1594         if automatic_captions and self.params.get('writeautomaticsub'):
1595             for lang, cap_info in automatic_captions.items():
1596                 if lang not in available_subs:
1597                     available_subs[lang] = cap_info
1598
1599         if (not self.params.get('writesubtitles') and not
1600                 self.params.get('writeautomaticsub') or not
1601                 available_subs):
1602             return None
1603
1604         if self.params.get('allsubtitles', False):
1605             requested_langs = available_subs.keys()
1606         else:
1607             if self.params.get('subtitleslangs', False):
1608                 requested_langs = self.params.get('subtitleslangs')
1609             elif 'en' in available_subs:
1610                 requested_langs = ['en']
1611             else:
1612                 requested_langs = [list(available_subs.keys())[0]]
1613
1614         formats_query = self.params.get('subtitlesformat', 'best')
1615         formats_preference = formats_query.split('/') if formats_query else []
1616         subs = {}
1617         for lang in requested_langs:
1618             formats = available_subs.get(lang)
1619             if formats is None:
1620                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1621                 continue
1622             for ext in formats_preference:
1623                 if ext == 'best':
1624                     f = formats[-1]
1625                     break
1626                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1627                 if matches:
1628                     f = matches[-1]
1629                     break
1630             else:
1631                 f = formats[-1]
1632                 self.report_warning(
1633                     'No subtitle format found matching "%s" for language %s, '
1634                     'using %s' % (formats_query, lang, f['ext']))
1635             subs[lang] = f
1636         return subs
1637
1638     def process_info(self, info_dict):
1639         """Process a single resolved IE result."""
1640
1641         assert info_dict.get('_type', 'video') == 'video'
1642
1643         max_downloads = self.params.get('max_downloads')
1644         if max_downloads is not None:
1645             if self._num_downloads >= int(max_downloads):
1646                 raise MaxDownloadsReached()
1647
1648         info_dict['fulltitle'] = info_dict['title']
1649         if len(info_dict['title']) > 200:
1650             info_dict['title'] = info_dict['title'][:197] + '...'
1651
1652         if 'format' not in info_dict:
1653             info_dict['format'] = info_dict['ext']
1654
1655         reason = self._match_entry(info_dict, incomplete=False)
1656         if reason is not None:
1657             self.to_screen('[download] ' + reason)
1658             return
1659
1660         self._num_downloads += 1
1661
1662         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1663
1664         # Forced printings
1665         if self.params.get('forcetitle', False):
1666             self.to_stdout(info_dict['fulltitle'])
1667         if self.params.get('forceid', False):
1668             self.to_stdout(info_dict['id'])
1669         if self.params.get('forceurl', False):
1670             if info_dict.get('requested_formats') is not None:
1671                 for f in info_dict['requested_formats']:
1672                     self.to_stdout(f['url'] + f.get('play_path', ''))
1673             else:
1674                 # For RTMP URLs, also include the playpath
1675                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1676         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1677             self.to_stdout(info_dict['thumbnail'])
1678         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1679             self.to_stdout(info_dict['description'])
1680         if self.params.get('forcefilename', False) and filename is not None:
1681             self.to_stdout(filename)
1682         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1683             self.to_stdout(formatSeconds(info_dict['duration']))
1684         if self.params.get('forceformat', False):
1685             self.to_stdout(info_dict['format'])
1686         if self.params.get('forcejson', False):
1687             self.to_stdout(json.dumps(info_dict))
1688
1689         # Do nothing else if in simulate mode
1690         if self.params.get('simulate', False):
1691             return
1692
1693         if filename is None:
1694             return
1695
1696         try:
1697             dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1698             if dn and not os.path.exists(dn):
1699                 os.makedirs(dn)
1700         except (OSError, IOError) as err:
1701             self.report_error('unable to create directory ' + error_to_compat_str(err))
1702             return
1703
1704         if self.params.get('writedescription', False):
1705             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1706             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1707                 self.to_screen('[info] Video description is already present')
1708             elif info_dict.get('description') is None:
1709                 self.report_warning('There\'s no description to write.')
1710             else:
1711                 try:
1712                     self.to_screen('[info] Writing video description to: ' + descfn)
1713                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1714                         descfile.write(info_dict['description'])
1715                 except (OSError, IOError):
1716                     self.report_error('Cannot write description file ' + descfn)
1717                     return
1718
1719         if self.params.get('writeannotations', False):
1720             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1721             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1722                 self.to_screen('[info] Video annotations are already present')
1723             else:
1724                 try:
1725                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1726                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1727                         annofile.write(info_dict['annotations'])
1728                 except (KeyError, TypeError):
1729                     self.report_warning('There are no annotations to write.')
1730                 except (OSError, IOError):
1731                     self.report_error('Cannot write annotations file: ' + annofn)
1732                     return
1733
1734         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1735                                        self.params.get('writeautomaticsub')])
1736
1737         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1738             # subtitles download errors are already managed as troubles in relevant IE
1739             # that way it will silently go on when used with unsupporting IE
1740             subtitles = info_dict['requested_subtitles']
1741             ie = self.get_info_extractor(info_dict['extractor_key'])
1742             for sub_lang, sub_info in subtitles.items():
1743                 sub_format = sub_info['ext']
1744                 if sub_info.get('data') is not None:
1745                     sub_data = sub_info['data']
1746                 else:
1747                     try:
1748                         sub_data = ie._download_webpage(
1749                             sub_info['url'], info_dict['id'], note=False)
1750                     except ExtractorError as err:
1751                         self.report_warning('Unable to download subtitle for "%s": %s' %
1752                                             (sub_lang, error_to_compat_str(err.cause)))
1753                         continue
1754                 try:
1755                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1756                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1757                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1758                     else:
1759                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1760                         # Use newline='' to prevent conversion of newline characters
1761                         # See https://github.com/rg3/youtube-dl/issues/10268
1762                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
1763                             subfile.write(sub_data)
1764                 except (OSError, IOError):
1765                     self.report_error('Cannot write subtitles file ' + sub_filename)
1766                     return
1767
1768         if self.params.get('writeinfojson', False):
1769             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1770             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1771                 self.to_screen('[info] Video description metadata is already present')
1772             else:
1773                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1774                 try:
1775                     write_json_file(self.filter_requested_info(info_dict), infofn)
1776                 except (OSError, IOError):
1777                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1778                     return
1779
1780         self._write_thumbnails(info_dict, filename)
1781
1782         if not self.params.get('skip_download', False):
1783             try:
1784                 def dl(name, info):
1785                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1786                     for ph in self._progress_hooks:
1787                         fd.add_progress_hook(ph)
1788                     if self.params.get('verbose'):
1789                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1790                     return fd.download(name, info)
1791
1792                 if info_dict.get('requested_formats') is not None:
1793                     downloaded = []
1794                     success = True
1795                     merger = FFmpegMergerPP(self)
1796                     if not merger.available:
1797                         postprocessors = []
1798                         self.report_warning('You have requested multiple '
1799                                             'formats but ffmpeg or avconv are not installed.'
1800                                             ' The formats won\'t be merged.')
1801                     else:
1802                         postprocessors = [merger]
1803
1804                     def compatible_formats(formats):
1805                         video, audio = formats
1806                         # Check extension
1807                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1808                         if video_ext and audio_ext:
1809                             COMPATIBLE_EXTS = (
1810                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'),
1811                                 ('webm')
1812                             )
1813                             for exts in COMPATIBLE_EXTS:
1814                                 if video_ext in exts and audio_ext in exts:
1815                                     return True
1816                         # TODO: Check acodec/vcodec
1817                         return False
1818
1819                     filename_real_ext = os.path.splitext(filename)[1][1:]
1820                     filename_wo_ext = (
1821                         os.path.splitext(filename)[0]
1822                         if filename_real_ext == info_dict['ext']
1823                         else filename)
1824                     requested_formats = info_dict['requested_formats']
1825                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1826                         info_dict['ext'] = 'mkv'
1827                         self.report_warning(
1828                             'Requested formats are incompatible for merge and will be merged into mkv.')
1829                     # Ensure filename always has a correct extension for successful merge
1830                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1831                     if os.path.exists(encodeFilename(filename)):
1832                         self.to_screen(
1833                             '[download] %s has already been downloaded and '
1834                             'merged' % filename)
1835                     else:
1836                         for f in requested_formats:
1837                             new_info = dict(info_dict)
1838                             new_info.update(f)
1839                             fname = self.prepare_filename(new_info)
1840                             fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1841                             downloaded.append(fname)
1842                             partial_success = dl(fname, new_info)
1843                             success = success and partial_success
1844                         info_dict['__postprocessors'] = postprocessors
1845                         info_dict['__files_to_merge'] = downloaded
1846                 else:
1847                     # Just a single file
1848                     success = dl(filename, info_dict)
1849             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1850                 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1851                 return
1852             except (OSError, IOError) as err:
1853                 raise UnavailableVideoError(err)
1854             except (ContentTooShortError, ) as err:
1855                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1856                 return
1857
1858             if success and filename != '-':
1859                 # Fixup content
1860                 fixup_policy = self.params.get('fixup')
1861                 if fixup_policy is None:
1862                     fixup_policy = 'detect_or_warn'
1863
1864                 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1865
1866                 stretched_ratio = info_dict.get('stretched_ratio')
1867                 if stretched_ratio is not None and stretched_ratio != 1:
1868                     if fixup_policy == 'warn':
1869                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1870                             info_dict['id'], stretched_ratio))
1871                     elif fixup_policy == 'detect_or_warn':
1872                         stretched_pp = FFmpegFixupStretchedPP(self)
1873                         if stretched_pp.available:
1874                             info_dict.setdefault('__postprocessors', [])
1875                             info_dict['__postprocessors'].append(stretched_pp)
1876                         else:
1877                             self.report_warning(
1878                                 '%s: Non-uniform pixel ratio (%s). %s'
1879                                 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1880                     else:
1881                         assert fixup_policy in ('ignore', 'never')
1882
1883                 if (info_dict.get('requested_formats') is None and
1884                         info_dict.get('container') == 'm4a_dash'):
1885                     if fixup_policy == 'warn':
1886                         self.report_warning(
1887                             '%s: writing DASH m4a. '
1888                             'Only some players support this container.'
1889                             % info_dict['id'])
1890                     elif fixup_policy == 'detect_or_warn':
1891                         fixup_pp = FFmpegFixupM4aPP(self)
1892                         if fixup_pp.available:
1893                             info_dict.setdefault('__postprocessors', [])
1894                             info_dict['__postprocessors'].append(fixup_pp)
1895                         else:
1896                             self.report_warning(
1897                                 '%s: writing DASH m4a. '
1898                                 'Only some players support this container. %s'
1899                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1900                     else:
1901                         assert fixup_policy in ('ignore', 'never')
1902
1903                 if (info_dict.get('protocol') == 'm3u8_native' or
1904                         info_dict.get('protocol') == 'm3u8' and
1905                         self.params.get('hls_prefer_native')):
1906                     if fixup_policy == 'warn':
1907                         self.report_warning('%s: malformed AAC bitstream detected.' % (
1908                             info_dict['id']))
1909                     elif fixup_policy == 'detect_or_warn':
1910                         fixup_pp = FFmpegFixupM3u8PP(self)
1911                         if fixup_pp.available:
1912                             info_dict.setdefault('__postprocessors', [])
1913                             info_dict['__postprocessors'].append(fixup_pp)
1914                         else:
1915                             self.report_warning(
1916                                 '%s: malformed AAC bitstream detected. %s'
1917                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1918                     else:
1919                         assert fixup_policy in ('ignore', 'never')
1920
1921                 try:
1922                     self.post_process(filename, info_dict)
1923                 except (PostProcessingError) as err:
1924                     self.report_error('postprocessing: %s' % str(err))
1925                     return
1926                 self.record_download_archive(info_dict)
1927
1928     def download(self, url_list):
1929         """Download a given list of URLs."""
1930         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1931         if (len(url_list) > 1 and
1932                 outtmpl != '-' and
1933                 '%' not in outtmpl and
1934                 self.params.get('max_downloads') != 1):
1935             raise SameFileError(outtmpl)
1936
1937         for url in url_list:
1938             try:
1939                 # It also downloads the videos
1940                 res = self.extract_info(
1941                     url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1942             except UnavailableVideoError:
1943                 self.report_error('unable to download video')
1944             except MaxDownloadsReached:
1945                 self.to_screen('[info] Maximum number of downloaded files reached.')
1946                 raise
1947             else:
1948                 if self.params.get('dump_single_json', False):
1949                     self.to_stdout(json.dumps(res))
1950
1951         return self._download_retcode
1952
1953     def download_with_info_file(self, info_filename):
1954         with contextlib.closing(fileinput.FileInput(
1955                 [info_filename], mode='r',
1956                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1957             # FileInput doesn't have a read method, we can't call json.load
1958             info = self.filter_requested_info(json.loads('\n'.join(f)))
1959         try:
1960             self.process_ie_result(info, download=True)
1961         except DownloadError:
1962             webpage_url = info.get('webpage_url')
1963             if webpage_url is not None:
1964                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1965                 return self.download([webpage_url])
1966             else:
1967                 raise
1968         return self._download_retcode
1969
1970     @staticmethod
1971     def filter_requested_info(info_dict):
1972         return dict(
1973             (k, v) for k, v in info_dict.items()
1974             if k not in ['requested_formats', 'requested_subtitles'])
1975
1976     def post_process(self, filename, ie_info):
1977         """Run all the postprocessors on the given file."""
1978         info = dict(ie_info)
1979         info['filepath'] = filename
1980         pps_chain = []
1981         if ie_info.get('__postprocessors') is not None:
1982             pps_chain.extend(ie_info['__postprocessors'])
1983         pps_chain.extend(self._pps)
1984         for pp in pps_chain:
1985             files_to_delete = []
1986             try:
1987                 files_to_delete, info = pp.run(info)
1988             except PostProcessingError as e:
1989                 self.report_error(e.msg)
1990             if files_to_delete and not self.params.get('keepvideo', False):
1991                 for old_filename in files_to_delete:
1992                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1993                     try:
1994                         os.remove(encodeFilename(old_filename))
1995                     except (IOError, OSError):
1996                         self.report_warning('Unable to remove downloaded original file')
1997
1998     def _make_archive_id(self, info_dict):
1999         # Future-proof against any change in case
2000         # and backwards compatibility with prior versions
2001         extractor = info_dict.get('extractor_key')
2002         if extractor is None:
2003             if 'id' in info_dict:
2004                 extractor = info_dict.get('ie_key')  # key in a playlist
2005         if extractor is None:
2006             return None  # Incomplete video information
2007         return extractor.lower() + ' ' + info_dict['id']
2008
2009     def in_download_archive(self, info_dict):
2010         fn = self.params.get('download_archive')
2011         if fn is None:
2012             return False
2013
2014         vid_id = self._make_archive_id(info_dict)
2015         if vid_id is None:
2016             return False  # Incomplete video information
2017
2018         try:
2019             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
2020                 for line in archive_file:
2021                     if line.strip() == vid_id:
2022                         return True
2023         except IOError as ioe:
2024             if ioe.errno != errno.ENOENT:
2025                 raise
2026         return False
2027
2028     def record_download_archive(self, info_dict):
2029         fn = self.params.get('download_archive')
2030         if fn is None:
2031             return
2032         vid_id = self._make_archive_id(info_dict)
2033         assert vid_id
2034         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
2035             archive_file.write(vid_id + '\n')
2036
2037     @staticmethod
2038     def format_resolution(format, default='unknown'):
2039         if format.get('vcodec') == 'none':
2040             return 'audio only'
2041         if format.get('resolution') is not None:
2042             return format['resolution']
2043         if format.get('height') is not None:
2044             if format.get('width') is not None:
2045                 res = '%sx%s' % (format['width'], format['height'])
2046             else:
2047                 res = '%sp' % format['height']
2048         elif format.get('width') is not None:
2049             res = '%dx?' % format['width']
2050         else:
2051             res = default
2052         return res
2053
2054     def _format_note(self, fdict):
2055         res = ''
2056         if fdict.get('ext') in ['f4f', 'f4m']:
2057             res += '(unsupported) '
2058         if fdict.get('language'):
2059             if res:
2060                 res += ' '
2061             res += '[%s] ' % fdict['language']
2062         if fdict.get('format_note') is not None:
2063             res += fdict['format_note'] + ' '
2064         if fdict.get('tbr') is not None:
2065             res += '%4dk ' % fdict['tbr']
2066         if fdict.get('container') is not None:
2067             if res:
2068                 res += ', '
2069             res += '%s container' % fdict['container']
2070         if (fdict.get('vcodec') is not None and
2071                 fdict.get('vcodec') != 'none'):
2072             if res:
2073                 res += ', '
2074             res += fdict['vcodec']
2075             if fdict.get('vbr') is not None:
2076                 res += '@'
2077         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
2078             res += 'video@'
2079         if fdict.get('vbr') is not None:
2080             res += '%4dk' % fdict['vbr']
2081         if fdict.get('fps') is not None:
2082             if res:
2083                 res += ', '
2084             res += '%sfps' % fdict['fps']
2085         if fdict.get('acodec') is not None:
2086             if res:
2087                 res += ', '
2088             if fdict['acodec'] == 'none':
2089                 res += 'video only'
2090             else:
2091                 res += '%-5s' % fdict['acodec']
2092         elif fdict.get('abr') is not None:
2093             if res:
2094                 res += ', '
2095             res += 'audio'
2096         if fdict.get('abr') is not None:
2097             res += '@%3dk' % fdict['abr']
2098         if fdict.get('asr') is not None:
2099             res += ' (%5dHz)' % fdict['asr']
2100         if fdict.get('filesize') is not None:
2101             if res:
2102                 res += ', '
2103             res += format_bytes(fdict['filesize'])
2104         elif fdict.get('filesize_approx') is not None:
2105             if res:
2106                 res += ', '
2107             res += '~' + format_bytes(fdict['filesize_approx'])
2108         return res
2109
2110     def list_formats(self, info_dict):
2111         formats = info_dict.get('formats', [info_dict])
2112         table = [
2113             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
2114             for f in formats
2115             if f.get('preference') is None or f['preference'] >= -1000]
2116         if len(formats) > 1:
2117             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
2118
2119         header_line = ['format code', 'extension', 'resolution', 'note']
2120         self.to_screen(
2121             '[info] Available formats for %s:\n%s' %
2122             (info_dict['id'], render_table(header_line, table)))
2123
2124     def list_thumbnails(self, info_dict):
2125         thumbnails = info_dict.get('thumbnails')
2126         if not thumbnails:
2127             self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
2128             return
2129
2130         self.to_screen(
2131             '[info] Thumbnails for %s:' % info_dict['id'])
2132         self.to_screen(render_table(
2133             ['ID', 'width', 'height', 'URL'],
2134             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
2135
2136     def list_subtitles(self, video_id, subtitles, name='subtitles'):
2137         if not subtitles:
2138             self.to_screen('%s has no %s' % (video_id, name))
2139             return
2140         self.to_screen(
2141             'Available %s for %s:' % (name, video_id))
2142         self.to_screen(render_table(
2143             ['Language', 'formats'],
2144             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
2145                 for lang, formats in subtitles.items()]))
2146
2147     def urlopen(self, req):
2148         """ Start an HTTP download """
2149         if isinstance(req, compat_basestring):
2150             req = sanitized_Request(req)
2151         return self._opener.open(req, timeout=self._socket_timeout)
2152
2153     def print_debug_header(self):
2154         if not self.params.get('verbose'):
2155             return
2156
2157         if type('') is not compat_str:
2158             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
2159             self.report_warning(
2160                 'Your Python is broken! Update to a newer and supported version')
2161
2162         stdout_encoding = getattr(
2163             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
2164         encoding_str = (
2165             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2166                 locale.getpreferredencoding(),
2167                 sys.getfilesystemencoding(),
2168                 stdout_encoding,
2169                 self.get_encoding()))
2170         write_string(encoding_str, encoding=None)
2171
2172         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
2173         if _LAZY_LOADER:
2174             self._write_string('[debug] Lazy loading extractors enabled' + '\n')
2175         try:
2176             sp = subprocess.Popen(
2177                 ['git', 'rev-parse', '--short', 'HEAD'],
2178                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2179                 cwd=os.path.dirname(os.path.abspath(__file__)))
2180             out, err = sp.communicate()
2181             out = out.decode().strip()
2182             if re.match('[0-9a-f]+', out):
2183                 self._write_string('[debug] Git HEAD: ' + out + '\n')
2184         except Exception:
2185             try:
2186                 sys.exc_clear()
2187             except Exception:
2188                 pass
2189         self._write_string('[debug] Python version %s - %s\n' % (
2190             platform.python_version(), platform_name()))
2191
2192         exe_versions = FFmpegPostProcessor.get_versions(self)
2193         exe_versions['rtmpdump'] = rtmpdump_version()
2194         exe_str = ', '.join(
2195             '%s %s' % (exe, v)
2196             for exe, v in sorted(exe_versions.items())
2197             if v
2198         )
2199         if not exe_str:
2200             exe_str = 'none'
2201         self._write_string('[debug] exe versions: %s\n' % exe_str)
2202
2203         proxy_map = {}
2204         for handler in self._opener.handlers:
2205             if hasattr(handler, 'proxies'):
2206                 proxy_map.update(handler.proxies)
2207         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2208
2209         if self.params.get('call_home', False):
2210             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2211             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2212             latest_version = self.urlopen(
2213                 'https://yt-dl.org/latest/version').read().decode('utf-8')
2214             if version_tuple(latest_version) > version_tuple(__version__):
2215                 self.report_warning(
2216                     'You are using an outdated version (newest version: %s)! '
2217                     'See https://yt-dl.org/update if you need help updating.' %
2218                     latest_version)
2219
2220     def _setup_opener(self):
2221         timeout_val = self.params.get('socket_timeout')
2222         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2223
2224         opts_cookiefile = self.params.get('cookiefile')
2225         opts_proxy = self.params.get('proxy')
2226
2227         if opts_cookiefile is None:
2228             self.cookiejar = compat_cookiejar.CookieJar()
2229         else:
2230             opts_cookiefile = expand_path(opts_cookiefile)
2231             self.cookiejar = compat_cookiejar.MozillaCookieJar(
2232                 opts_cookiefile)
2233             if os.access(opts_cookiefile, os.R_OK):
2234                 self.cookiejar.load()
2235
2236         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2237         if opts_proxy is not None:
2238             if opts_proxy == '':
2239                 proxies = {}
2240             else:
2241                 proxies = {'http': opts_proxy, 'https': opts_proxy}
2242         else:
2243             proxies = compat_urllib_request.getproxies()
2244             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2245             if 'http' in proxies and 'https' not in proxies:
2246                 proxies['https'] = proxies['http']
2247         proxy_handler = PerRequestProxyHandler(proxies)
2248
2249         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2250         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2251         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2252         data_handler = compat_urllib_request_DataHandler()
2253
2254         # When passing our own FileHandler instance, build_opener won't add the
2255         # default FileHandler and allows us to disable the file protocol, which
2256         # can be used for malicious purposes (see
2257         # https://github.com/rg3/youtube-dl/issues/8227)
2258         file_handler = compat_urllib_request.FileHandler()
2259
2260         def file_open(*args, **kwargs):
2261             raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2262         file_handler.file_open = file_open
2263
2264         opener = compat_urllib_request.build_opener(
2265             proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2266
2267         # Delete the default user-agent header, which would otherwise apply in
2268         # cases where our custom HTTP handler doesn't come into play
2269         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2270         opener.addheaders = []
2271         self._opener = opener
2272
2273     def encode(self, s):
2274         if isinstance(s, bytes):
2275             return s  # Already encoded
2276
2277         try:
2278             return s.encode(self.get_encoding())
2279         except UnicodeEncodeError as err:
2280             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2281             raise
2282
2283     def get_encoding(self):
2284         encoding = self.params.get('encoding')
2285         if encoding is None:
2286             encoding = preferredencoding()
2287         return encoding
2288
2289     def _write_thumbnails(self, info_dict, filename):
2290         if self.params.get('writethumbnail', False):
2291             thumbnails = info_dict.get('thumbnails')
2292             if thumbnails:
2293                 thumbnails = [thumbnails[-1]]
2294         elif self.params.get('write_all_thumbnails', False):
2295             thumbnails = info_dict.get('thumbnails')
2296         else:
2297             return
2298
2299         if not thumbnails:
2300             # No thumbnails present, so return immediately
2301             return
2302
2303         for t in thumbnails:
2304             thumb_ext = determine_ext(t['url'], 'jpg')
2305             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2306             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2307             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2308
2309             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2310                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2311                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2312             else:
2313                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2314                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2315                 try:
2316                     uf = self.urlopen(t['url'])
2317                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2318                         shutil.copyfileobj(uf, thumbf)
2319                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2320                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2321                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2322                     self.report_warning('Unable to download thumbnail "%s": %s' %
2323                                         (t['url'], error_to_compat_str(err)))