_ Git - youtube-dl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # coding: utf-8
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import contextlib
   8 import copy
   9 import datetime
  10 import errno
  11 import fileinput
  12 import io
  13 import itertools
  14 import json
  15 import locale
  16 import operator
  17 import os
  18 import platform
  19 import re
  20 import shutil
  21 import subprocess
  22 import socket
  23 import sys
  24 import time
  25 import tokenize
  26 import traceback
  27 import random
  28
  29 from string import ascii_letters
  30
  31 from .compat import (
  32     compat_basestring,
  33     compat_cookiejar,
  34     compat_get_terminal_size,
  35     compat_http_client,
  36     compat_kwargs,
  37     compat_numeric_types,
  38     compat_os_name,
  39     compat_str,
  40     compat_tokenize_tokenize,
  41     compat_urllib_error,
  42     compat_urllib_request,
  43     compat_urllib_request_DataHandler,
  44 )
  45 from .utils import (
  46     age_restricted,
  47     args_to_str,
  48     ContentTooShortError,
  49     date_from_str,
  50     DateRange,
  51     DEFAULT_OUTTMPL,
  52     determine_ext,
  53     determine_protocol,
  54     DownloadError,
  55     encode_compat_str,
  56     encodeFilename,
  57     error_to_compat_str,
  58     expand_path,
  59     ExtractorError,
  60     format_bytes,
  61     formatSeconds,
  62     GeoRestrictedError,
  63     int_or_none,
  64     ISO3166Utils,
  65     locked_file,
  66     make_HTTPS_handler,
  67     MaxDownloadsReached,
  68     PagedList,
  69     parse_filesize,
  70     PerRequestProxyHandler,
  71     platform_name,
  72     PostProcessingError,
  73     preferredencoding,
  74     prepend_extension,
  75     register_socks_protocols,
  76     render_table,
  77     replace_extension,
  78     SameFileError,
  79     sanitize_filename,
  80     sanitize_path,
  81     sanitize_url,
  82     sanitized_Request,
  83     std_headers,
  84     subtitles_filename,
  85     UnavailableVideoError,
  86     url_basename,
  87     version_tuple,
  88     write_json_file,
  89     write_string,
  90     YoutubeDLCookieProcessor,
  91     YoutubeDLHandler,
  92     PhantomJSwrapper,
  93 )
  94 from .cache import Cache
  95 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
  96 from .downloader import get_suitable_downloader
  97 from .downloader.rtmp import rtmpdump_version
  98 from .postprocessor import (
  99     FFmpegFixupM3u8PP,
 100     FFmpegFixupM4aPP,
 101     FFmpegFixupStretchedPP,
 102     FFmpegMergerPP,
 103     FFmpegPostProcessor,
 104     get_postprocessor,
 105 )
 106 from .version import __version__
 107
 108 if compat_os_name == 'nt':
 109     import ctypes
 110
 111
 112 class YoutubeDL(object):
 113     """YoutubeDL class.
 114
 115     YoutubeDL objects are the ones responsible of downloading the
 116     actual video file and writing it to disk if the user has requested
 117     it, among some other tasks. In most cases there should be one per
 118     program. As, given a video URL, the downloader doesn't know how to
 119     extract all the needed information, task that InfoExtractors do, it
 120     has to pass the URL to one of them.
 121
 122     For this, YoutubeDL objects have a method that allows
 123     InfoExtractors to be registered in a given order. When it is passed
 124     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 125     finds that reports being able to handle it. The InfoExtractor extracts
 126     all the information about the video or videos the URL refers to, and
 127     YoutubeDL process the extracted information, possibly using a File
 128     Downloader to download the video.
 129
 130     YoutubeDL objects accept a lot of parameters. In order not to saturate
 131     the object constructor with arguments, it receives a dictionary of
 132     options instead. These options are available through the params
 133     attribute for the InfoExtractors to use. The YoutubeDL also
 134     registers itself as the downloader in charge for the InfoExtractors
 135     that are added to it, so this is a "mutual registration".
 136
 137     Available options:
 138
 139     username:          Username for authentication purposes.
 140     password:          Password for authentication purposes.
 141     videopassword:     Password for accessing a video.
 142     ap_mso:            Adobe Pass multiple-system operator identifier.
 143     ap_username:       Multiple-system operator account username.
 144     ap_password:       Multiple-system operator account password.
 145     usenetrc:          Use netrc for authentication instead.
 146     verbose:           Print additional info to stdout.
 147     quiet:             Do not print messages to stdout.
 148     no_warnings:       Do not print out anything for warnings.
 149     forceurl:          Force printing final URL.
 150     forcetitle:        Force printing title.
 151     forceid:           Force printing ID.
 152     forcethumbnail:    Force printing thumbnail URL.
 153     forcedescription:  Force printing description.
 154     forcefilename:     Force printing final filename.
 155     forceduration:     Force printing duration.
 156     forcejson:         Force printing info_dict as JSON.
 157     dump_single_json:  Force printing the info_dict of the whole playlist
 158                        (or video) as a single JSON line.
 159     simulate:          Do not download the video files.
 160     format:            Video format code. See options.py for more information.
 161     outtmpl:           Template for output names.
 162     restrictfilenames: Do not allow "&" and spaces in file names
 163     ignoreerrors:      Do not stop on download errors.
 164     force_generic_extractor: Force downloader to use the generic extractor
 165     nooverwrites:      Prevent overwriting files.
 166     playliststart:     Playlist item to start at.
 167     playlistend:       Playlist item to end at.
 168     playlist_items:    Specific indices of playlist to download.
 169     playlistreverse:   Download playlist items in reverse order.
 170     playlistrandom:    Download playlist items in random order.
 171     matchtitle:        Download only matching titles.
 172     rejecttitle:       Reject downloads for matching titles.
 173     logger:            Log messages to a logging.Logger instance.
 174     logtostderr:       Log messages to stderr instead of stdout.
 175     writedescription:  Write the video description to a .description file
 176     writeinfojson:     Write the video description to a .info.json file
 177     writeannotations:  Write the video annotations to a .annotations.xml file
 178     writethumbnail:    Write the thumbnail image to a file
 179     write_all_thumbnails:  Write all thumbnail formats to files
 180     writesubtitles:    Write the video subtitles to a file
 181     writeautomaticsub: Write the automatically generated subtitles to a file
 182     allsubtitles:      Downloads all the subtitles of the video
 183                        (requires writesubtitles or writeautomaticsub)
 184     listsubtitles:     Lists all available subtitles for the video
 185     subtitlesformat:   The format code for subtitles
 186     subtitleslangs:    List of languages of the subtitles to download
 187     keepvideo:         Keep the video file after post-processing
 188     daterange:         A DateRange object, download only if the upload_date is in the range.
 189     skip_download:     Skip the actual download of the video file
 190     cachedir:          Location of the cache files in the filesystem.
 191                        False to disable filesystem cache.
 192     noplaylist:        Download single video instead of a playlist if in doubt.
 193     age_limit:         An integer representing the user's age in years.
 194                        Unsuitable videos for the given age are skipped.
 195     min_views:         An integer representing the minimum view count the video
 196                        must have in order to not be skipped.
 197                        Videos without view count information are always
 198                        downloaded. None for no limit.
 199     max_views:         An integer representing the maximum view count.
 200                        Videos that are more popular than that are not
 201                        downloaded.
 202                        Videos without view count information are always
 203                        downloaded. None for no limit.
 204     download_archive:  File name of a file where all downloads are recorded.
 205                        Videos already present in the file are not downloaded
 206                        again.
 207     cookiefile:        File name where cookies should be read from and dumped to.
 208     nocheckcertificate:Do not verify SSL certificates
 209     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 210                        At the moment, this is only supported by YouTube.
 211     proxy:             URL of the proxy server to use
 212     geo_verification_proxy:  URL of the proxy to use for IP address verification
 213                        on geo-restricted sites. (Experimental)
 214     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 215     bidi_workaround:   Work around buggy terminals without bidirectional text
 216                        support, using fridibi
 217     debug_printtraffic:Print out sent and received HTTP traffic
 218     include_ads:       Download ads as well
 219     default_search:    Prepend this string if an input url is not valid.
 220                        'auto' for elaborate guessing
 221     encoding:          Use this encoding instead of the system-specified.
 222     extract_flat:      Do not resolve URLs, return the immediate result.
 223                        Pass in 'in_playlist' to only show this behavior for
 224                        playlist items.
 225     postprocessors:    A list of dictionaries, each with an entry
 226                        * key:  The name of the postprocessor. See
 227                                youtube_dl/postprocessor/__init__.py for a list.
 228                        as well as any further keyword arguments for the
 229                        postprocessor.
 230     progress_hooks:    A list of functions that get called on download
 231                        progress, with a dictionary with the entries
 232                        * status: One of "downloading", "error", or "finished".
 233                                  Check this first and ignore unknown values.
 234
 235                        If status is one of "downloading", or "finished", the
 236                        following properties may also be present:
 237                        * filename: The final filename (always present)
 238                        * tmpfilename: The filename we're currently writing to
 239                        * downloaded_bytes: Bytes on disk
 240                        * total_bytes: Size of the whole file, None if unknown
 241                        * total_bytes_estimate: Guess of the eventual file size,
 242                                                None if unavailable.
 243                        * elapsed: The number of seconds since download started.
 244                        * eta: The estimated time in seconds, None if unknown
 245                        * speed: The download speed in bytes/second, None if
 246                                 unknown
 247                        * fragment_index: The counter of the currently
 248                                          downloaded video fragment.
 249                        * fragment_count: The number of fragments (= individual
 250                                          files that will be merged)
 251
 252                        Progress hooks are guaranteed to be called at least once
 253                        (with status "finished") if the download is successful.
 254     merge_output_format: Extension to use when merging formats.
 255     fixup:             Automatically correct known faults of the file.
 256                        One of:
 257                        - "never": do nothing
 258                        - "warn": only emit a warning
 259                        - "detect_or_warn": check whether we can do anything
 260                                            about it, warn otherwise (default)
 261     source_address:    (Experimental) Client-side IP address to bind to.
 262     call_home:         Boolean, true iff we are allowed to contact the
 263                        youtube-dl servers for debugging.
 264     sleep_interval:    Number of seconds to sleep before each download when
 265                        used alone or a lower bound of a range for randomized
 266                        sleep before each download (minimum possible number
 267                        of seconds to sleep) when used along with
 268                        max_sleep_interval.
 269     max_sleep_interval:Upper bound of a range for randomized sleep before each
 270                        download (maximum possible number of seconds to sleep).
 271                        Must only be used along with sleep_interval.
 272                        Actual sleep time will be a random float from range
 273                        [sleep_interval; max_sleep_interval].
 274     listformats:       Print an overview of available video formats and exit.
 275     list_thumbnails:   Print a table of all thumbnails and exit.
 276     match_filter:      A function that gets called with the info_dict of
 277                        every video.
 278                        If it returns a message, the video is ignored.
 279                        If it returns None, the video is downloaded.
 280                        match_filter_func in utils.py is one example for this.
 281     no_color:          Do not emit color codes in output.
 282     geo_bypass:        Bypass geographic restriction via faking X-Forwarded-For
 283                        HTTP header (experimental)
 284     geo_bypass_country:
 285                        Two-letter ISO 3166-2 country code that will be used for
 286                        explicit geographic restriction bypassing via faking
 287                        X-Forwarded-For HTTP header (experimental)
 288
 289     The following options determine which downloader is picked:
 290     external_downloader: Executable of the external downloader to call.
 291                        None or unset for standard (built-in) downloader.
 292     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
 293                        if True, otherwise use ffmpeg/avconv if False, otherwise
 294                        use downloader suggested by extractor if None.
 295
 296     The following parameters are not used by YoutubeDL itself, they are used by
 297     the downloader (see youtube_dl/downloader/common.py):
 298     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 299     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 300     xattr_set_filesize, external_downloader_args, hls_use_mpegts.
 301
 302     The following options are used by the post processors:
 303     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 304                        otherwise prefer avconv.
 305     postprocessor_args: A list of additional command-line arguments for the
 306                         postprocessor.
 307     """
 308
 309     _NUMERIC_FIELDS = set((
 310         'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
 311         'timestamp', 'upload_year', 'upload_month', 'upload_day',
 312         'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
 313         'average_rating', 'comment_count', 'age_limit',
 314         'start_time', 'end_time',
 315         'chapter_number', 'season_number', 'episode_number',
 316         'track_number', 'disc_number', 'release_year',
 317         'playlist_index',
 318     ))
 319
 320     params = None
 321     _ies = []
 322     _pps = []
 323     _download_retcode = None
 324     _num_downloads = None
 325     _screen_file = None
 326
 327     def __init__(self, params=None, auto_init=True):
 328         """Create a FileDownloader object with the given options."""
 329         if params is None:
 330             params = {}
 331         self._ies = []
 332         self._ies_instances = {}
 333         self._pps = []
 334         self._progress_hooks = []
 335         self._download_retcode = 0
 336         self._num_downloads = 0
 337         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 338         self._err_file = sys.stderr
 339         self.params = {
 340             # Default parameters
 341             'nocheckcertificate': False,
 342         }
 343         self.params.update(params)
 344         self.cache = Cache(self)
 345
 346         def check_deprecated(param, option, suggestion):
 347             if self.params.get(param) is not None:
 348                 self.report_warning(
 349                     '%s is deprecated. Use %s instead.' % (option, suggestion))
 350                 return True
 351             return False
 352
 353         if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
 354             if self.params.get('geo_verification_proxy') is None:
 355                 self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
 356
 357         check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
 358         check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
 359         check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
 360
 361         if params.get('bidi_workaround', False):
 362             try:
 363                 import pty
 364                 master, slave = pty.openpty()
 365                 width = compat_get_terminal_size().columns
 366                 if width is None:
 367                     width_args = []
 368                 else:
 369                     width_args = ['-w', str(width)]
 370                 sp_kwargs = dict(
 371                     stdin=subprocess.PIPE,
 372                     stdout=slave,
 373                     stderr=self._err_file)
 374                 try:
 375                     self._output_process = subprocess.Popen(
 376                         ['bidiv'] + width_args, **sp_kwargs
 377                     )
 378                 except OSError:
 379                     self._output_process = subprocess.Popen(
 380                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 381                 self._output_channel = os.fdopen(master, 'rb')
 382             except OSError as ose:
 383                 if ose.errno == errno.ENOENT:
 384                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 385                 else:
 386                     raise
 387
 388         if (sys.platform != 'win32' and
 389                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
 390                 not params.get('restrictfilenames', False)):
 391             # Unicode filesystem API will throw errors (#1474, #13027)
 392             self.report_warning(
 393                 'Assuming --restrict-filenames since file system encoding '
 394                 'cannot encode all characters. '
 395                 'Set the LC_ALL environment variable to fix this.')
 396             self.params['restrictfilenames'] = True
 397
 398         if isinstance(params.get('outtmpl'), bytes):
 399             self.report_warning(
 400                 'Parameter outtmpl is bytes, but should be a unicode string. '
 401                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 402
 403         self._setup_opener()
 404
 405         if auto_init:
 406             self.print_debug_header()
 407             self.add_default_info_extractors()
 408
 409         for pp_def_raw in self.params.get('postprocessors', []):
 410             pp_class = get_postprocessor(pp_def_raw['key'])
 411             pp_def = dict(pp_def_raw)
 412             del pp_def['key']
 413             pp = pp_class(self, **compat_kwargs(pp_def))
 414             self.add_post_processor(pp)
 415
 416         for ph in self.params.get('progress_hooks', []):
 417             self.add_progress_hook(ph)
 418
 419         register_socks_protocols()
 420
 421     def warn_if_short_id(self, argv):
 422         # short YouTube ID starting with dash?
 423         idxs = [
 424             i for i, a in enumerate(argv)
 425             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 426         if idxs:
 427             correct_argv = (
 428                 ['youtube-dl'] +
 429                 [a for i, a in enumerate(argv) if i not in idxs] +
 430                 ['--'] + [argv[i] for i in idxs]
 431             )
 432             self.report_warning(
 433                 'Long argument string detected. '
 434                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 435                 args_to_str(correct_argv))
 436
 437     def add_info_extractor(self, ie):
 438         """Add an InfoExtractor object to the end of the list."""
 439         self._ies.append(ie)
 440         if not isinstance(ie, type):
 441             self._ies_instances[ie.ie_key()] = ie
 442             ie.set_downloader(self)
 443
 444     def get_info_extractor(self, ie_key):
 445         """
 446         Get an instance of an IE with name ie_key, it will try to get one from
 447         the _ies list, if there's no instance it will create a new one and add
 448         it to the extractor list.
 449         """
 450         ie = self._ies_instances.get(ie_key)
 451         if ie is None:
 452             ie = get_info_extractor(ie_key)()
 453             self.add_info_extractor(ie)
 454         return ie
 455
 456     def add_default_info_extractors(self):
 457         """
 458         Add the InfoExtractors returned by gen_extractors to the end of the list
 459         """
 460         for ie in gen_extractor_classes():
 461             self.add_info_extractor(ie)
 462
 463     def add_post_processor(self, pp):
 464         """Add a PostProcessor object to the end of the chain."""
 465         self._pps.append(pp)
 466         pp.set_downloader(self)
 467
 468     def add_progress_hook(self, ph):
 469         """Add the progress hook (currently only for the file downloader)"""
 470         self._progress_hooks.append(ph)
 471
 472     def _bidi_workaround(self, message):
 473         if not hasattr(self, '_output_channel'):
 474             return message
 475
 476         assert hasattr(self, '_output_process')
 477         assert isinstance(message, compat_str)
 478         line_count = message.count('\n') + 1
 479         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 480         self._output_process.stdin.flush()
 481         res = ''.join(self._output_channel.readline().decode('utf-8')
 482                       for _ in range(line_count))
 483         return res[:-len('\n')]
 484
 485     def to_screen(self, message, skip_eol=False):
 486         """Print message to stdout if not in quiet mode."""
 487         return self.to_stdout(message, skip_eol, check_quiet=True)
 488
 489     def _write_string(self, s, out=None):
 490         write_string(s, out=out, encoding=self.params.get('encoding'))
 491
 492     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 493         """Print message to stdout if not in quiet mode."""
 494         if self.params.get('logger'):
 495             self.params['logger'].debug(message)
 496         elif not check_quiet or not self.params.get('quiet', False):
 497             message = self._bidi_workaround(message)
 498             terminator = ['\n', ''][skip_eol]
 499             output = message + terminator
 500
 501             self._write_string(output, self._screen_file)
 502
 503     def to_stderr(self, message):
 504         """Print message to stderr."""
 505         assert isinstance(message, compat_str)
 506         if self.params.get('logger'):
 507             self.params['logger'].error(message)
 508         else:
 509             message = self._bidi_workaround(message)
 510             output = message + '\n'
 511             self._write_string(output, self._err_file)
 512
 513     def to_console_title(self, message):
 514         if not self.params.get('consoletitle', False):
 515             return
 516         if compat_os_name == 'nt':
 517             if ctypes.windll.kernel32.GetConsoleWindow():
 518                 # c_wchar_p() might not be necessary if `message` is
 519                 # already of type unicode()
 520                 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 521         elif 'TERM' in os.environ:
 522             self._write_string('\033]0;%s\007' % message, self._screen_file)
 523
 524     def save_console_title(self):
 525         if not self.params.get('consoletitle', False):
 526             return
 527         if compat_os_name != 'nt' and 'TERM' in os.environ:
 528             # Save the title on stack
 529             self._write_string('\033[22;0t', self._screen_file)
 530
 531     def restore_console_title(self):
 532         if not self.params.get('consoletitle', False):
 533             return
 534         if compat_os_name != 'nt' and 'TERM' in os.environ:
 535             # Restore the title from stack
 536             self._write_string('\033[23;0t', self._screen_file)
 537
 538     def __enter__(self):
 539         self.save_console_title()
 540         return self
 541
 542     def __exit__(self, *args):
 543         self.restore_console_title()
 544
 545         if self.params.get('cookiefile') is not None:
 546             self.cookiejar.save()
 547
 548     def trouble(self, message=None, tb=None):
 549         """Determine action to take when a download problem appears.
 550
 551         Depending on if the downloader has been configured to ignore
 552         download errors or not, this method may throw an exception or
 553         not when errors are found, after printing the message.
 554
 555         tb, if given, is additional traceback information.
 556         """
 557         if message is not None:
 558             self.to_stderr(message)
 559         if self.params.get('verbose'):
 560             if tb is None:
 561                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 562                     tb = ''
 563                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 564                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 565                     tb += encode_compat_str(traceback.format_exc())
 566                 else:
 567                     tb_data = traceback.format_list(traceback.extract_stack())
 568                     tb = ''.join(tb_data)
 569             self.to_stderr(tb)
 570         if not self.params.get('ignoreerrors', False):
 571             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 572                 exc_info = sys.exc_info()[1].exc_info
 573             else:
 574                 exc_info = sys.exc_info()
 575             raise DownloadError(message, exc_info)
 576         self._download_retcode = 1
 577
 578     def report_warning(self, message):
 579         '''
 580         Print the message to stderr, it will be prefixed with 'WARNING:'
 581         If stderr is a tty file the 'WARNING:' will be colored
 582         '''
 583         if self.params.get('logger') is not None:
 584             self.params['logger'].warning(message)
 585         else:
 586             if self.params.get('no_warnings'):
 587                 return
 588             if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 589                 _msg_header = '\033[0;33mWARNING:\033[0m'
 590             else:
 591                 _msg_header = 'WARNING:'
 592             warning_message = '%s %s' % (_msg_header, message)
 593             self.to_stderr(warning_message)
 594
 595     def report_error(self, message, tb=None):
 596         '''
 597         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 598         in red if stderr is a tty file.
 599         '''
 600         if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 601             _msg_header = '\033[0;31mERROR:\033[0m'
 602         else:
 603             _msg_header = 'ERROR:'
 604         error_message = '%s %s' % (_msg_header, message)
 605         self.trouble(error_message, tb)
 606
 607     def report_file_already_downloaded(self, file_name):
 608         """Report file has already been fully downloaded."""
 609         try:
 610             self.to_screen('[download] %s has already been downloaded' % file_name)
 611         except UnicodeEncodeError:
 612             self.to_screen('[download] The file has already been downloaded')
 613
 614     def prepare_filename(self, info_dict):
 615         """Generate the output filename."""
 616         try:
 617             template_dict = dict(info_dict)
 618
 619             template_dict['epoch'] = int(time.time())
 620             autonumber_size = self.params.get('autonumber_size')
 621             if autonumber_size is None:
 622                 autonumber_size = 5
 623             template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
 624             if template_dict.get('resolution') is None:
 625                 if template_dict.get('width') and template_dict.get('height'):
 626                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 627                 elif template_dict.get('height'):
 628                     template_dict['resolution'] = '%sp' % template_dict['height']
 629                 elif template_dict.get('width'):
 630                     template_dict['resolution'] = '%dx?' % template_dict['width']
 631
 632             sanitize = lambda k, v: sanitize_filename(
 633                 compat_str(v),
 634                 restricted=self.params.get('restrictfilenames'),
 635                 is_id=(k == 'id' or k.endswith('_id')))
 636             template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
 637                                  for k, v in template_dict.items()
 638                                  if v is not None and not isinstance(v, (list, tuple, dict)))
 639             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 640
 641             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 642
 643             # For fields playlist_index and autonumber convert all occurrences
 644             # of %(field)s to %(field)0Nd for backward compatibility
 645             field_size_compat_map = {
 646                 'playlist_index': len(str(template_dict['n_entries'])),
 647                 'autonumber': autonumber_size,
 648             }
 649             FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
 650             mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl)
 651             if mobj:
 652                 outtmpl = re.sub(
 653                     FIELD_SIZE_COMPAT_RE,
 654                     r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
 655                     outtmpl)
 656
 657             # Missing numeric fields used together with integer presentation types
 658             # in format specification will break the argument substitution since
 659             # string 'NA' is returned for missing fields. We will patch output
 660             # template for missing fields to meet string presentation type.
 661             for numeric_field in self._NUMERIC_FIELDS:
 662                 if numeric_field not in template_dict:
 663                     # As of [1] format syntax is:
 664                     #  %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
 665                     # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
 666                     FORMAT_RE = r'''(?x)
 667                         (?<!%)
 668                         %
 669                         \({0}\)  # mapping key
 670                         (?:[#0\-+ ]+)?  # conversion flags (optional)
 671                         (?:\d+)?  # minimum field width (optional)
 672                         (?:\.\d+)?  # precision (optional)
 673                         [hlL]?  # length modifier (optional)
 674                         [diouxXeEfFgGcrs%]  # conversion type
 675                     '''
 676                     outtmpl = re.sub(
 677                         FORMAT_RE.format(numeric_field),
 678                         r'%({0})s'.format(numeric_field), outtmpl)
 679
 680             # expand_path translates '%%' into '%' and '$$' into '$'
 681             # correspondingly that is not what we want since we need to keep
 682             # '%%' intact for template dict substitution step. Working around
 683             # with boundary-alike separator hack.
 684             sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
 685             outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
 686
 687             # outtmpl should be expand_path'ed before template dict substitution
 688             # because meta fields may contain env variables we don't want to
 689             # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
 690             # title "Hello $PATH", we don't want `$PATH` to be expanded.
 691             filename = expand_path(outtmpl).replace(sep, '') % template_dict
 692
 693             # Temporary fix for #4787
 694             # 'Treat' all problem characters by passing filename through preferredencoding
 695             # to workaround encoding issues with subprocess on python2 @ Windows
 696             if sys.version_info < (3, 0) and sys.platform == 'win32':
 697                 filename = encodeFilename(filename, True).decode(preferredencoding())
 698             return sanitize_path(filename)
 699         except ValueError as err:
 700             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 701             return None
 702
 703     def _match_entry(self, info_dict, incomplete):
 704         """ Returns None iff the file should be downloaded """
 705
 706         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 707         if 'title' in info_dict:
 708             # This can happen when we're just evaluating the playlist
 709             title = info_dict['title']
 710             matchtitle = self.params.get('matchtitle', False)
 711             if matchtitle:
 712                 if not re.search(matchtitle, title, re.IGNORECASE):
 713                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 714             rejecttitle = self.params.get('rejecttitle', False)
 715             if rejecttitle:
 716                 if re.search(rejecttitle, title, re.IGNORECASE):
 717                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 718         date = info_dict.get('upload_date')
 719         if date is not None:
 720             dateRange = self.params.get('daterange', DateRange())
 721             if date not in dateRange:
 722                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 723         view_count = info_dict.get('view_count')
 724         if view_count is not None:
 725             min_views = self.params.get('min_views')
 726             if min_views is not None and view_count < min_views:
 727                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 728             max_views = self.params.get('max_views')
 729             if max_views is not None and view_count > max_views:
 730                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 731         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 732             return 'Skipping "%s" because it is age restricted' % video_title
 733         if self.in_download_archive(info_dict):
 734             return '%s has already been recorded in archive' % video_title
 735
 736         if not incomplete:
 737             match_filter = self.params.get('match_filter')
 738             if match_filter is not None:
 739                 ret = match_filter(info_dict)
 740                 if ret is not None:
 741                     return ret
 742
 743         return None
 744
 745     @staticmethod
 746     def add_extra_info(info_dict, extra_info):
 747         '''Set the keys from extra_info in info dict if they are missing'''
 748         for key, value in extra_info.items():
 749             info_dict.setdefault(key, value)
 750
 751     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 752                      process=True, force_generic_extractor=False):
 753         '''
 754         Returns a list with a dictionary for each video we find.
 755         If 'download', also downloads the videos.
 756         extra_info is a dict containing the extra values to add to each result
 757         '''
 758
 759         if not ie_key and force_generic_extractor:
 760             ie_key = 'Generic'
 761
 762         if ie_key:
 763             ies = [self.get_info_extractor(ie_key)]
 764         else:
 765             ies = self._ies
 766
 767         for ie in ies:
 768             if not ie.suitable(url):
 769                 continue
 770
 771             ie = self.get_info_extractor(ie.ie_key())
 772             if not ie.working():
 773                 self.report_warning('The program functionality for this site has been marked as broken, '
 774                                     'and will probably not work.')
 775
 776             try:
 777                 ie_result = ie.extract(url)
 778                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 779                     break
 780                 if isinstance(ie_result, list):
 781                     # Backwards compatibility: old IE result format
 782                     ie_result = {
 783                         '_type': 'compat_list',
 784                         'entries': ie_result,
 785                     }
 786                 self.add_default_extra_info(ie_result, ie, url)
 787                 if process:
 788                     return self.process_ie_result(ie_result, download, extra_info)
 789                 else:
 790                     return ie_result
 791             except GeoRestrictedError as e:
 792                 msg = e.msg
 793                 if e.countries:
 794                     msg += '\nThis video is available in %s.' % ', '.join(
 795                         map(ISO3166Utils.short2full, e.countries))
 796                 msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
 797                 self.report_error(msg)
 798                 break
 799             except ExtractorError as e:  # An error we somewhat expected
 800                 self.report_error(compat_str(e), e.format_traceback())
 801                 break
 802             except MaxDownloadsReached:
 803                 raise
 804             except Exception as e:
 805                 if self.params.get('ignoreerrors', False):
 806                     self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
 807                     break
 808                 else:
 809                     raise
 810         else:
 811             self.report_error('no suitable InfoExtractor for URL %s' % url)
 812
 813     def add_default_extra_info(self, ie_result, ie, url):
 814         self.add_extra_info(ie_result, {
 815             'extractor': ie.IE_NAME,
 816             'webpage_url': url,
 817             'webpage_url_basename': url_basename(url),
 818             'extractor_key': ie.ie_key(),
 819         })
 820
 821     def process_ie_result(self, ie_result, download=True, extra_info={}):
 822         """
 823         Take the result of the ie(may be modified) and resolve all unresolved
 824         references (URLs, playlist items).
 825
 826         It will also download the videos if 'download'.
 827         Returns the resolved ie_result.
 828         """
 829         result_type = ie_result.get('_type', 'video')
 830
 831         if result_type in ('url', 'url_transparent'):
 832             ie_result['url'] = sanitize_url(ie_result['url'])
 833             extract_flat = self.params.get('extract_flat', False)
 834             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 835                     extract_flat is True):
 836                 if self.params.get('forcejson', False):
 837                     self.to_stdout(json.dumps(ie_result))
 838                 return ie_result
 839
 840         if result_type == 'video':
 841             self.add_extra_info(ie_result, extra_info)
 842             return self.process_video_result(ie_result, download=download)
 843         elif result_type == 'url':
 844             # We have to add extra_info to the results because it may be
 845             # contained in a playlist
 846             return self.extract_info(ie_result['url'],
 847                                      download,
 848                                      ie_key=ie_result.get('ie_key'),
 849                                      extra_info=extra_info)
 850         elif result_type == 'url_transparent':
 851             # Use the information from the embedding page
 852             info = self.extract_info(
 853                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 854                 extra_info=extra_info, download=False, process=False)
 855
 856             # extract_info may return None when ignoreerrors is enabled and
 857             # extraction failed with an error, don't crash and return early
 858             # in this case
 859             if not info:
 860                 return info
 861
 862             force_properties = dict(
 863                 (k, v) for k, v in ie_result.items() if v is not None)
 864             for f in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
 865                 if f in force_properties:
 866                     del force_properties[f]
 867             new_result = info.copy()
 868             new_result.update(force_properties)
 869
 870             # Extracted info may not be a video result (i.e.
 871             # info.get('_type', 'video') != video) but rather an url or
 872             # url_transparent. In such cases outer metadata (from ie_result)
 873             # should be propagated to inner one (info). For this to happen
 874             # _type of info should be overridden with url_transparent. This
 875             # fixes issue from https://github.com/rg3/youtube-dl/pull/11163.
 876             if new_result.get('_type') == 'url':
 877                 new_result['_type'] = 'url_transparent'
 878
 879             return self.process_ie_result(
 880                 new_result, download=download, extra_info=extra_info)
 881         elif result_type in ('playlist', 'multi_video'):
 882             # We process each entry in the playlist
 883             playlist = ie_result.get('title') or ie_result.get('id')
 884             self.to_screen('[download] Downloading playlist: %s' % playlist)
 885
 886             playlist_results = []
 887
 888             playliststart = self.params.get('playliststart', 1) - 1
 889             playlistend = self.params.get('playlistend')
 890             # For backwards compatibility, interpret -1 as whole list
 891             if playlistend == -1:
 892                 playlistend = None
 893
 894             playlistitems_str = self.params.get('playlist_items')
 895             playlistitems = None
 896             if playlistitems_str is not None:
 897                 def iter_playlistitems(format):
 898                     for string_segment in format.split(','):
 899                         if '-' in string_segment:
 900                             start, end = string_segment.split('-')
 901                             for item in range(int(start), int(end) + 1):
 902                                 yield int(item)
 903                         else:
 904                             yield int(string_segment)
 905                 playlistitems = iter_playlistitems(playlistitems_str)
 906
 907             ie_entries = ie_result['entries']
 908             if isinstance(ie_entries, list):
 909                 n_all_entries = len(ie_entries)
 910                 if playlistitems:
 911                     entries = [
 912                         ie_entries[i - 1] for i in playlistitems
 913                         if -n_all_entries <= i - 1 < n_all_entries]
 914                 else:
 915                     entries = ie_entries[playliststart:playlistend]
 916                 n_entries = len(entries)
 917                 self.to_screen(
 918                     '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
 919                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 920             elif isinstance(ie_entries, PagedList):
 921                 if playlistitems:
 922                     entries = []
 923                     for item in playlistitems:
 924                         entries.extend(ie_entries.getslice(
 925                             item - 1, item
 926                         ))
 927                 else:
 928                     entries = ie_entries.getslice(
 929                         playliststart, playlistend)
 930                 n_entries = len(entries)
 931                 self.to_screen(
 932                     '[%s] playlist %s: Downloading %d videos' %
 933                     (ie_result['extractor'], playlist, n_entries))
 934             else:  # iterable
 935                 if playlistitems:
 936                     entry_list = list(ie_entries)
 937                     entries = [entry_list[i - 1] for i in playlistitems]
 938                 else:
 939                     entries = list(itertools.islice(
 940                         ie_entries, playliststart, playlistend))
 941                 n_entries = len(entries)
 942                 self.to_screen(
 943                     '[%s] playlist %s: Downloading %d videos' %
 944                     (ie_result['extractor'], playlist, n_entries))
 945
 946             if self.params.get('playlistreverse', False):
 947                 entries = entries[::-1]
 948
 949             if self.params.get('playlistrandom', False):
 950                 random.shuffle(entries)
 951
 952             x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
 953
 954             for i, entry in enumerate(entries, 1):
 955                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 956                 # This __x_forwarded_for_ip thing is a bit ugly but requires
 957                 # minimal changes
 958                 if x_forwarded_for:
 959                     entry['__x_forwarded_for_ip'] = x_forwarded_for
 960                 extra = {
 961                     'n_entries': n_entries,
 962                     'playlist': playlist,
 963                     'playlist_id': ie_result.get('id'),
 964                     'playlist_title': ie_result.get('title'),
 965                     'playlist_index': i + playliststart,
 966                     'extractor': ie_result['extractor'],
 967                     'webpage_url': ie_result['webpage_url'],
 968                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 969                     'extractor_key': ie_result['extractor_key'],
 970                 }
 971
 972                 reason = self._match_entry(entry, incomplete=True)
 973                 if reason is not None:
 974                     self.to_screen('[download] ' + reason)
 975                     continue
 976
 977                 entry_result = self.process_ie_result(entry,
 978                                                       download=download,
 979                                                       extra_info=extra)
 980                 playlist_results.append(entry_result)
 981             ie_result['entries'] = playlist_results
 982             self.to_screen('[download] Finished downloading playlist: %s' % playlist)
 983             return ie_result
 984         elif result_type == 'compat_list':
 985             self.report_warning(
 986                 'Extractor %s returned a compat_list result. '
 987                 'It needs to be updated.' % ie_result.get('extractor'))
 988
 989             def _fixup(r):
 990                 self.add_extra_info(
 991                     r,
 992                     {
 993                         'extractor': ie_result['extractor'],
 994                         'webpage_url': ie_result['webpage_url'],
 995                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 996                         'extractor_key': ie_result['extractor_key'],
 997                     }
 998                 )
 999                 return r
1000             ie_result['entries'] = [
1001                 self.process_ie_result(_fixup(r), download, extra_info)
1002                 for r in ie_result['entries']
1003             ]
1004             return ie_result
1005         else:
1006             raise Exception('Invalid result type: %s' % result_type)
1007
1008     def _build_format_filter(self, filter_spec):
1009         " Returns a function to filter the formats according to the filter_spec "
1010
1011         OPERATORS = {
1012             '<': operator.lt,
1013             '<=': operator.le,
1014             '>': operator.gt,
1015             '>=': operator.ge,
1016             '=': operator.eq,
1017             '!=': operator.ne,
1018         }
1019         operator_rex = re.compile(r'''(?x)\s*
1020             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
1021             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1022             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
1023             $
1024             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
1025         m = operator_rex.search(filter_spec)
1026         if m:
1027             try:
1028                 comparison_value = int(m.group('value'))
1029             except ValueError:
1030                 comparison_value = parse_filesize(m.group('value'))
1031                 if comparison_value is None:
1032                     comparison_value = parse_filesize(m.group('value') + 'B')
1033                 if comparison_value is None:
1034                     raise ValueError(
1035                         'Invalid value %r in format specification %r' % (
1036                             m.group('value'), filter_spec))
1037             op = OPERATORS[m.group('op')]
1038
1039         if not m:
1040             STR_OPERATORS = {
1041                 '=': operator.eq,
1042                 '!=': operator.ne,
1043                 '^=': lambda attr, value: attr.startswith(value),
1044                 '$=': lambda attr, value: attr.endswith(value),
1045                 '*=': lambda attr, value: value in attr,
1046             }
1047             str_operator_rex = re.compile(r'''(?x)
1048                 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
1049                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
1050                 \s*(?P<value>[a-zA-Z0-9._-]+)
1051                 \s*$
1052                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
1053             m = str_operator_rex.search(filter_spec)
1054             if m:
1055                 comparison_value = m.group('value')
1056                 op = STR_OPERATORS[m.group('op')]
1057
1058         if not m:
1059             raise ValueError('Invalid filter specification %r' % filter_spec)
1060
1061         def _filter(f):
1062             actual_value = f.get(m.group('key'))
1063             if actual_value is None:
1064                 return m.group('none_inclusive')
1065             return op(actual_value, comparison_value)
1066         return _filter
1067
1068     def _default_format_spec(self, info_dict, download=True):
1069         req_format_list = []
1070
1071         def can_have_partial_formats():
1072             if self.params.get('simulate', False):
1073                 return True
1074             if not download:
1075                 return True
1076             if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
1077                 return False
1078             if info_dict.get('is_live'):
1079                 return False
1080             merger = FFmpegMergerPP(self)
1081             return merger.available and merger.can_merge()
1082         if can_have_partial_formats():
1083             req_format_list.append('bestvideo+bestaudio')
1084         req_format_list.append('best')
1085         return '/'.join(req_format_list)
1086
1087     def build_format_selector(self, format_spec):
1088         def syntax_error(note, start):
1089             message = (
1090                 'Invalid format specification: '
1091                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
1092             return SyntaxError(message)
1093
1094         PICKFIRST = 'PICKFIRST'
1095         MERGE = 'MERGE'
1096         SINGLE = 'SINGLE'
1097         GROUP = 'GROUP'
1098         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1099
1100         def _parse_filter(tokens):
1101             filter_parts = []
1102             for type, string, start, _, _ in tokens:
1103                 if type == tokenize.OP and string == ']':
1104                     return ''.join(filter_parts)
1105                 else:
1106                     filter_parts.append(string)
1107
1108         def _remove_unused_ops(tokens):
1109             # Remove operators that we don't use and join them with the surrounding strings
1110             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1111             ALLOWED_OPS = ('/', '+', ',', '(', ')')
1112             last_string, last_start, last_end, last_line = None, None, None, None
1113             for type, string, start, end, line in tokens:
1114                 if type == tokenize.OP and string == '[':
1115                     if last_string:
1116                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1117                         last_string = None
1118                     yield type, string, start, end, line
1119                     # everything inside brackets will be handled by _parse_filter
1120                     for type, string, start, end, line in tokens:
1121                         yield type, string, start, end, line
1122                         if type == tokenize.OP and string == ']':
1123                             break
1124                 elif type == tokenize.OP and string in ALLOWED_OPS:
1125                     if last_string:
1126                         yield tokenize.NAME, last_string, last_start, last_end, last_line
1127                         last_string = None
1128                     yield type, string, start, end, line
1129                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
1130                     if not last_string:
1131                         last_string = string
1132                         last_start = start
1133                         last_end = end
1134                     else:
1135                         last_string += string
1136             if last_string:
1137                 yield tokenize.NAME, last_string, last_start, last_end, last_line
1138
1139         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
1140             selectors = []
1141             current_selector = None
1142             for type, string, start, _, _ in tokens:
1143                 # ENCODING is only defined in python 3.x
1144                 if type == getattr(tokenize, 'ENCODING', None):
1145                     continue
1146                 elif type in [tokenize.NAME, tokenize.NUMBER]:
1147                     current_selector = FormatSelector(SINGLE, string, [])
1148                 elif type == tokenize.OP:
1149                     if string == ')':
1150                         if not inside_group:
1151                             # ')' will be handled by the parentheses group
1152                             tokens.restore_last_token()
1153                         break
1154                     elif inside_merge and string in ['/', ',']:
1155                         tokens.restore_last_token()
1156                         break
1157                     elif inside_choice and string == ',':
1158                         tokens.restore_last_token()
1159                         break
1160                     elif string == ',':
1161                         if not current_selector:
1162                             raise syntax_error('"," must follow a format selector', start)
1163                         selectors.append(current_selector)
1164                         current_selector = None
1165                     elif string == '/':
1166                         if not current_selector:
1167                             raise syntax_error('"/" must follow a format selector', start)
1168                         first_choice = current_selector
1169                         second_choice = _parse_format_selection(tokens, inside_choice=True)
1170                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1171                     elif string == '[':
1172                         if not current_selector:
1173                             current_selector = FormatSelector(SINGLE, 'best', [])
1174                         format_filter = _parse_filter(tokens)
1175                         current_selector.filters.append(format_filter)
1176                     elif string == '(':
1177                         if current_selector:
1178                             raise syntax_error('Unexpected "("', start)
1179                         group = _parse_format_selection(tokens, inside_group=True)
1180                         current_selector = FormatSelector(GROUP, group, [])
1181                     elif string == '+':
1182                         video_selector = current_selector
1183                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
1184                         if not video_selector or not audio_selector:
1185                             raise syntax_error('"+" must be between two format selectors', start)
1186                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1187                     else:
1188                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1189                 elif type == tokenize.ENDMARKER:
1190                     break
1191             if current_selector:
1192                 selectors.append(current_selector)
1193             return selectors
1194
1195         def _build_selector_function(selector):
1196             if isinstance(selector, list):
1197                 fs = [_build_selector_function(s) for s in selector]
1198
1199                 def selector_function(ctx):
1200                     for f in fs:
1201                         for format in f(ctx):
1202                             yield format
1203                 return selector_function
1204             elif selector.type == GROUP:
1205                 selector_function = _build_selector_function(selector.selector)
1206             elif selector.type == PICKFIRST:
1207                 fs = [_build_selector_function(s) for s in selector.selector]
1208
1209                 def selector_function(ctx):
1210                     for f in fs:
1211                         picked_formats = list(f(ctx))
1212                         if picked_formats:
1213                             return picked_formats
1214                     return []
1215             elif selector.type == SINGLE:
1216                 format_spec = selector.selector
1217
1218                 def selector_function(ctx):
1219                     formats = list(ctx['formats'])
1220                     if not formats:
1221                         return
1222                     if format_spec == 'all':
1223                         for f in formats:
1224                             yield f
1225                     elif format_spec in ['best', 'worst', None]:
1226                         format_idx = 0 if format_spec == 'worst' else -1
1227                         audiovideo_formats = [
1228                             f for f in formats
1229                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1230                         if audiovideo_formats:
1231                             yield audiovideo_formats[format_idx]
1232                         # for extractors with incomplete formats (audio only (soundcloud)
1233                         # or video only (imgur)) we will fallback to best/worst
1234                         # {video,audio}-only format
1235                         elif ctx['incomplete_formats']:
1236                             yield formats[format_idx]
1237                     elif format_spec == 'bestaudio':
1238                         audio_formats = [
1239                             f for f in formats
1240                             if f.get('vcodec') == 'none']
1241                         if audio_formats:
1242                             yield audio_formats[-1]
1243                     elif format_spec == 'worstaudio':
1244                         audio_formats = [
1245                             f for f in formats
1246                             if f.get('vcodec') == 'none']
1247                         if audio_formats:
1248                             yield audio_formats[0]
1249                     elif format_spec == 'bestvideo':
1250                         video_formats = [
1251                             f for f in formats
1252                             if f.get('acodec') == 'none']
1253                         if video_formats:
1254                             yield video_formats[-1]
1255                     elif format_spec == 'worstvideo':
1256                         video_formats = [
1257                             f for f in formats
1258                             if f.get('acodec') == 'none']
1259                         if video_formats:
1260                             yield video_formats[0]
1261                     else:
1262                         extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1263                         if format_spec in extensions:
1264                             filter_f = lambda f: f['ext'] == format_spec
1265                         else:
1266                             filter_f = lambda f: f['format_id'] == format_spec
1267                         matches = list(filter(filter_f, formats))
1268                         if matches:
1269                             yield matches[-1]
1270             elif selector.type == MERGE:
1271                 def _merge(formats_info):
1272                     format_1, format_2 = [f['format_id'] for f in formats_info]
1273                     # The first format must contain the video and the
1274                     # second the audio
1275                     if formats_info[0].get('vcodec') == 'none':
1276                         self.report_error('The first format must '
1277                                           'contain the video, try using '
1278                                           '"-f %s+%s"' % (format_2, format_1))
1279                         return
1280                     # Formats must be opposite (video+audio)
1281                     if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1282                         self.report_error(
1283                             'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1284                             % (format_1, format_2))
1285                         return
1286                     output_ext = (
1287                         formats_info[0]['ext']
1288                         if self.params.get('merge_output_format') is None
1289                         else self.params['merge_output_format'])
1290                     return {
1291                         'requested_formats': formats_info,
1292                         'format': '%s+%s' % (formats_info[0].get('format'),
1293                                              formats_info[1].get('format')),
1294                         'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1295                                                 formats_info[1].get('format_id')),
1296                         'width': formats_info[0].get('width'),
1297                         'height': formats_info[0].get('height'),
1298                         'resolution': formats_info[0].get('resolution'),
1299                         'fps': formats_info[0].get('fps'),
1300                         'vcodec': formats_info[0].get('vcodec'),
1301                         'vbr': formats_info[0].get('vbr'),
1302                         'stretched_ratio': formats_info[0].get('stretched_ratio'),
1303                         'acodec': formats_info[1].get('acodec'),
1304                         'abr': formats_info[1].get('abr'),
1305                         'ext': output_ext,
1306                     }
1307                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1308
1309                 def selector_function(ctx):
1310                     for pair in itertools.product(
1311                             video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
1312                         yield _merge(pair)
1313
1314             filters = [self._build_format_filter(f) for f in selector.filters]
1315
1316             def final_selector(ctx):
1317                 ctx_copy = copy.deepcopy(ctx)
1318                 for _filter in filters:
1319                     ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
1320                 return selector_function(ctx_copy)
1321             return final_selector
1322
1323         stream = io.BytesIO(format_spec.encode('utf-8'))
1324         try:
1325             tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1326         except tokenize.TokenError:
1327             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1328
1329         class TokenIterator(object):
1330             def __init__(self, tokens):
1331                 self.tokens = tokens
1332                 self.counter = 0
1333
1334             def __iter__(self):
1335                 return self
1336
1337             def __next__(self):
1338                 if self.counter >= len(self.tokens):
1339                     raise StopIteration()
1340                 value = self.tokens[self.counter]
1341                 self.counter += 1
1342                 return value
1343
1344             next = __next__
1345
1346             def restore_last_token(self):
1347                 self.counter -= 1
1348
1349         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1350         return _build_selector_function(parsed_selector)
1351
1352     def _calc_headers(self, info_dict):
1353         res = std_headers.copy()
1354
1355         add_headers = info_dict.get('http_headers')
1356         if add_headers:
1357             res.update(add_headers)
1358
1359         cookies = self._calc_cookies(info_dict)
1360         if cookies:
1361             res['Cookie'] = cookies
1362
1363         if 'X-Forwarded-For' not in res:
1364             x_forwarded_for_ip = info_dict.get('__x_forwarded_for_ip')
1365             if x_forwarded_for_ip:
1366                 res['X-Forwarded-For'] = x_forwarded_for_ip
1367
1368         return res
1369
1370     def _calc_cookies(self, info_dict):
1371         pr = sanitized_Request(info_dict['url'])
1372         self.cookiejar.add_cookie_header(pr)
1373         return pr.get_header('Cookie')
1374
1375     def process_video_result(self, info_dict, download=True):
1376         assert info_dict.get('_type', 'video') == 'video'
1377
1378         if 'id' not in info_dict:
1379             raise ExtractorError('Missing "id" field in extractor result')
1380         if 'title' not in info_dict:
1381             raise ExtractorError('Missing "title" field in extractor result')
1382
1383         def report_force_conversion(field, field_not, conversion):
1384             self.report_warning(
1385                 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1386                 % (field, field_not, conversion))
1387
1388         def sanitize_string_field(info, string_field):
1389             field = info.get(string_field)
1390             if field is None or isinstance(field, compat_str):
1391                 return
1392             report_force_conversion(string_field, 'a string', 'string')
1393             info[string_field] = compat_str(field)
1394
1395         def sanitize_numeric_fields(info):
1396             for numeric_field in self._NUMERIC_FIELDS:
1397                 field = info.get(numeric_field)
1398                 if field is None or isinstance(field, compat_numeric_types):
1399                     continue
1400                 report_force_conversion(numeric_field, 'numeric', 'int')
1401                 info[numeric_field] = int_or_none(field)
1402
1403         sanitize_string_field(info_dict, 'id')
1404         sanitize_numeric_fields(info_dict)
1405
1406         if 'playlist' not in info_dict:
1407             # It isn't part of a playlist
1408             info_dict['playlist'] = None
1409             info_dict['playlist_index'] = None
1410
1411         thumbnails = info_dict.get('thumbnails')
1412         if thumbnails is None:
1413             thumbnail = info_dict.get('thumbnail')
1414             if thumbnail:
1415                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1416         if thumbnails:
1417             thumbnails.sort(key=lambda t: (
1418                 t.get('preference') if t.get('preference') is not None else -1,
1419                 t.get('width') if t.get('width') is not None else -1,
1420                 t.get('height') if t.get('height') is not None else -1,
1421                 t.get('id') if t.get('id') is not None else '', t.get('url')))
1422             for i, t in enumerate(thumbnails):
1423                 t['url'] = sanitize_url(t['url'])
1424                 if t.get('width') and t.get('height'):
1425                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1426                 if t.get('id') is None:
1427                     t['id'] = '%d' % i
1428
1429         if self.params.get('list_thumbnails'):
1430             self.list_thumbnails(info_dict)
1431             return
1432
1433         thumbnail = info_dict.get('thumbnail')
1434         if thumbnail:
1435             info_dict['thumbnail'] = sanitize_url(thumbnail)
1436         elif thumbnails:
1437             info_dict['thumbnail'] = thumbnails[-1]['url']
1438
1439         if 'display_id' not in info_dict and 'id' in info_dict:
1440             info_dict['display_id'] = info_dict['id']
1441
1442         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1443             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1444             # see http://bugs.python.org/issue1646728)
1445             try:
1446                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1447                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1448             except (ValueError, OverflowError, OSError):
1449                 pass
1450
1451         # Auto generate title fields corresponding to the *_number fields when missing
1452         # in order to always have clean titles. This is very common for TV series.
1453         for field in ('chapter', 'season', 'episode'):
1454             if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1455                 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1456
1457         subtitles = info_dict.get('subtitles')
1458         if subtitles:
1459             for _, subtitle in subtitles.items():
1460                 for subtitle_format in subtitle:
1461                     if subtitle_format.get('url'):
1462                         subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1463                     if subtitle_format.get('ext') is None:
1464                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1465
1466         if self.params.get('listsubtitles', False):
1467             if 'automatic_captions' in info_dict:
1468                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1469             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1470             return
1471         info_dict['requested_subtitles'] = self.process_subtitles(
1472             info_dict['id'], subtitles,
1473             info_dict.get('automatic_captions'))
1474
1475         # We now pick which formats have to be downloaded
1476         if info_dict.get('formats') is None:
1477             # There's only one format available
1478             formats = [info_dict]
1479         else:
1480             formats = info_dict['formats']
1481
1482         if not formats:
1483             raise ExtractorError('No video formats found!')
1484
1485         def is_wellformed(f):
1486             url = f.get('url')
1487             if not url:
1488                 self.report_warning(
1489                     '"url" field is missing or empty - skipping format, '
1490                     'there is an error in extractor')
1491                 return False
1492             if isinstance(url, bytes):
1493                 sanitize_string_field(f, 'url')
1494             return True
1495
1496         # Filter out malformed formats for better extraction robustness
1497         formats = list(filter(is_wellformed, formats))
1498
1499         formats_dict = {}
1500
1501         # We check that all the formats have the format and format_id fields
1502         for i, format in enumerate(formats):
1503             sanitize_string_field(format, 'format_id')
1504             sanitize_numeric_fields(format)
1505             format['url'] = sanitize_url(format['url'])
1506             if not format.get('format_id'):
1507                 format['format_id'] = compat_str(i)
1508             else:
1509                 # Sanitize format_id from characters used in format selector expression
1510                 format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
1511             format_id = format['format_id']
1512             if format_id not in formats_dict:
1513                 formats_dict[format_id] = []
1514             formats_dict[format_id].append(format)
1515
1516         # Make sure all formats have unique format_id
1517         for format_id, ambiguous_formats in formats_dict.items():
1518             if len(ambiguous_formats) > 1:
1519                 for i, format in enumerate(ambiguous_formats):
1520                     format['format_id'] = '%s-%d' % (format_id, i)
1521
1522         for i, format in enumerate(formats):
1523             if format.get('format') is None:
1524                 format['format'] = '{id} - {res}{note}'.format(
1525                     id=format['format_id'],
1526                     res=self.format_resolution(format),
1527                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1528                 )
1529             # Automatically determine file extension if missing
1530             if format.get('ext') is None:
1531                 format['ext'] = determine_ext(format['url']).lower()
1532             # Automatically determine protocol if missing (useful for format
1533             # selection purposes)
1534             if format.get('protocol') is None:
1535                 format['protocol'] = determine_protocol(format)
1536             # Add HTTP headers, so that external programs can use them from the
1537             # json output
1538             full_format_info = info_dict.copy()
1539             full_format_info.update(format)
1540             format['http_headers'] = self._calc_headers(full_format_info)
1541         # Remove private housekeeping stuff
1542         if '__x_forwarded_for_ip' in info_dict:
1543             del info_dict['__x_forwarded_for_ip']
1544
1545         # TODO Central sorting goes here
1546
1547         if formats[0] is not info_dict:
1548             # only set the 'formats' fields if the original info_dict list them
1549             # otherwise we end up with a circular reference, the first (and unique)
1550             # element in the 'formats' field in info_dict is info_dict itself,
1551             # which can't be exported to json
1552             info_dict['formats'] = formats
1553         if self.params.get('listformats'):
1554             self.list_formats(info_dict)
1555             return
1556
1557         req_format = self.params.get('format')
1558         if req_format is None:
1559             req_format = self._default_format_spec(info_dict, download=download)
1560             if self.params.get('verbose'):
1561                 self.to_stdout('[debug] Default format spec: %s' % req_format)
1562
1563         format_selector = self.build_format_selector(req_format)
1564
1565         # While in format selection we may need to have an access to the original
1566         # format set in order to calculate some metrics or do some processing.
1567         # For now we need to be able to guess whether original formats provided
1568         # by extractor are incomplete or not (i.e. whether extractor provides only
1569         # video-only or audio-only formats) for proper formats selection for
1570         # extractors with such incomplete formats (see
1571         # https://github.com/rg3/youtube-dl/pull/5556).
1572         # Since formats may be filtered during format selection and may not match
1573         # the original formats the results may be incorrect. Thus original formats
1574         # or pre-calculated metrics should be passed to format selection routines
1575         # as well.
1576         # We will pass a context object containing all necessary additional data
1577         # instead of just formats.
1578         # This fixes incorrect format selection issue (see
1579         # https://github.com/rg3/youtube-dl/issues/10083).
1580         incomplete_formats = (
1581             # All formats are video-only or
1582             all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
1583             # all formats are audio-only
1584             all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
1585
1586         ctx = {
1587             'formats': formats,
1588             'incomplete_formats': incomplete_formats,
1589         }
1590
1591         formats_to_download = list(format_selector(ctx))
1592         if not formats_to_download:
1593             raise ExtractorError('requested format not available',
1594                                  expected=True)
1595
1596         if download:
1597             if len(formats_to_download) > 1:
1598                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1599             for format in formats_to_download:
1600                 new_info = dict(info_dict)
1601                 new_info.update(format)
1602                 self.process_info(new_info)
1603         # We update the info dict with the best quality format (backwards compatibility)
1604         info_dict.update(formats_to_download[-1])
1605         return info_dict
1606
1607     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1608         """Select the requested subtitles and their format"""
1609         available_subs = {}
1610         if normal_subtitles and self.params.get('writesubtitles'):
1611             available_subs.update(normal_subtitles)
1612         if automatic_captions and self.params.get('writeautomaticsub'):
1613             for lang, cap_info in automatic_captions.items():
1614                 if lang not in available_subs:
1615                     available_subs[lang] = cap_info
1616
1617         if (not self.params.get('writesubtitles') and not
1618                 self.params.get('writeautomaticsub') or not
1619                 available_subs):
1620             return None
1621
1622         if self.params.get('allsubtitles', False):
1623             requested_langs = available_subs.keys()
1624         else:
1625             if self.params.get('subtitleslangs', False):
1626                 requested_langs = self.params.get('subtitleslangs')
1627             elif 'en' in available_subs:
1628                 requested_langs = ['en']
1629             else:
1630                 requested_langs = [list(available_subs.keys())[0]]
1631
1632         formats_query = self.params.get('subtitlesformat', 'best')
1633         formats_preference = formats_query.split('/') if formats_query else []
1634         subs = {}
1635         for lang in requested_langs:
1636             formats = available_subs.get(lang)
1637             if formats is None:
1638                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1639                 continue
1640             for ext in formats_preference:
1641                 if ext == 'best':
1642                     f = formats[-1]
1643                     break
1644                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1645                 if matches:
1646                     f = matches[-1]
1647                     break
1648             else:
1649                 f = formats[-1]
1650                 self.report_warning(
1651                     'No subtitle format found matching "%s" for language %s, '
1652                     'using %s' % (formats_query, lang, f['ext']))
1653             subs[lang] = f
1654         return subs
1655
1656     def process_info(self, info_dict):
1657         """Process a single resolved IE result."""
1658
1659         assert info_dict.get('_type', 'video') == 'video'
1660
1661         max_downloads = self.params.get('max_downloads')
1662         if max_downloads is not None:
1663             if self._num_downloads >= int(max_downloads):
1664                 raise MaxDownloadsReached()
1665
1666         info_dict['fulltitle'] = info_dict['title']
1667         if len(info_dict['title']) > 200:
1668             info_dict['title'] = info_dict['title'][:197] + '...'
1669
1670         if 'format' not in info_dict:
1671             info_dict['format'] = info_dict['ext']
1672
1673         reason = self._match_entry(info_dict, incomplete=False)
1674         if reason is not None:
1675             self.to_screen('[download] ' + reason)
1676             return
1677
1678         self._num_downloads += 1
1679
1680         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1681
1682         # Forced printings
1683         if self.params.get('forcetitle', False):
1684             self.to_stdout(info_dict['fulltitle'])
1685         if self.params.get('forceid', False):
1686             self.to_stdout(info_dict['id'])
1687         if self.params.get('forceurl', False):
1688             if info_dict.get('requested_formats') is not None:
1689                 for f in info_dict['requested_formats']:
1690                     self.to_stdout(f['url'] + f.get('play_path', ''))
1691             else:
1692                 # For RTMP URLs, also include the playpath
1693                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1694         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1695             self.to_stdout(info_dict['thumbnail'])
1696         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1697             self.to_stdout(info_dict['description'])
1698         if self.params.get('forcefilename', False) and filename is not None:
1699             self.to_stdout(filename)
1700         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1701             self.to_stdout(formatSeconds(info_dict['duration']))
1702         if self.params.get('forceformat', False):
1703             self.to_stdout(info_dict['format'])
1704         if self.params.get('forcejson', False):
1705             self.to_stdout(json.dumps(info_dict))
1706
1707         # Do nothing else if in simulate mode
1708         if self.params.get('simulate', False):
1709             return
1710
1711         if filename is None:
1712             return
1713
1714         def ensure_dir_exists(path):
1715             try:
1716                 dn = os.path.dirname(path)
1717                 if dn and not os.path.exists(dn):
1718                     os.makedirs(dn)
1719                 return True
1720             except (OSError, IOError) as err:
1721                 self.report_error('unable to create directory ' + error_to_compat_str(err))
1722                 return False
1723
1724         if not ensure_dir_exists(sanitize_path(encodeFilename(filename))):
1725             return
1726
1727         if self.params.get('writedescription', False):
1728             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1729             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1730                 self.to_screen('[info] Video description is already present')
1731             elif info_dict.get('description') is None:
1732                 self.report_warning('There\'s no description to write.')
1733             else:
1734                 try:
1735                     self.to_screen('[info] Writing video description to: ' + descfn)
1736                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1737                         descfile.write(info_dict['description'])
1738                 except (OSError, IOError):
1739                     self.report_error('Cannot write description file ' + descfn)
1740                     return
1741
1742         if self.params.get('writeannotations', False):
1743             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1744             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1745                 self.to_screen('[info] Video annotations are already present')
1746             else:
1747                 try:
1748                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1749                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1750                         annofile.write(info_dict['annotations'])
1751                 except (KeyError, TypeError):
1752                     self.report_warning('There are no annotations to write.')
1753                 except (OSError, IOError):
1754                     self.report_error('Cannot write annotations file: ' + annofn)
1755                     return
1756
1757         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1758                                        self.params.get('writeautomaticsub')])
1759
1760         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1761             # subtitles download errors are already managed as troubles in relevant IE
1762             # that way it will silently go on when used with unsupporting IE
1763             subtitles = info_dict['requested_subtitles']
1764             ie = self.get_info_extractor(info_dict['extractor_key'])
1765             for sub_lang, sub_info in subtitles.items():
1766                 sub_format = sub_info['ext']
1767                 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1768                 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1769                     self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
1770                 else:
1771                     self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1772                     if sub_info.get('data') is not None:
1773                         try:
1774                             # Use newline='' to prevent conversion of newline characters
1775                             # See https://github.com/rg3/youtube-dl/issues/10268
1776                             with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
1777                                 subfile.write(sub_info['data'])
1778                         except (OSError, IOError):
1779                             self.report_error('Cannot write subtitles file ' + sub_filename)
1780                             return
1781                     else:
1782                         try:
1783                             sub_data = ie._request_webpage(
1784                                 sub_info['url'], info_dict['id'], note=False).read()
1785                             with io.open(encodeFilename(sub_filename), 'wb') as subfile:
1786                                 subfile.write(sub_data)
1787                         except (ExtractorError, IOError, OSError, ValueError) as err:
1788                             self.report_warning('Unable to download subtitle for "%s": %s' %
1789                                                 (sub_lang, error_to_compat_str(err)))
1790                             continue
1791
1792         if self.params.get('writeinfojson', False):
1793             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1794             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1795                 self.to_screen('[info] Video description metadata is already present')
1796             else:
1797                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1798                 try:
1799                     write_json_file(self.filter_requested_info(info_dict), infofn)
1800                 except (OSError, IOError):
1801                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1802                     return
1803
1804         self._write_thumbnails(info_dict, filename)
1805
1806         if not self.params.get('skip_download', False):
1807             try:
1808                 def dl(name, info):
1809                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1810                     for ph in self._progress_hooks:
1811                         fd.add_progress_hook(ph)
1812                     if self.params.get('verbose'):
1813                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1814                     return fd.download(name, info)
1815
1816                 if info_dict.get('requested_formats') is not None:
1817                     downloaded = []
1818                     success = True
1819                     merger = FFmpegMergerPP(self)
1820                     if not merger.available:
1821                         postprocessors = []
1822                         self.report_warning('You have requested multiple '
1823                                             'formats but ffmpeg or avconv are not installed.'
1824                                             ' The formats won\'t be merged.')
1825                     else:
1826                         postprocessors = [merger]
1827
1828                     def compatible_formats(formats):
1829                         video, audio = formats
1830                         # Check extension
1831                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1832                         if video_ext and audio_ext:
1833                             COMPATIBLE_EXTS = (
1834                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'),
1835                                 ('webm')
1836                             )
1837                             for exts in COMPATIBLE_EXTS:
1838                                 if video_ext in exts and audio_ext in exts:
1839                                     return True
1840                         # TODO: Check acodec/vcodec
1841                         return False
1842
1843                     filename_real_ext = os.path.splitext(filename)[1][1:]
1844                     filename_wo_ext = (
1845                         os.path.splitext(filename)[0]
1846                         if filename_real_ext == info_dict['ext']
1847                         else filename)
1848                     requested_formats = info_dict['requested_formats']
1849                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1850                         info_dict['ext'] = 'mkv'
1851                         self.report_warning(
1852                             'Requested formats are incompatible for merge and will be merged into mkv.')
1853                     # Ensure filename always has a correct extension for successful merge
1854                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1855                     if os.path.exists(encodeFilename(filename)):
1856                         self.to_screen(
1857                             '[download] %s has already been downloaded and '
1858                             'merged' % filename)
1859                     else:
1860                         for f in requested_formats:
1861                             new_info = dict(info_dict)
1862                             new_info.update(f)
1863                             fname = prepend_extension(
1864                                 self.prepare_filename(new_info),
1865                                 'f%s' % f['format_id'], new_info['ext'])
1866                             if not ensure_dir_exists(fname):
1867                                 return
1868                             downloaded.append(fname)
1869                             partial_success = dl(fname, new_info)
1870                             success = success and partial_success
1871                         info_dict['__postprocessors'] = postprocessors
1872                         info_dict['__files_to_merge'] = downloaded
1873                 else:
1874                     # Just a single file
1875                     success = dl(filename, info_dict)
1876             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1877                 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1878                 return
1879             except (OSError, IOError) as err:
1880                 raise UnavailableVideoError(err)
1881             except (ContentTooShortError, ) as err:
1882                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1883                 return
1884
1885             if success and filename != '-':
1886                 # Fixup content
1887                 fixup_policy = self.params.get('fixup')
1888                 if fixup_policy is None:
1889                     fixup_policy = 'detect_or_warn'
1890
1891                 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1892
1893                 stretched_ratio = info_dict.get('stretched_ratio')
1894                 if stretched_ratio is not None and stretched_ratio != 1:
1895                     if fixup_policy == 'warn':
1896                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1897                             info_dict['id'], stretched_ratio))
1898                     elif fixup_policy == 'detect_or_warn':
1899                         stretched_pp = FFmpegFixupStretchedPP(self)
1900                         if stretched_pp.available:
1901                             info_dict.setdefault('__postprocessors', [])
1902                             info_dict['__postprocessors'].append(stretched_pp)
1903                         else:
1904                             self.report_warning(
1905                                 '%s: Non-uniform pixel ratio (%s). %s'
1906                                 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1907                     else:
1908                         assert fixup_policy in ('ignore', 'never')
1909
1910                 if (info_dict.get('requested_formats') is None and
1911                         info_dict.get('container') == 'm4a_dash'):
1912                     if fixup_policy == 'warn':
1913                         self.report_warning(
1914                             '%s: writing DASH m4a. '
1915                             'Only some players support this container.'
1916                             % info_dict['id'])
1917                     elif fixup_policy == 'detect_or_warn':
1918                         fixup_pp = FFmpegFixupM4aPP(self)
1919                         if fixup_pp.available:
1920                             info_dict.setdefault('__postprocessors', [])
1921                             info_dict['__postprocessors'].append(fixup_pp)
1922                         else:
1923                             self.report_warning(
1924                                 '%s: writing DASH m4a. '
1925                                 'Only some players support this container. %s'
1926                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1927                     else:
1928                         assert fixup_policy in ('ignore', 'never')
1929
1930                 if (info_dict.get('protocol') == 'm3u8_native' or
1931                         info_dict.get('protocol') == 'm3u8' and
1932                         self.params.get('hls_prefer_native')):
1933                     if fixup_policy == 'warn':
1934                         self.report_warning('%s: malformed AAC bitstream detected.' % (
1935                             info_dict['id']))
1936                     elif fixup_policy == 'detect_or_warn':
1937                         fixup_pp = FFmpegFixupM3u8PP(self)
1938                         if fixup_pp.available:
1939                             info_dict.setdefault('__postprocessors', [])
1940                             info_dict['__postprocessors'].append(fixup_pp)
1941                         else:
1942                             self.report_warning(
1943                                 '%s: malformed AAC bitstream detected. %s'
1944                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1945                     else:
1946                         assert fixup_policy in ('ignore', 'never')
1947
1948                 try:
1949                     self.post_process(filename, info_dict)
1950                 except (PostProcessingError) as err:
1951                     self.report_error('postprocessing: %s' % str(err))
1952                     return
1953                 self.record_download_archive(info_dict)
1954
1955     def download(self, url_list):
1956         """Download a given list of URLs."""
1957         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1958         if (len(url_list) > 1 and
1959                 outtmpl != '-' and
1960                 '%' not in outtmpl and
1961                 self.params.get('max_downloads') != 1):
1962             raise SameFileError(outtmpl)
1963
1964         for url in url_list:
1965             try:
1966                 # It also downloads the videos
1967                 res = self.extract_info(
1968                     url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1969             except UnavailableVideoError:
1970                 self.report_error('unable to download video')
1971             except MaxDownloadsReached:
1972                 self.to_screen('[info] Maximum number of downloaded files reached.')
1973                 raise
1974             else:
1975                 if self.params.get('dump_single_json', False):
1976                     self.to_stdout(json.dumps(res))
1977
1978         return self._download_retcode
1979
1980     def download_with_info_file(self, info_filename):
1981         with contextlib.closing(fileinput.FileInput(
1982                 [info_filename], mode='r',
1983                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1984             # FileInput doesn't have a read method, we can't call json.load
1985             info = self.filter_requested_info(json.loads('\n'.join(f)))
1986         try:
1987             self.process_ie_result(info, download=True)
1988         except DownloadError:
1989             webpage_url = info.get('webpage_url')
1990             if webpage_url is not None:
1991                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1992                 return self.download([webpage_url])
1993             else:
1994                 raise
1995         return self._download_retcode
1996
1997     @staticmethod
1998     def filter_requested_info(info_dict):
1999         return dict(
2000             (k, v) for k, v in info_dict.items()
2001             if k not in ['requested_formats', 'requested_subtitles'])
2002
2003     def post_process(self, filename, ie_info):
2004         """Run all the postprocessors on the given file."""
2005         info = dict(ie_info)
2006         info['filepath'] = filename
2007         pps_chain = []
2008         if ie_info.get('__postprocessors') is not None:
2009             pps_chain.extend(ie_info['__postprocessors'])
2010         pps_chain.extend(self._pps)
2011         for pp in pps_chain:
2012             files_to_delete = []
2013             try:
2014                 files_to_delete, info = pp.run(info)
2015             except PostProcessingError as e:
2016                 self.report_error(e.msg)
2017             if files_to_delete and not self.params.get('keepvideo', False):
2018                 for old_filename in files_to_delete:
2019                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
2020                     try:
2021                         os.remove(encodeFilename(old_filename))
2022                     except (IOError, OSError):
2023                         self.report_warning('Unable to remove downloaded original file')
2024
2025     def _make_archive_id(self, info_dict):
2026         # Future-proof against any change in case
2027         # and backwards compatibility with prior versions
2028         extractor = info_dict.get('extractor_key')
2029         if extractor is None:
2030             if 'id' in info_dict:
2031                 extractor = info_dict.get('ie_key')  # key in a playlist
2032         if extractor is None:
2033             return None  # Incomplete video information
2034         return extractor.lower() + ' ' + info_dict['id']
2035
2036     def in_download_archive(self, info_dict):
2037         fn = self.params.get('download_archive')
2038         if fn is None:
2039             return False
2040
2041         vid_id = self._make_archive_id(info_dict)
2042         if vid_id is None:
2043             return False  # Incomplete video information
2044
2045         try:
2046             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
2047                 for line in archive_file:
2048                     if line.strip() == vid_id:
2049                         return True
2050         except IOError as ioe:
2051             if ioe.errno != errno.ENOENT:
2052                 raise
2053         return False
2054
2055     def record_download_archive(self, info_dict):
2056         fn = self.params.get('download_archive')
2057         if fn is None:
2058             return
2059         vid_id = self._make_archive_id(info_dict)
2060         assert vid_id
2061         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
2062             archive_file.write(vid_id + '\n')
2063
2064     @staticmethod
2065     def format_resolution(format, default='unknown'):
2066         if format.get('vcodec') == 'none':
2067             return 'audio only'
2068         if format.get('resolution') is not None:
2069             return format['resolution']
2070         if format.get('height') is not None:
2071             if format.get('width') is not None:
2072                 res = '%sx%s' % (format['width'], format['height'])
2073             else:
2074                 res = '%sp' % format['height']
2075         elif format.get('width') is not None:
2076             res = '%dx?' % format['width']
2077         else:
2078             res = default
2079         return res
2080
2081     def _format_note(self, fdict):
2082         res = ''
2083         if fdict.get('ext') in ['f4f', 'f4m']:
2084             res += '(unsupported) '
2085         if fdict.get('language'):
2086             if res:
2087                 res += ' '
2088             res += '[%s] ' % fdict['language']
2089         if fdict.get('format_note') is not None:
2090             res += fdict['format_note'] + ' '
2091         if fdict.get('tbr') is not None:
2092             res += '%4dk ' % fdict['tbr']
2093         if fdict.get('container') is not None:
2094             if res:
2095                 res += ', '
2096             res += '%s container' % fdict['container']
2097         if (fdict.get('vcodec') is not None and
2098                 fdict.get('vcodec') != 'none'):
2099             if res:
2100                 res += ', '
2101             res += fdict['vcodec']
2102             if fdict.get('vbr') is not None:
2103                 res += '@'
2104         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
2105             res += 'video@'
2106         if fdict.get('vbr') is not None:
2107             res += '%4dk' % fdict['vbr']
2108         if fdict.get('fps') is not None:
2109             if res:
2110                 res += ', '
2111             res += '%sfps' % fdict['fps']
2112         if fdict.get('acodec') is not None:
2113             if res:
2114                 res += ', '
2115             if fdict['acodec'] == 'none':
2116                 res += 'video only'
2117             else:
2118                 res += '%-5s' % fdict['acodec']
2119         elif fdict.get('abr') is not None:
2120             if res:
2121                 res += ', '
2122             res += 'audio'
2123         if fdict.get('abr') is not None:
2124             res += '@%3dk' % fdict['abr']
2125         if fdict.get('asr') is not None:
2126             res += ' (%5dHz)' % fdict['asr']
2127         if fdict.get('filesize') is not None:
2128             if res:
2129                 res += ', '
2130             res += format_bytes(fdict['filesize'])
2131         elif fdict.get('filesize_approx') is not None:
2132             if res:
2133                 res += ', '
2134             res += '~' + format_bytes(fdict['filesize_approx'])
2135         return res
2136
2137     def list_formats(self, info_dict):
2138         formats = info_dict.get('formats', [info_dict])
2139         table = [
2140             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
2141             for f in formats
2142             if f.get('preference') is None or f['preference'] >= -1000]
2143         if len(formats) > 1:
2144             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
2145
2146         header_line = ['format code', 'extension', 'resolution', 'note']
2147         self.to_screen(
2148             '[info] Available formats for %s:\n%s' %
2149             (info_dict['id'], render_table(header_line, table)))
2150
2151     def list_thumbnails(self, info_dict):
2152         thumbnails = info_dict.get('thumbnails')
2153         if not thumbnails:
2154             self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
2155             return
2156
2157         self.to_screen(
2158             '[info] Thumbnails for %s:' % info_dict['id'])
2159         self.to_screen(render_table(
2160             ['ID', 'width', 'height', 'URL'],
2161             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
2162
2163     def list_subtitles(self, video_id, subtitles, name='subtitles'):
2164         if not subtitles:
2165             self.to_screen('%s has no %s' % (video_id, name))
2166             return
2167         self.to_screen(
2168             'Available %s for %s:' % (name, video_id))
2169         self.to_screen(render_table(
2170             ['Language', 'formats'],
2171             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
2172                 for lang, formats in subtitles.items()]))
2173
2174     def urlopen(self, req):
2175         """ Start an HTTP download """
2176         if isinstance(req, compat_basestring):
2177             req = sanitized_Request(req)
2178         return self._opener.open(req, timeout=self._socket_timeout)
2179
2180     def print_debug_header(self):
2181         if not self.params.get('verbose'):
2182             return
2183
2184         if type('') is not compat_str:
2185             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
2186             self.report_warning(
2187                 'Your Python is broken! Update to a newer and supported version')
2188
2189         stdout_encoding = getattr(
2190             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
2191         encoding_str = (
2192             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2193                 locale.getpreferredencoding(),
2194                 sys.getfilesystemencoding(),
2195                 stdout_encoding,
2196                 self.get_encoding()))
2197         write_string(encoding_str, encoding=None)
2198
2199         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
2200         if _LAZY_LOADER:
2201             self._write_string('[debug] Lazy loading extractors enabled' + '\n')
2202         try:
2203             sp = subprocess.Popen(
2204                 ['git', 'rev-parse', '--short', 'HEAD'],
2205                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2206                 cwd=os.path.dirname(os.path.abspath(__file__)))
2207             out, err = sp.communicate()
2208             out = out.decode().strip()
2209             if re.match('[0-9a-f]+', out):
2210                 self._write_string('[debug] Git HEAD: ' + out + '\n')
2211         except Exception:
2212             try:
2213                 sys.exc_clear()
2214             except Exception:
2215                 pass
2216         self._write_string('[debug] Python version %s - %s\n' % (
2217             platform.python_version(), platform_name()))
2218
2219         exe_versions = FFmpegPostProcessor.get_versions(self)
2220         exe_versions['rtmpdump'] = rtmpdump_version()
2221         exe_versions['phantomjs'] = PhantomJSwrapper._version()
2222         exe_str = ', '.join(
2223             '%s %s' % (exe, v)
2224             for exe, v in sorted(exe_versions.items())
2225             if v
2226         )
2227         if not exe_str:
2228             exe_str = 'none'
2229         self._write_string('[debug] exe versions: %s\n' % exe_str)
2230
2231         proxy_map = {}
2232         for handler in self._opener.handlers:
2233             if hasattr(handler, 'proxies'):
2234                 proxy_map.update(handler.proxies)
2235         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2236
2237         if self.params.get('call_home', False):
2238             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2239             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2240             latest_version = self.urlopen(
2241                 'https://yt-dl.org/latest/version').read().decode('utf-8')
2242             if version_tuple(latest_version) > version_tuple(__version__):
2243                 self.report_warning(
2244                     'You are using an outdated version (newest version: %s)! '
2245                     'See https://yt-dl.org/update if you need help updating.' %
2246                     latest_version)
2247
2248     def _setup_opener(self):
2249         timeout_val = self.params.get('socket_timeout')
2250         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2251
2252         opts_cookiefile = self.params.get('cookiefile')
2253         opts_proxy = self.params.get('proxy')
2254
2255         if opts_cookiefile is None:
2256             self.cookiejar = compat_cookiejar.CookieJar()
2257         else:
2258             opts_cookiefile = expand_path(opts_cookiefile)
2259             self.cookiejar = compat_cookiejar.MozillaCookieJar(
2260                 opts_cookiefile)
2261             if os.access(opts_cookiefile, os.R_OK):
2262                 self.cookiejar.load()
2263
2264         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2265         if opts_proxy is not None:
2266             if opts_proxy == '':
2267                 proxies = {}
2268             else:
2269                 proxies = {'http': opts_proxy, 'https': opts_proxy}
2270         else:
2271             proxies = compat_urllib_request.getproxies()
2272             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2273             if 'http' in proxies and 'https' not in proxies:
2274                 proxies['https'] = proxies['http']
2275         proxy_handler = PerRequestProxyHandler(proxies)
2276
2277         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2278         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2279         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2280         data_handler = compat_urllib_request_DataHandler()
2281
2282         # When passing our own FileHandler instance, build_opener won't add the
2283         # default FileHandler and allows us to disable the file protocol, which
2284         # can be used for malicious purposes (see
2285         # https://github.com/rg3/youtube-dl/issues/8227)
2286         file_handler = compat_urllib_request.FileHandler()
2287
2288         def file_open(*args, **kwargs):
2289             raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2290         file_handler.file_open = file_open
2291
2292         opener = compat_urllib_request.build_opener(
2293             proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2294
2295         # Delete the default user-agent header, which would otherwise apply in
2296         # cases where our custom HTTP handler doesn't come into play
2297         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2298         opener.addheaders = []
2299         self._opener = opener
2300
2301     def encode(self, s):
2302         if isinstance(s, bytes):
2303             return s  # Already encoded
2304
2305         try:
2306             return s.encode(self.get_encoding())
2307         except UnicodeEncodeError as err:
2308             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2309             raise
2310
2311     def get_encoding(self):
2312         encoding = self.params.get('encoding')
2313         if encoding is None:
2314             encoding = preferredencoding()
2315         return encoding
2316
2317     def _write_thumbnails(self, info_dict, filename):
2318         if self.params.get('writethumbnail', False):
2319             thumbnails = info_dict.get('thumbnails')
2320             if thumbnails:
2321                 thumbnails = [thumbnails[-1]]
2322         elif self.params.get('write_all_thumbnails', False):
2323             thumbnails = info_dict.get('thumbnails')
2324         else:
2325             return
2326
2327         if not thumbnails:
2328             # No thumbnails present, so return immediately
2329             return
2330
2331         for t in thumbnails:
2332             thumb_ext = determine_ext(t['url'], 'jpg')
2333             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2334             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2335             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2336
2337             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2338                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2339                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2340             else:
2341                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2342                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2343                 try:
2344                     uf = self.urlopen(t['url'])
2345                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2346                         shutil.copyfileobj(uf, thumbf)
2347                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2348                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2349                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2350                     self.report_warning('Unable to download thumbnail "%s": %s' %
2351                                         (t['url'], error_to_compat_str(err)))