_ Git - youtube-dl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import contextlib
   8 import datetime
   9 import errno
  10 import fileinput
  11 import io
  12 import itertools
  13 import json
  14 import locale
  15 import operator
  16 import os
  17 import platform
  18 import re
  19 import shutil
  20 import subprocess
  21 import socket
  22 import sys
  23 import time
  24 import tokenize
  25 import traceback
  26
  27 from .compat import (
  28     compat_basestring,
  29     compat_cookiejar,
  30     compat_expanduser,
  31     compat_get_terminal_size,
  32     compat_http_client,
  33     compat_kwargs,
  34     compat_os_name,
  35     compat_str,
  36     compat_tokenize_tokenize,
  37     compat_urllib_error,
  38     compat_urllib_request,
  39     compat_urllib_request_DataHandler,
  40 )
  41 from .utils import (
  42     age_restricted,
  43     args_to_str,
  44     ContentTooShortError,
  45     date_from_str,
  46     DateRange,
  47     DEFAULT_OUTTMPL,
  48     determine_ext,
  49     determine_protocol,
  50     DownloadError,
  51     encode_compat_str,
  52     encodeFilename,
  53     error_to_compat_str,
  54     ExtractorError,
  55     format_bytes,
  56     formatSeconds,
  57     locked_file,
  58     make_HTTPS_handler,
  59     MaxDownloadsReached,
  60     PagedList,
  61     parse_filesize,
  62     PerRequestProxyHandler,
  63     platform_name,
  64     PostProcessingError,
  65     preferredencoding,
  66     prepend_extension,
  67     register_socks_protocols,
  68     render_table,
  69     replace_extension,
  70     SameFileError,
  71     sanitize_filename,
  72     sanitize_path,
  73     sanitize_url,
  74     sanitized_Request,
  75     std_headers,
  76     subtitles_filename,
  77     UnavailableVideoError,
  78     url_basename,
  79     version_tuple,
  80     write_json_file,
  81     write_string,
  82     YoutubeDLCookieProcessor,
  83     YoutubeDLHandler,
  84 )
  85 from .cache import Cache
  86 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
  87 from .downloader import get_suitable_downloader
  88 from .downloader.rtmp import rtmpdump_version
  89 from .postprocessor import (
  90     FFmpegFixupM3u8PP,
  91     FFmpegFixupM4aPP,
  92     FFmpegFixupStretchedPP,
  93     FFmpegMergerPP,
  94     FFmpegPostProcessor,
  95     get_postprocessor,
  96 )
  97 from .version import __version__
  98
  99 if compat_os_name == 'nt':
 100     import ctypes
 101
 102
 103 class YoutubeDL(object):
 104     """YoutubeDL class.
 105
 106     YoutubeDL objects are the ones responsible of downloading the
 107     actual video file and writing it to disk if the user has requested
 108     it, among some other tasks. In most cases there should be one per
 109     program. As, given a video URL, the downloader doesn't know how to
 110     extract all the needed information, task that InfoExtractors do, it
 111     has to pass the URL to one of them.
 112
 113     For this, YoutubeDL objects have a method that allows
 114     InfoExtractors to be registered in a given order. When it is passed
 115     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 116     finds that reports being able to handle it. The InfoExtractor extracts
 117     all the information about the video or videos the URL refers to, and
 118     YoutubeDL process the extracted information, possibly using a File
 119     Downloader to download the video.
 120
 121     YoutubeDL objects accept a lot of parameters. In order not to saturate
 122     the object constructor with arguments, it receives a dictionary of
 123     options instead. These options are available through the params
 124     attribute for the InfoExtractors to use. The YoutubeDL also
 125     registers itself as the downloader in charge for the InfoExtractors
 126     that are added to it, so this is a "mutual registration".
 127
 128     Available options:
 129
 130     username:          Username for authentication purposes.
 131     password:          Password for authentication purposes.
 132     videopassword:     Password for accessing a video.
 133     usenetrc:          Use netrc for authentication instead.
 134     verbose:           Print additional info to stdout.
 135     quiet:             Do not print messages to stdout.
 136     no_warnings:       Do not print out anything for warnings.
 137     forceurl:          Force printing final URL.
 138     forcetitle:        Force printing title.
 139     forceid:           Force printing ID.
 140     forcethumbnail:    Force printing thumbnail URL.
 141     forcedescription:  Force printing description.
 142     forcefilename:     Force printing final filename.
 143     forceduration:     Force printing duration.
 144     forcejson:         Force printing info_dict as JSON.
 145     dump_single_json:  Force printing the info_dict of the whole playlist
 146                        (or video) as a single JSON line.
 147     simulate:          Do not download the video files.
 148     format:            Video format code. See options.py for more information.
 149     outtmpl:           Template for output names.
 150     restrictfilenames: Do not allow "&" and spaces in file names
 151     ignoreerrors:      Do not stop on download errors.
 152     force_generic_extractor: Force downloader to use the generic extractor
 153     nooverwrites:      Prevent overwriting files.
 154     playliststart:     Playlist item to start at.
 155     playlistend:       Playlist item to end at.
 156     playlist_items:    Specific indices of playlist to download.
 157     playlistreverse:   Download playlist items in reverse order.
 158     matchtitle:        Download only matching titles.
 159     rejecttitle:       Reject downloads for matching titles.
 160     logger:            Log messages to a logging.Logger instance.
 161     logtostderr:       Log messages to stderr instead of stdout.
 162     writedescription:  Write the video description to a .description file
 163     writeinfojson:     Write the video description to a .info.json file
 164     writeannotations:  Write the video annotations to a .annotations.xml file
 165     writethumbnail:    Write the thumbnail image to a file
 166     write_all_thumbnails:  Write all thumbnail formats to files
 167     writesubtitles:    Write the video subtitles to a file
 168     writeautomaticsub: Write the automatically generated subtitles to a file
 169     allsubtitles:      Downloads all the subtitles of the video
 170                        (requires writesubtitles or writeautomaticsub)
 171     listsubtitles:     Lists all available subtitles for the video
 172     subtitlesformat:   The format code for subtitles
 173     subtitleslangs:    List of languages of the subtitles to download
 174     keepvideo:         Keep the video file after post-processing
 175     daterange:         A DateRange object, download only if the upload_date is in the range.
 176     skip_download:     Skip the actual download of the video file
 177     cachedir:          Location of the cache files in the filesystem.
 178                        False to disable filesystem cache.
 179     noplaylist:        Download single video instead of a playlist if in doubt.
 180     age_limit:         An integer representing the user's age in years.
 181                        Unsuitable videos for the given age are skipped.
 182     min_views:         An integer representing the minimum view count the video
 183                        must have in order to not be skipped.
 184                        Videos without view count information are always
 185                        downloaded. None for no limit.
 186     max_views:         An integer representing the maximum view count.
 187                        Videos that are more popular than that are not
 188                        downloaded.
 189                        Videos without view count information are always
 190                        downloaded. None for no limit.
 191     download_archive:  File name of a file where all downloads are recorded.
 192                        Videos already present in the file are not downloaded
 193                        again.
 194     cookiefile:        File name where cookies should be read from and dumped to.
 195     nocheckcertificate:Do not verify SSL certificates
 196     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 197                        At the moment, this is only supported by YouTube.
 198     proxy:             URL of the proxy server to use
 199     cn_verification_proxy:  URL of the proxy to use for IP address verification
 200                        on Chinese sites. (Experimental)
 201     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 202     bidi_workaround:   Work around buggy terminals without bidirectional text
 203                        support, using fridibi
 204     debug_printtraffic:Print out sent and received HTTP traffic
 205     include_ads:       Download ads as well
 206     default_search:    Prepend this string if an input url is not valid.
 207                        'auto' for elaborate guessing
 208     encoding:          Use this encoding instead of the system-specified.
 209     extract_flat:      Do not resolve URLs, return the immediate result.
 210                        Pass in 'in_playlist' to only show this behavior for
 211                        playlist items.
 212     postprocessors:    A list of dictionaries, each with an entry
 213                        * key:  The name of the postprocessor. See
 214                                youtube_dl/postprocessor/__init__.py for a list.
 215                        as well as any further keyword arguments for the
 216                        postprocessor.
 217     progress_hooks:    A list of functions that get called on download
 218                        progress, with a dictionary with the entries
 219                        * status: One of "downloading", "error", or "finished".
 220                                  Check this first and ignore unknown values.
 221
 222                        If status is one of "downloading", or "finished", the
 223                        following properties may also be present:
 224                        * filename: The final filename (always present)
 225                        * tmpfilename: The filename we're currently writing to
 226                        * downloaded_bytes: Bytes on disk
 227                        * total_bytes: Size of the whole file, None if unknown
 228                        * total_bytes_estimate: Guess of the eventual file size,
 229                                                None if unavailable.
 230                        * elapsed: The number of seconds since download started.
 231                        * eta: The estimated time in seconds, None if unknown
 232                        * speed: The download speed in bytes/second, None if
 233                                 unknown
 234                        * fragment_index: The counter of the currently
 235                                          downloaded video fragment.
 236                        * fragment_count: The number of fragments (= individual
 237                                          files that will be merged)
 238
 239                        Progress hooks are guaranteed to be called at least once
 240                        (with status "finished") if the download is successful.
 241     merge_output_format: Extension to use when merging formats.
 242     fixup:             Automatically correct known faults of the file.
 243                        One of:
 244                        - "never": do nothing
 245                        - "warn": only emit a warning
 246                        - "detect_or_warn": check whether we can do anything
 247                                            about it, warn otherwise (default)
 248     source_address:    (Experimental) Client-side IP address to bind to.
 249     call_home:         Boolean, true iff we are allowed to contact the
 250                        youtube-dl servers for debugging.
 251     sleep_interval:    Number of seconds to sleep before each download.
 252     listformats:       Print an overview of available video formats and exit.
 253     list_thumbnails:   Print a table of all thumbnails and exit.
 254     match_filter:      A function that gets called with the info_dict of
 255                        every video.
 256                        If it returns a message, the video is ignored.
 257                        If it returns None, the video is downloaded.
 258                        match_filter_func in utils.py is one example for this.
 259     no_color:          Do not emit color codes in output.
 260
 261     The following options determine which downloader is picked:
 262     external_downloader: Executable of the external downloader to call.
 263                        None or unset for standard (built-in) downloader.
 264     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
 265                        if True, otherwise use ffmpeg/avconv if False, otherwise
 266                        use downloader suggested by extractor if None.
 267
 268     The following parameters are not used by YoutubeDL itself, they are used by
 269     the downloader (see youtube_dl/downloader/common.py):
 270     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 271     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 272     xattr_set_filesize, external_downloader_args, hls_use_mpegts.
 273
 274     The following options are used by the post processors:
 275     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 276                        otherwise prefer avconv.
 277     postprocessor_args: A list of additional command-line arguments for the
 278                         postprocessor.
 279     """
 280
 281     params = None
 282     _ies = []
 283     _pps = []
 284     _download_retcode = None
 285     _num_downloads = None
 286     _screen_file = None
 287
 288     def __init__(self, params=None, auto_init=True):
 289         """Create a FileDownloader object with the given options."""
 290         if params is None:
 291             params = {}
 292         self._ies = []
 293         self._ies_instances = {}
 294         self._pps = []
 295         self._progress_hooks = []
 296         self._download_retcode = 0
 297         self._num_downloads = 0
 298         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 299         self._err_file = sys.stderr
 300         self.params = {
 301             # Default parameters
 302             'nocheckcertificate': False,
 303         }
 304         self.params.update(params)
 305         self.cache = Cache(self)
 306
 307         if params.get('bidi_workaround', False):
 308             try:
 309                 import pty
 310                 master, slave = pty.openpty()
 311                 width = compat_get_terminal_size().columns
 312                 if width is None:
 313                     width_args = []
 314                 else:
 315                     width_args = ['-w', str(width)]
 316                 sp_kwargs = dict(
 317                     stdin=subprocess.PIPE,
 318                     stdout=slave,
 319                     stderr=self._err_file)
 320                 try:
 321                     self._output_process = subprocess.Popen(
 322                         ['bidiv'] + width_args, **sp_kwargs
 323                     )
 324                 except OSError:
 325                     self._output_process = subprocess.Popen(
 326                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 327                 self._output_channel = os.fdopen(master, 'rb')
 328             except OSError as ose:
 329                 if ose.errno == errno.ENOENT:
 330                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 331                 else:
 332                     raise
 333
 334         if (sys.version_info >= (3,) and sys.platform != 'win32' and
 335                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
 336                 not params.get('restrictfilenames', False)):
 337             # On Python 3, the Unicode filesystem API will throw errors (#1474)
 338             self.report_warning(
 339                 'Assuming --restrict-filenames since file system encoding '
 340                 'cannot encode all characters. '
 341                 'Set the LC_ALL environment variable to fix this.')
 342             self.params['restrictfilenames'] = True
 343
 344         if isinstance(params.get('outtmpl'), bytes):
 345             self.report_warning(
 346                 'Parameter outtmpl is bytes, but should be a unicode string. '
 347                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 348
 349         self._setup_opener()
 350
 351         if auto_init:
 352             self.print_debug_header()
 353             self.add_default_info_extractors()
 354
 355         for pp_def_raw in self.params.get('postprocessors', []):
 356             pp_class = get_postprocessor(pp_def_raw['key'])
 357             pp_def = dict(pp_def_raw)
 358             del pp_def['key']
 359             pp = pp_class(self, **compat_kwargs(pp_def))
 360             self.add_post_processor(pp)
 361
 362         for ph in self.params.get('progress_hooks', []):
 363             self.add_progress_hook(ph)
 364
 365         register_socks_protocols()
 366
 367     def warn_if_short_id(self, argv):
 368         # short YouTube ID starting with dash?
 369         idxs = [
 370             i for i, a in enumerate(argv)
 371             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 372         if idxs:
 373             correct_argv = (
 374                 ['youtube-dl'] +
 375                 [a for i, a in enumerate(argv) if i not in idxs] +
 376                 ['--'] + [argv[i] for i in idxs]
 377             )
 378             self.report_warning(
 379                 'Long argument string detected. '
 380                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 381                 args_to_str(correct_argv))
 382
 383     def add_info_extractor(self, ie):
 384         """Add an InfoExtractor object to the end of the list."""
 385         self._ies.append(ie)
 386         if not isinstance(ie, type):
 387             self._ies_instances[ie.ie_key()] = ie
 388             ie.set_downloader(self)
 389
 390     def get_info_extractor(self, ie_key):
 391         """
 392         Get an instance of an IE with name ie_key, it will try to get one from
 393         the _ies list, if there's no instance it will create a new one and add
 394         it to the extractor list.
 395         """
 396         ie = self._ies_instances.get(ie_key)
 397         if ie is None:
 398             ie = get_info_extractor(ie_key)()
 399             self.add_info_extractor(ie)
 400         return ie
 401
 402     def add_default_info_extractors(self):
 403         """
 404         Add the InfoExtractors returned by gen_extractors to the end of the list
 405         """
 406         for ie in gen_extractor_classes():
 407             self.add_info_extractor(ie)
 408
 409     def add_post_processor(self, pp):
 410         """Add a PostProcessor object to the end of the chain."""
 411         self._pps.append(pp)
 412         pp.set_downloader(self)
 413
 414     def add_progress_hook(self, ph):
 415         """Add the progress hook (currently only for the file downloader)"""
 416         self._progress_hooks.append(ph)
 417
 418     def _bidi_workaround(self, message):
 419         if not hasattr(self, '_output_channel'):
 420             return message
 421
 422         assert hasattr(self, '_output_process')
 423         assert isinstance(message, compat_str)
 424         line_count = message.count('\n') + 1
 425         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 426         self._output_process.stdin.flush()
 427         res = ''.join(self._output_channel.readline().decode('utf-8')
 428                       for _ in range(line_count))
 429         return res[:-len('\n')]
 430
 431     def to_screen(self, message, skip_eol=False):
 432         """Print message to stdout if not in quiet mode."""
 433         return self.to_stdout(message, skip_eol, check_quiet=True)
 434
 435     def _write_string(self, s, out=None):
 436         write_string(s, out=out, encoding=self.params.get('encoding'))
 437
 438     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 439         """Print message to stdout if not in quiet mode."""
 440         if self.params.get('logger'):
 441             self.params['logger'].debug(message)
 442         elif not check_quiet or not self.params.get('quiet', False):
 443             message = self._bidi_workaround(message)
 444             terminator = ['\n', ''][skip_eol]
 445             output = message + terminator
 446
 447             self._write_string(output, self._screen_file)
 448
 449     def to_stderr(self, message):
 450         """Print message to stderr."""
 451         assert isinstance(message, compat_str)
 452         if self.params.get('logger'):
 453             self.params['logger'].error(message)
 454         else:
 455             message = self._bidi_workaround(message)
 456             output = message + '\n'
 457             self._write_string(output, self._err_file)
 458
 459     def to_console_title(self, message):
 460         if not self.params.get('consoletitle', False):
 461             return
 462         if compat_os_name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 463             # c_wchar_p() might not be necessary if `message` is
 464             # already of type unicode()
 465             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 466         elif 'TERM' in os.environ:
 467             self._write_string('\033]0;%s\007' % message, self._screen_file)
 468
 469     def save_console_title(self):
 470         if not self.params.get('consoletitle', False):
 471             return
 472         if 'TERM' in os.environ:
 473             # Save the title on stack
 474             self._write_string('\033[22;0t', self._screen_file)
 475
 476     def restore_console_title(self):
 477         if not self.params.get('consoletitle', False):
 478             return
 479         if 'TERM' in os.environ:
 480             # Restore the title from stack
 481             self._write_string('\033[23;0t', self._screen_file)
 482
 483     def __enter__(self):
 484         self.save_console_title()
 485         return self
 486
 487     def __exit__(self, *args):
 488         self.restore_console_title()
 489
 490         if self.params.get('cookiefile') is not None:
 491             self.cookiejar.save()
 492
 493     def trouble(self, message=None, tb=None):
 494         """Determine action to take when a download problem appears.
 495
 496         Depending on if the downloader has been configured to ignore
 497         download errors or not, this method may throw an exception or
 498         not when errors are found, after printing the message.
 499
 500         tb, if given, is additional traceback information.
 501         """
 502         if message is not None:
 503             self.to_stderr(message)
 504         if self.params.get('verbose'):
 505             if tb is None:
 506                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 507                     tb = ''
 508                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 509                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 510                     tb += encode_compat_str(traceback.format_exc())
 511                 else:
 512                     tb_data = traceback.format_list(traceback.extract_stack())
 513                     tb = ''.join(tb_data)
 514             self.to_stderr(tb)
 515         if not self.params.get('ignoreerrors', False):
 516             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 517                 exc_info = sys.exc_info()[1].exc_info
 518             else:
 519                 exc_info = sys.exc_info()
 520             raise DownloadError(message, exc_info)
 521         self._download_retcode = 1
 522
 523     def report_warning(self, message):
 524         '''
 525         Print the message to stderr, it will be prefixed with 'WARNING:'
 526         If stderr is a tty file the 'WARNING:' will be colored
 527         '''
 528         if self.params.get('logger') is not None:
 529             self.params['logger'].warning(message)
 530         else:
 531             if self.params.get('no_warnings'):
 532                 return
 533             if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 534                 _msg_header = '\033[0;33mWARNING:\033[0m'
 535             else:
 536                 _msg_header = 'WARNING:'
 537             warning_message = '%s %s' % (_msg_header, message)
 538             self.to_stderr(warning_message)
 539
 540     def report_error(self, message, tb=None):
 541         '''
 542         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 543         in red if stderr is a tty file.
 544         '''
 545         if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 546             _msg_header = '\033[0;31mERROR:\033[0m'
 547         else:
 548             _msg_header = 'ERROR:'
 549         error_message = '%s %s' % (_msg_header, message)
 550         self.trouble(error_message, tb)
 551
 552     def report_file_already_downloaded(self, file_name):
 553         """Report file has already been fully downloaded."""
 554         try:
 555             self.to_screen('[download] %s has already been downloaded' % file_name)
 556         except UnicodeEncodeError:
 557             self.to_screen('[download] The file has already been downloaded')
 558
 559     def prepare_filename(self, info_dict):
 560         """Generate the output filename."""
 561         try:
 562             template_dict = dict(info_dict)
 563
 564             template_dict['epoch'] = int(time.time())
 565             autonumber_size = self.params.get('autonumber_size')
 566             if autonumber_size is None:
 567                 autonumber_size = 5
 568             autonumber_templ = '%0' + str(autonumber_size) + 'd'
 569             template_dict['autonumber'] = autonumber_templ % self._num_downloads
 570             if template_dict.get('playlist_index') is not None:
 571                 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
 572             if template_dict.get('resolution') is None:
 573                 if template_dict.get('width') and template_dict.get('height'):
 574                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 575                 elif template_dict.get('height'):
 576                     template_dict['resolution'] = '%sp' % template_dict['height']
 577                 elif template_dict.get('width'):
 578                     template_dict['resolution'] = '%dx?' % template_dict['width']
 579
 580             sanitize = lambda k, v: sanitize_filename(
 581                 compat_str(v),
 582                 restricted=self.params.get('restrictfilenames'),
 583                 is_id=(k == 'id'))
 584             template_dict = dict((k, sanitize(k, v))
 585                                  for k, v in template_dict.items()
 586                                  if v is not None and not isinstance(v, (list, tuple, dict)))
 587             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 588
 589             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 590             tmpl = compat_expanduser(outtmpl)
 591             filename = tmpl % template_dict
 592             # Temporary fix for #4787
 593             # 'Treat' all problem characters by passing filename through preferredencoding
 594             # to workaround encoding issues with subprocess on python2 @ Windows
 595             if sys.version_info < (3, 0) and sys.platform == 'win32':
 596                 filename = encodeFilename(filename, True).decode(preferredencoding())
 597             return sanitize_path(filename)
 598         except ValueError as err:
 599             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 600             return None
 601
 602     def _match_entry(self, info_dict, incomplete):
 603         """ Returns None iff the file should be downloaded """
 604
 605         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 606         if 'title' in info_dict:
 607             # This can happen when we're just evaluating the playlist
 608             title = info_dict['title']
 609             matchtitle = self.params.get('matchtitle', False)
 610             if matchtitle:
 611                 if not re.search(matchtitle, title, re.IGNORECASE):
 612                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 613             rejecttitle = self.params.get('rejecttitle', False)
 614             if rejecttitle:
 615                 if re.search(rejecttitle, title, re.IGNORECASE):
 616                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 617         date = info_dict.get('upload_date')
 618         if date is not None:
 619             dateRange = self.params.get('daterange', DateRange())
 620             if date not in dateRange:
 621                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 622         view_count = info_dict.get('view_count')
 623         if view_count is not None:
 624             min_views = self.params.get('min_views')
 625             if min_views is not None and view_count < min_views:
 626                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 627             max_views = self.params.get('max_views')
 628             if max_views is not None and view_count > max_views:
 629                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 630         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 631             return 'Skipping "%s" because it is age restricted' % video_title
 632         if self.in_download_archive(info_dict):
 633             return '%s has already been recorded in archive' % video_title
 634
 635         if not incomplete:
 636             match_filter = self.params.get('match_filter')
 637             if match_filter is not None:
 638                 ret = match_filter(info_dict)
 639                 if ret is not None:
 640                     return ret
 641
 642         return None
 643
 644     @staticmethod
 645     def add_extra_info(info_dict, extra_info):
 646         '''Set the keys from extra_info in info dict if they are missing'''
 647         for key, value in extra_info.items():
 648             info_dict.setdefault(key, value)
 649
 650     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 651                      process=True, force_generic_extractor=False):
 652         '''
 653         Returns a list with a dictionary for each video we find.
 654         If 'download', also downloads the videos.
 655         extra_info is a dict containing the extra values to add to each result
 656         '''
 657
 658         if not ie_key and force_generic_extractor:
 659             ie_key = 'Generic'
 660
 661         if ie_key:
 662             ies = [self.get_info_extractor(ie_key)]
 663         else:
 664             ies = self._ies
 665
 666         for ie in ies:
 667             if not ie.suitable(url):
 668                 continue
 669
 670             ie = self.get_info_extractor(ie.ie_key())
 671             if not ie.working():
 672                 self.report_warning('The program functionality for this site has been marked as broken, '
 673                                     'and will probably not work.')
 674
 675             try:
 676                 ie_result = ie.extract(url)
 677                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 678                     break
 679                 if isinstance(ie_result, list):
 680                     # Backwards compatibility: old IE result format
 681                     ie_result = {
 682                         '_type': 'compat_list',
 683                         'entries': ie_result,
 684                     }
 685                 self.add_default_extra_info(ie_result, ie, url)
 686                 if process:
 687                     return self.process_ie_result(ie_result, download, extra_info)
 688                 else:
 689                     return ie_result
 690             except ExtractorError as e:  # An error we somewhat expected
 691                 self.report_error(compat_str(e), e.format_traceback())
 692                 break
 693             except MaxDownloadsReached:
 694                 raise
 695             except Exception as e:
 696                 if self.params.get('ignoreerrors', False):
 697                     self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
 698                     break
 699                 else:
 700                     raise
 701         else:
 702             self.report_error('no suitable InfoExtractor for URL %s' % url)
 703
 704     def add_default_extra_info(self, ie_result, ie, url):
 705         self.add_extra_info(ie_result, {
 706             'extractor': ie.IE_NAME,
 707             'webpage_url': url,
 708             'webpage_url_basename': url_basename(url),
 709             'extractor_key': ie.ie_key(),
 710         })
 711
 712     def process_ie_result(self, ie_result, download=True, extra_info={}):
 713         """
 714         Take the result of the ie(may be modified) and resolve all unresolved
 715         references (URLs, playlist items).
 716
 717         It will also download the videos if 'download'.
 718         Returns the resolved ie_result.
 719         """
 720         result_type = ie_result.get('_type', 'video')
 721
 722         if result_type in ('url', 'url_transparent'):
 723             ie_result['url'] = sanitize_url(ie_result['url'])
 724             extract_flat = self.params.get('extract_flat', False)
 725             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 726                     extract_flat is True):
 727                 if self.params.get('forcejson', False):
 728                     self.to_stdout(json.dumps(ie_result))
 729                 return ie_result
 730
 731         if result_type == 'video':
 732             self.add_extra_info(ie_result, extra_info)
 733             return self.process_video_result(ie_result, download=download)
 734         elif result_type == 'url':
 735             # We have to add extra_info to the results because it may be
 736             # contained in a playlist
 737             return self.extract_info(ie_result['url'],
 738                                      download,
 739                                      ie_key=ie_result.get('ie_key'),
 740                                      extra_info=extra_info)
 741         elif result_type == 'url_transparent':
 742             # Use the information from the embedding page
 743             info = self.extract_info(
 744                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 745                 extra_info=extra_info, download=False, process=False)
 746
 747             force_properties = dict(
 748                 (k, v) for k, v in ie_result.items() if v is not None)
 749             for f in ('_type', 'url', 'ie_key'):
 750                 if f in force_properties:
 751                     del force_properties[f]
 752             new_result = info.copy()
 753             new_result.update(force_properties)
 754
 755             assert new_result.get('_type') != 'url_transparent'
 756
 757             return self.process_ie_result(
 758                 new_result, download=download, extra_info=extra_info)
 759         elif result_type == 'playlist' or result_type == 'multi_video':
 760             # We process each entry in the playlist
 761             playlist = ie_result.get('title') or ie_result.get('id')
 762             self.to_screen('[download] Downloading playlist: %s' % playlist)
 763
 764             playlist_results = []
 765
 766             playliststart = self.params.get('playliststart', 1) - 1
 767             playlistend = self.params.get('playlistend')
 768             # For backwards compatibility, interpret -1 as whole list
 769             if playlistend == -1:
 770                 playlistend = None
 771
 772             playlistitems_str = self.params.get('playlist_items')
 773             playlistitems = None
 774             if playlistitems_str is not None:
 775                 def iter_playlistitems(format):
 776                     for string_segment in format.split(','):
 777                         if '-' in string_segment:
 778                             start, end = string_segment.split('-')
 779                             for item in range(int(start), int(end) + 1):
 780                                 yield int(item)
 781                         else:
 782                             yield int(string_segment)
 783                 playlistitems = iter_playlistitems(playlistitems_str)
 784
 785             ie_entries = ie_result['entries']
 786             if isinstance(ie_entries, list):
 787                 n_all_entries = len(ie_entries)
 788                 if playlistitems:
 789                     entries = [
 790                         ie_entries[i - 1] for i in playlistitems
 791                         if -n_all_entries <= i - 1 < n_all_entries]
 792                 else:
 793                     entries = ie_entries[playliststart:playlistend]
 794                 n_entries = len(entries)
 795                 self.to_screen(
 796                     '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
 797                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 798             elif isinstance(ie_entries, PagedList):
 799                 if playlistitems:
 800                     entries = []
 801                     for item in playlistitems:
 802                         entries.extend(ie_entries.getslice(
 803                             item - 1, item
 804                         ))
 805                 else:
 806                     entries = ie_entries.getslice(
 807                         playliststart, playlistend)
 808                 n_entries = len(entries)
 809                 self.to_screen(
 810                     '[%s] playlist %s: Downloading %d videos' %
 811                     (ie_result['extractor'], playlist, n_entries))
 812             else:  # iterable
 813                 if playlistitems:
 814                     entry_list = list(ie_entries)
 815                     entries = [entry_list[i - 1] for i in playlistitems]
 816                 else:
 817                     entries = list(itertools.islice(
 818                         ie_entries, playliststart, playlistend))
 819                 n_entries = len(entries)
 820                 self.to_screen(
 821                     '[%s] playlist %s: Downloading %d videos' %
 822                     (ie_result['extractor'], playlist, n_entries))
 823
 824             if self.params.get('playlistreverse', False):
 825                 entries = entries[::-1]
 826
 827             for i, entry in enumerate(entries, 1):
 828                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 829                 extra = {
 830                     'n_entries': n_entries,
 831                     'playlist': playlist,
 832                     'playlist_id': ie_result.get('id'),
 833                     'playlist_title': ie_result.get('title'),
 834                     'playlist_index': i + playliststart,
 835                     'extractor': ie_result['extractor'],
 836                     'webpage_url': ie_result['webpage_url'],
 837                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 838                     'extractor_key': ie_result['extractor_key'],
 839                 }
 840
 841                 reason = self._match_entry(entry, incomplete=True)
 842                 if reason is not None:
 843                     self.to_screen('[download] ' + reason)
 844                     continue
 845
 846                 entry_result = self.process_ie_result(entry,
 847                                                       download=download,
 848                                                       extra_info=extra)
 849                 playlist_results.append(entry_result)
 850             ie_result['entries'] = playlist_results
 851             self.to_screen('[download] Finished downloading playlist: %s' % playlist)
 852             return ie_result
 853         elif result_type == 'compat_list':
 854             self.report_warning(
 855                 'Extractor %s returned a compat_list result. '
 856                 'It needs to be updated.' % ie_result.get('extractor'))
 857
 858             def _fixup(r):
 859                 self.add_extra_info(
 860                     r,
 861                     {
 862                         'extractor': ie_result['extractor'],
 863                         'webpage_url': ie_result['webpage_url'],
 864                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 865                         'extractor_key': ie_result['extractor_key'],
 866                     }
 867                 )
 868                 return r
 869             ie_result['entries'] = [
 870                 self.process_ie_result(_fixup(r), download, extra_info)
 871                 for r in ie_result['entries']
 872             ]
 873             return ie_result
 874         else:
 875             raise Exception('Invalid result type: %s' % result_type)
 876
 877     def _build_format_filter(self, filter_spec):
 878         " Returns a function to filter the formats according to the filter_spec "
 879
 880         OPERATORS = {
 881             '<': operator.lt,
 882             '<=': operator.le,
 883             '>': operator.gt,
 884             '>=': operator.ge,
 885             '=': operator.eq,
 886             '!=': operator.ne,
 887         }
 888         operator_rex = re.compile(r'''(?x)\s*
 889             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
 890             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
 891             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
 892             $
 893             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
 894         m = operator_rex.search(filter_spec)
 895         if m:
 896             try:
 897                 comparison_value = int(m.group('value'))
 898             except ValueError:
 899                 comparison_value = parse_filesize(m.group('value'))
 900                 if comparison_value is None:
 901                     comparison_value = parse_filesize(m.group('value') + 'B')
 902                 if comparison_value is None:
 903                     raise ValueError(
 904                         'Invalid value %r in format specification %r' % (
 905                             m.group('value'), filter_spec))
 906             op = OPERATORS[m.group('op')]
 907
 908         if not m:
 909             STR_OPERATORS = {
 910                 '=': operator.eq,
 911                 '!=': operator.ne,
 912                 '^=': lambda attr, value: attr.startswith(value),
 913                 '$=': lambda attr, value: attr.endswith(value),
 914                 '*=': lambda attr, value: value in attr,
 915             }
 916             str_operator_rex = re.compile(r'''(?x)
 917                 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
 918                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
 919                 \s*(?P<value>[a-zA-Z0-9._-]+)
 920                 \s*$
 921                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
 922             m = str_operator_rex.search(filter_spec)
 923             if m:
 924                 comparison_value = m.group('value')
 925                 op = STR_OPERATORS[m.group('op')]
 926
 927         if not m:
 928             raise ValueError('Invalid filter specification %r' % filter_spec)
 929
 930         def _filter(f):
 931             actual_value = f.get(m.group('key'))
 932             if actual_value is None:
 933                 return m.group('none_inclusive')
 934             return op(actual_value, comparison_value)
 935         return _filter
 936
 937     def build_format_selector(self, format_spec):
 938         def syntax_error(note, start):
 939             message = (
 940                 'Invalid format specification: '
 941                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
 942             return SyntaxError(message)
 943
 944         PICKFIRST = 'PICKFIRST'
 945         MERGE = 'MERGE'
 946         SINGLE = 'SINGLE'
 947         GROUP = 'GROUP'
 948         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
 949
 950         def _parse_filter(tokens):
 951             filter_parts = []
 952             for type, string, start, _, _ in tokens:
 953                 if type == tokenize.OP and string == ']':
 954                     return ''.join(filter_parts)
 955                 else:
 956                     filter_parts.append(string)
 957
 958         def _remove_unused_ops(tokens):
 959             # Remove operators that we don't use and join them with the surrounding strings
 960             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
 961             ALLOWED_OPS = ('/', '+', ',', '(', ')')
 962             last_string, last_start, last_end, last_line = None, None, None, None
 963             for type, string, start, end, line in tokens:
 964                 if type == tokenize.OP and string == '[':
 965                     if last_string:
 966                         yield tokenize.NAME, last_string, last_start, last_end, last_line
 967                         last_string = None
 968                     yield type, string, start, end, line
 969                     # everything inside brackets will be handled by _parse_filter
 970                     for type, string, start, end, line in tokens:
 971                         yield type, string, start, end, line
 972                         if type == tokenize.OP and string == ']':
 973                             break
 974                 elif type == tokenize.OP and string in ALLOWED_OPS:
 975                     if last_string:
 976                         yield tokenize.NAME, last_string, last_start, last_end, last_line
 977                         last_string = None
 978                     yield type, string, start, end, line
 979                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
 980                     if not last_string:
 981                         last_string = string
 982                         last_start = start
 983                         last_end = end
 984                     else:
 985                         last_string += string
 986             if last_string:
 987                 yield tokenize.NAME, last_string, last_start, last_end, last_line
 988
 989         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
 990             selectors = []
 991             current_selector = None
 992             for type, string, start, _, _ in tokens:
 993                 # ENCODING is only defined in python 3.x
 994                 if type == getattr(tokenize, 'ENCODING', None):
 995                     continue
 996                 elif type in [tokenize.NAME, tokenize.NUMBER]:
 997                     current_selector = FormatSelector(SINGLE, string, [])
 998                 elif type == tokenize.OP:
 999                     if string == ')':
1000                         if not inside_group:
1001                             # ')' will be handled by the parentheses group
1002                             tokens.restore_last_token()
1003                         break
1004                     elif inside_merge and string in ['/', ',']:
1005                         tokens.restore_last_token()
1006                         break
1007                     elif inside_choice and string == ',':
1008                         tokens.restore_last_token()
1009                         break
1010                     elif string == ',':
1011                         if not current_selector:
1012                             raise syntax_error('"," must follow a format selector', start)
1013                         selectors.append(current_selector)
1014                         current_selector = None
1015                     elif string == '/':
1016                         if not current_selector:
1017                             raise syntax_error('"/" must follow a format selector', start)
1018                         first_choice = current_selector
1019                         second_choice = _parse_format_selection(tokens, inside_choice=True)
1020                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1021                     elif string == '[':
1022                         if not current_selector:
1023                             current_selector = FormatSelector(SINGLE, 'best', [])
1024                         format_filter = _parse_filter(tokens)
1025                         current_selector.filters.append(format_filter)
1026                     elif string == '(':
1027                         if current_selector:
1028                             raise syntax_error('Unexpected "("', start)
1029                         group = _parse_format_selection(tokens, inside_group=True)
1030                         current_selector = FormatSelector(GROUP, group, [])
1031                     elif string == '+':
1032                         video_selector = current_selector
1033                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
1034                         if not video_selector or not audio_selector:
1035                             raise syntax_error('"+" must be between two format selectors', start)
1036                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1037                     else:
1038                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1039                 elif type == tokenize.ENDMARKER:
1040                     break
1041             if current_selector:
1042                 selectors.append(current_selector)
1043             return selectors
1044
1045         def _build_selector_function(selector):
1046             if isinstance(selector, list):
1047                 fs = [_build_selector_function(s) for s in selector]
1048
1049                 def selector_function(formats):
1050                     for f in fs:
1051                         for format in f(formats):
1052                             yield format
1053                 return selector_function
1054             elif selector.type == GROUP:
1055                 selector_function = _build_selector_function(selector.selector)
1056             elif selector.type == PICKFIRST:
1057                 fs = [_build_selector_function(s) for s in selector.selector]
1058
1059                 def selector_function(formats):
1060                     for f in fs:
1061                         picked_formats = list(f(formats))
1062                         if picked_formats:
1063                             return picked_formats
1064                     return []
1065             elif selector.type == SINGLE:
1066                 format_spec = selector.selector
1067
1068                 def selector_function(formats):
1069                     formats = list(formats)
1070                     if not formats:
1071                         return
1072                     if format_spec == 'all':
1073                         for f in formats:
1074                             yield f
1075                     elif format_spec in ['best', 'worst', None]:
1076                         format_idx = 0 if format_spec == 'worst' else -1
1077                         audiovideo_formats = [
1078                             f for f in formats
1079                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1080                         if audiovideo_formats:
1081                             yield audiovideo_formats[format_idx]
1082                         # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
1083                         elif (all(f.get('acodec') != 'none' for f in formats) or
1084                               all(f.get('vcodec') != 'none' for f in formats)):
1085                             yield formats[format_idx]
1086                     elif format_spec == 'bestaudio':
1087                         audio_formats = [
1088                             f for f in formats
1089                             if f.get('vcodec') == 'none']
1090                         if audio_formats:
1091                             yield audio_formats[-1]
1092                     elif format_spec == 'worstaudio':
1093                         audio_formats = [
1094                             f for f in formats
1095                             if f.get('vcodec') == 'none']
1096                         if audio_formats:
1097                             yield audio_formats[0]
1098                     elif format_spec == 'bestvideo':
1099                         video_formats = [
1100                             f for f in formats
1101                             if f.get('acodec') == 'none']
1102                         if video_formats:
1103                             yield video_formats[-1]
1104                     elif format_spec == 'worstvideo':
1105                         video_formats = [
1106                             f for f in formats
1107                             if f.get('acodec') == 'none']
1108                         if video_formats:
1109                             yield video_formats[0]
1110                     else:
1111                         extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1112                         if format_spec in extensions:
1113                             filter_f = lambda f: f['ext'] == format_spec
1114                         else:
1115                             filter_f = lambda f: f['format_id'] == format_spec
1116                         matches = list(filter(filter_f, formats))
1117                         if matches:
1118                             yield matches[-1]
1119             elif selector.type == MERGE:
1120                 def _merge(formats_info):
1121                     format_1, format_2 = [f['format_id'] for f in formats_info]
1122                     # The first format must contain the video and the
1123                     # second the audio
1124                     if formats_info[0].get('vcodec') == 'none':
1125                         self.report_error('The first format must '
1126                                           'contain the video, try using '
1127                                           '"-f %s+%s"' % (format_2, format_1))
1128                         return
1129                     # Formats must be opposite (video+audio)
1130                     if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1131                         self.report_error(
1132                             'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1133                             % (format_1, format_2))
1134                         return
1135                     output_ext = (
1136                         formats_info[0]['ext']
1137                         if self.params.get('merge_output_format') is None
1138                         else self.params['merge_output_format'])
1139                     return {
1140                         'requested_formats': formats_info,
1141                         'format': '%s+%s' % (formats_info[0].get('format'),
1142                                              formats_info[1].get('format')),
1143                         'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1144                                                 formats_info[1].get('format_id')),
1145                         'width': formats_info[0].get('width'),
1146                         'height': formats_info[0].get('height'),
1147                         'resolution': formats_info[0].get('resolution'),
1148                         'fps': formats_info[0].get('fps'),
1149                         'vcodec': formats_info[0].get('vcodec'),
1150                         'vbr': formats_info[0].get('vbr'),
1151                         'stretched_ratio': formats_info[0].get('stretched_ratio'),
1152                         'acodec': formats_info[1].get('acodec'),
1153                         'abr': formats_info[1].get('abr'),
1154                         'ext': output_ext,
1155                     }
1156                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1157
1158                 def selector_function(formats):
1159                     formats = list(formats)
1160                     for pair in itertools.product(video_selector(formats), audio_selector(formats)):
1161                         yield _merge(pair)
1162
1163             filters = [self._build_format_filter(f) for f in selector.filters]
1164
1165             def final_selector(formats):
1166                 for _filter in filters:
1167                     formats = list(filter(_filter, formats))
1168                 return selector_function(formats)
1169             return final_selector
1170
1171         stream = io.BytesIO(format_spec.encode('utf-8'))
1172         try:
1173             tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1174         except tokenize.TokenError:
1175             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1176
1177         class TokenIterator(object):
1178             def __init__(self, tokens):
1179                 self.tokens = tokens
1180                 self.counter = 0
1181
1182             def __iter__(self):
1183                 return self
1184
1185             def __next__(self):
1186                 if self.counter >= len(self.tokens):
1187                     raise StopIteration()
1188                 value = self.tokens[self.counter]
1189                 self.counter += 1
1190                 return value
1191
1192             next = __next__
1193
1194             def restore_last_token(self):
1195                 self.counter -= 1
1196
1197         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1198         return _build_selector_function(parsed_selector)
1199
1200     def _calc_headers(self, info_dict):
1201         res = std_headers.copy()
1202
1203         add_headers = info_dict.get('http_headers')
1204         if add_headers:
1205             res.update(add_headers)
1206
1207         cookies = self._calc_cookies(info_dict)
1208         if cookies:
1209             res['Cookie'] = cookies
1210
1211         return res
1212
1213     def _calc_cookies(self, info_dict):
1214         pr = sanitized_Request(info_dict['url'])
1215         self.cookiejar.add_cookie_header(pr)
1216         return pr.get_header('Cookie')
1217
1218     def process_video_result(self, info_dict, download=True):
1219         assert info_dict.get('_type', 'video') == 'video'
1220
1221         if 'id' not in info_dict:
1222             raise ExtractorError('Missing "id" field in extractor result')
1223         if 'title' not in info_dict:
1224             raise ExtractorError('Missing "title" field in extractor result')
1225
1226         if not isinstance(info_dict['id'], compat_str):
1227             self.report_warning('"id" field is not a string - forcing string conversion')
1228             info_dict['id'] = compat_str(info_dict['id'])
1229
1230         if 'playlist' not in info_dict:
1231             # It isn't part of a playlist
1232             info_dict['playlist'] = None
1233             info_dict['playlist_index'] = None
1234
1235         thumbnails = info_dict.get('thumbnails')
1236         if thumbnails is None:
1237             thumbnail = info_dict.get('thumbnail')
1238             if thumbnail:
1239                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1240         if thumbnails:
1241             thumbnails.sort(key=lambda t: (
1242                 t.get('preference'), t.get('width'), t.get('height'),
1243                 t.get('id'), t.get('url')))
1244             for i, t in enumerate(thumbnails):
1245                 t['url'] = sanitize_url(t['url'])
1246                 if t.get('width') and t.get('height'):
1247                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1248                 if t.get('id') is None:
1249                     t['id'] = '%d' % i
1250
1251         if self.params.get('list_thumbnails'):
1252             self.list_thumbnails(info_dict)
1253             return
1254
1255         thumbnail = info_dict.get('thumbnail')
1256         if thumbnail:
1257             info_dict['thumbnail'] = sanitize_url(thumbnail)
1258         elif thumbnails:
1259             info_dict['thumbnail'] = thumbnails[-1]['url']
1260
1261         if 'display_id' not in info_dict and 'id' in info_dict:
1262             info_dict['display_id'] = info_dict['id']
1263
1264         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1265             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1266             # see http://bugs.python.org/issue1646728)
1267             try:
1268                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1269                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1270             except (ValueError, OverflowError, OSError):
1271                 pass
1272
1273         # Auto generate title fields corresponding to the *_number fields when missing
1274         # in order to always have clean titles. This is very common for TV series.
1275         for field in ('chapter', 'season', 'episode'):
1276             if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1277                 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1278
1279         subtitles = info_dict.get('subtitles')
1280         if subtitles:
1281             for _, subtitle in subtitles.items():
1282                 for subtitle_format in subtitle:
1283                     if subtitle_format.get('url'):
1284                         subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1285                     if 'ext' not in subtitle_format:
1286                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1287
1288         if self.params.get('listsubtitles', False):
1289             if 'automatic_captions' in info_dict:
1290                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1291             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1292             return
1293         info_dict['requested_subtitles'] = self.process_subtitles(
1294             info_dict['id'], subtitles,
1295             info_dict.get('automatic_captions'))
1296
1297         # We now pick which formats have to be downloaded
1298         if info_dict.get('formats') is None:
1299             # There's only one format available
1300             formats = [info_dict]
1301         else:
1302             formats = info_dict['formats']
1303
1304         if not formats:
1305             raise ExtractorError('No video formats found!')
1306
1307         formats_dict = {}
1308
1309         # We check that all the formats have the format and format_id fields
1310         for i, format in enumerate(formats):
1311             if 'url' not in format:
1312                 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1313
1314             format['url'] = sanitize_url(format['url'])
1315
1316             if format.get('format_id') is None:
1317                 format['format_id'] = compat_str(i)
1318             else:
1319                 # Sanitize format_id from characters used in format selector expression
1320                 format['format_id'] = re.sub('[\s,/+\[\]()]', '_', format['format_id'])
1321             format_id = format['format_id']
1322             if format_id not in formats_dict:
1323                 formats_dict[format_id] = []
1324             formats_dict[format_id].append(format)
1325
1326         # Make sure all formats have unique format_id
1327         for format_id, ambiguous_formats in formats_dict.items():
1328             if len(ambiguous_formats) > 1:
1329                 for i, format in enumerate(ambiguous_formats):
1330                     format['format_id'] = '%s-%d' % (format_id, i)
1331
1332         for i, format in enumerate(formats):
1333             if format.get('format') is None:
1334                 format['format'] = '{id} - {res}{note}'.format(
1335                     id=format['format_id'],
1336                     res=self.format_resolution(format),
1337                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1338                 )
1339             # Automatically determine file extension if missing
1340             if 'ext' not in format:
1341                 format['ext'] = determine_ext(format['url']).lower()
1342             # Automatically determine protocol if missing (useful for format
1343             # selection purposes)
1344             if 'protocol' not in format:
1345                 format['protocol'] = determine_protocol(format)
1346             # Add HTTP headers, so that external programs can use them from the
1347             # json output
1348             full_format_info = info_dict.copy()
1349             full_format_info.update(format)
1350             format['http_headers'] = self._calc_headers(full_format_info)
1351
1352         # TODO Central sorting goes here
1353
1354         if formats[0] is not info_dict:
1355             # only set the 'formats' fields if the original info_dict list them
1356             # otherwise we end up with a circular reference, the first (and unique)
1357             # element in the 'formats' field in info_dict is info_dict itself,
1358             # which can't be exported to json
1359             info_dict['formats'] = formats
1360         if self.params.get('listformats'):
1361             self.list_formats(info_dict)
1362             return
1363
1364         req_format = self.params.get('format')
1365         if req_format is None:
1366             req_format_list = []
1367             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1368                     not info_dict.get('is_live')):
1369                 merger = FFmpegMergerPP(self)
1370                 if merger.available and merger.can_merge():
1371                     req_format_list.append('bestvideo+bestaudio')
1372             req_format_list.append('best')
1373             req_format = '/'.join(req_format_list)
1374         format_selector = self.build_format_selector(req_format)
1375         formats_to_download = list(format_selector(formats))
1376         if not formats_to_download:
1377             raise ExtractorError('requested format not available',
1378                                  expected=True)
1379
1380         if download:
1381             if len(formats_to_download) > 1:
1382                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1383             for format in formats_to_download:
1384                 new_info = dict(info_dict)
1385                 new_info.update(format)
1386                 self.process_info(new_info)
1387         # We update the info dict with the best quality format (backwards compatibility)
1388         info_dict.update(formats_to_download[-1])
1389         return info_dict
1390
1391     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1392         """Select the requested subtitles and their format"""
1393         available_subs = {}
1394         if normal_subtitles and self.params.get('writesubtitles'):
1395             available_subs.update(normal_subtitles)
1396         if automatic_captions and self.params.get('writeautomaticsub'):
1397             for lang, cap_info in automatic_captions.items():
1398                 if lang not in available_subs:
1399                     available_subs[lang] = cap_info
1400
1401         if (not self.params.get('writesubtitles') and not
1402                 self.params.get('writeautomaticsub') or not
1403                 available_subs):
1404             return None
1405
1406         if self.params.get('allsubtitles', False):
1407             requested_langs = available_subs.keys()
1408         else:
1409             if self.params.get('subtitleslangs', False):
1410                 requested_langs = self.params.get('subtitleslangs')
1411             elif 'en' in available_subs:
1412                 requested_langs = ['en']
1413             else:
1414                 requested_langs = [list(available_subs.keys())[0]]
1415
1416         formats_query = self.params.get('subtitlesformat', 'best')
1417         formats_preference = formats_query.split('/') if formats_query else []
1418         subs = {}
1419         for lang in requested_langs:
1420             formats = available_subs.get(lang)
1421             if formats is None:
1422                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1423                 continue
1424             for ext in formats_preference:
1425                 if ext == 'best':
1426                     f = formats[-1]
1427                     break
1428                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1429                 if matches:
1430                     f = matches[-1]
1431                     break
1432             else:
1433                 f = formats[-1]
1434                 self.report_warning(
1435                     'No subtitle format found matching "%s" for language %s, '
1436                     'using %s' % (formats_query, lang, f['ext']))
1437             subs[lang] = f
1438         return subs
1439
1440     def process_info(self, info_dict):
1441         """Process a single resolved IE result."""
1442
1443         assert info_dict.get('_type', 'video') == 'video'
1444
1445         max_downloads = self.params.get('max_downloads')
1446         if max_downloads is not None:
1447             if self._num_downloads >= int(max_downloads):
1448                 raise MaxDownloadsReached()
1449
1450         info_dict['fulltitle'] = info_dict['title']
1451         if len(info_dict['title']) > 200:
1452             info_dict['title'] = info_dict['title'][:197] + '...'
1453
1454         if 'format' not in info_dict:
1455             info_dict['format'] = info_dict['ext']
1456
1457         reason = self._match_entry(info_dict, incomplete=False)
1458         if reason is not None:
1459             self.to_screen('[download] ' + reason)
1460             return
1461
1462         self._num_downloads += 1
1463
1464         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1465
1466         # Forced printings
1467         if self.params.get('forcetitle', False):
1468             self.to_stdout(info_dict['fulltitle'])
1469         if self.params.get('forceid', False):
1470             self.to_stdout(info_dict['id'])
1471         if self.params.get('forceurl', False):
1472             if info_dict.get('requested_formats') is not None:
1473                 for f in info_dict['requested_formats']:
1474                     self.to_stdout(f['url'] + f.get('play_path', ''))
1475             else:
1476                 # For RTMP URLs, also include the playpath
1477                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1478         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1479             self.to_stdout(info_dict['thumbnail'])
1480         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1481             self.to_stdout(info_dict['description'])
1482         if self.params.get('forcefilename', False) and filename is not None:
1483             self.to_stdout(filename)
1484         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1485             self.to_stdout(formatSeconds(info_dict['duration']))
1486         if self.params.get('forceformat', False):
1487             self.to_stdout(info_dict['format'])
1488         if self.params.get('forcejson', False):
1489             self.to_stdout(json.dumps(info_dict))
1490
1491         # Do nothing else if in simulate mode
1492         if self.params.get('simulate', False):
1493             return
1494
1495         if filename is None:
1496             return
1497
1498         try:
1499             dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1500             if dn and not os.path.exists(dn):
1501                 os.makedirs(dn)
1502         except (OSError, IOError) as err:
1503             self.report_error('unable to create directory ' + error_to_compat_str(err))
1504             return
1505
1506         if self.params.get('writedescription', False):
1507             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1508             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1509                 self.to_screen('[info] Video description is already present')
1510             elif info_dict.get('description') is None:
1511                 self.report_warning('There\'s no description to write.')
1512             else:
1513                 try:
1514                     self.to_screen('[info] Writing video description to: ' + descfn)
1515                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1516                         descfile.write(info_dict['description'])
1517                 except (OSError, IOError):
1518                     self.report_error('Cannot write description file ' + descfn)
1519                     return
1520
1521         if self.params.get('writeannotations', False):
1522             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1523             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1524                 self.to_screen('[info] Video annotations are already present')
1525             else:
1526                 try:
1527                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1528                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1529                         annofile.write(info_dict['annotations'])
1530                 except (KeyError, TypeError):
1531                     self.report_warning('There are no annotations to write.')
1532                 except (OSError, IOError):
1533                     self.report_error('Cannot write annotations file: ' + annofn)
1534                     return
1535
1536         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1537                                        self.params.get('writeautomaticsub')])
1538
1539         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1540             # subtitles download errors are already managed as troubles in relevant IE
1541             # that way it will silently go on when used with unsupporting IE
1542             subtitles = info_dict['requested_subtitles']
1543             ie = self.get_info_extractor(info_dict['extractor_key'])
1544             for sub_lang, sub_info in subtitles.items():
1545                 sub_format = sub_info['ext']
1546                 if sub_info.get('data') is not None:
1547                     sub_data = sub_info['data']
1548                 else:
1549                     try:
1550                         sub_data = ie._download_webpage(
1551                             sub_info['url'], info_dict['id'], note=False)
1552                     except ExtractorError as err:
1553                         self.report_warning('Unable to download subtitle for "%s": %s' %
1554                                             (sub_lang, error_to_compat_str(err.cause)))
1555                         continue
1556                 try:
1557                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1558                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1559                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1560                     else:
1561                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1562                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1563                             subfile.write(sub_data)
1564                 except (OSError, IOError):
1565                     self.report_error('Cannot write subtitles file ' + sub_filename)
1566                     return
1567
1568         if self.params.get('writeinfojson', False):
1569             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1570             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1571                 self.to_screen('[info] Video description metadata is already present')
1572             else:
1573                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1574                 try:
1575                     write_json_file(self.filter_requested_info(info_dict), infofn)
1576                 except (OSError, IOError):
1577                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1578                     return
1579
1580         self._write_thumbnails(info_dict, filename)
1581
1582         if not self.params.get('skip_download', False):
1583             try:
1584                 def dl(name, info):
1585                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1586                     for ph in self._progress_hooks:
1587                         fd.add_progress_hook(ph)
1588                     if self.params.get('verbose'):
1589                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1590                     return fd.download(name, info)
1591
1592                 if info_dict.get('requested_formats') is not None:
1593                     downloaded = []
1594                     success = True
1595                     merger = FFmpegMergerPP(self)
1596                     if not merger.available:
1597                         postprocessors = []
1598                         self.report_warning('You have requested multiple '
1599                                             'formats but ffmpeg or avconv are not installed.'
1600                                             ' The formats won\'t be merged.')
1601                     else:
1602                         postprocessors = [merger]
1603
1604                     def compatible_formats(formats):
1605                         video, audio = formats
1606                         # Check extension
1607                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1608                         if video_ext and audio_ext:
1609                             COMPATIBLE_EXTS = (
1610                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1611                                 ('webm')
1612                             )
1613                             for exts in COMPATIBLE_EXTS:
1614                                 if video_ext in exts and audio_ext in exts:
1615                                     return True
1616                         # TODO: Check acodec/vcodec
1617                         return False
1618
1619                     filename_real_ext = os.path.splitext(filename)[1][1:]
1620                     filename_wo_ext = (
1621                         os.path.splitext(filename)[0]
1622                         if filename_real_ext == info_dict['ext']
1623                         else filename)
1624                     requested_formats = info_dict['requested_formats']
1625                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1626                         info_dict['ext'] = 'mkv'
1627                         self.report_warning(
1628                             'Requested formats are incompatible for merge and will be merged into mkv.')
1629                     # Ensure filename always has a correct extension for successful merge
1630                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1631                     if os.path.exists(encodeFilename(filename)):
1632                         self.to_screen(
1633                             '[download] %s has already been downloaded and '
1634                             'merged' % filename)
1635                     else:
1636                         for f in requested_formats:
1637                             new_info = dict(info_dict)
1638                             new_info.update(f)
1639                             fname = self.prepare_filename(new_info)
1640                             fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1641                             downloaded.append(fname)
1642                             partial_success = dl(fname, new_info)
1643                             success = success and partial_success
1644                         info_dict['__postprocessors'] = postprocessors
1645                         info_dict['__files_to_merge'] = downloaded
1646                 else:
1647                     # Just a single file
1648                     success = dl(filename, info_dict)
1649             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1650                 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1651                 return
1652             except (OSError, IOError) as err:
1653                 raise UnavailableVideoError(err)
1654             except (ContentTooShortError, ) as err:
1655                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1656                 return
1657
1658             if success and filename != '-':
1659                 # Fixup content
1660                 fixup_policy = self.params.get('fixup')
1661                 if fixup_policy is None:
1662                     fixup_policy = 'detect_or_warn'
1663
1664                 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1665
1666                 stretched_ratio = info_dict.get('stretched_ratio')
1667                 if stretched_ratio is not None and stretched_ratio != 1:
1668                     if fixup_policy == 'warn':
1669                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1670                             info_dict['id'], stretched_ratio))
1671                     elif fixup_policy == 'detect_or_warn':
1672                         stretched_pp = FFmpegFixupStretchedPP(self)
1673                         if stretched_pp.available:
1674                             info_dict.setdefault('__postprocessors', [])
1675                             info_dict['__postprocessors'].append(stretched_pp)
1676                         else:
1677                             self.report_warning(
1678                                 '%s: Non-uniform pixel ratio (%s). %s'
1679                                 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1680                     else:
1681                         assert fixup_policy in ('ignore', 'never')
1682
1683                 if (info_dict.get('requested_formats') is None and
1684                         info_dict.get('container') == 'm4a_dash'):
1685                     if fixup_policy == 'warn':
1686                         self.report_warning(
1687                             '%s: writing DASH m4a. '
1688                             'Only some players support this container.'
1689                             % info_dict['id'])
1690                     elif fixup_policy == 'detect_or_warn':
1691                         fixup_pp = FFmpegFixupM4aPP(self)
1692                         if fixup_pp.available:
1693                             info_dict.setdefault('__postprocessors', [])
1694                             info_dict['__postprocessors'].append(fixup_pp)
1695                         else:
1696                             self.report_warning(
1697                                 '%s: writing DASH m4a. '
1698                                 'Only some players support this container. %s'
1699                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1700                     else:
1701                         assert fixup_policy in ('ignore', 'never')
1702
1703                 if (info_dict.get('protocol') == 'm3u8_native' or
1704                         info_dict.get('protocol') == 'm3u8' and
1705                         self.params.get('hls_prefer_native')):
1706                     if fixup_policy == 'warn':
1707                         self.report_warning('%s: malformated aac bitstream.' % (
1708                             info_dict['id']))
1709                     elif fixup_policy == 'detect_or_warn':
1710                         fixup_pp = FFmpegFixupM3u8PP(self)
1711                         if fixup_pp.available:
1712                             info_dict.setdefault('__postprocessors', [])
1713                             info_dict['__postprocessors'].append(fixup_pp)
1714                         else:
1715                             self.report_warning(
1716                                 '%s: malformated aac bitstream. %s'
1717                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1718                     else:
1719                         assert fixup_policy in ('ignore', 'never')
1720
1721                 try:
1722                     self.post_process(filename, info_dict)
1723                 except (PostProcessingError) as err:
1724                     self.report_error('postprocessing: %s' % str(err))
1725                     return
1726                 self.record_download_archive(info_dict)
1727
1728     def download(self, url_list):
1729         """Download a given list of URLs."""
1730         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1731         if (len(url_list) > 1 and
1732                 '%' not in outtmpl and
1733                 self.params.get('max_downloads') != 1):
1734             raise SameFileError(outtmpl)
1735
1736         for url in url_list:
1737             try:
1738                 # It also downloads the videos
1739                 res = self.extract_info(
1740                     url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1741             except UnavailableVideoError:
1742                 self.report_error('unable to download video')
1743             except MaxDownloadsReached:
1744                 self.to_screen('[info] Maximum number of downloaded files reached.')
1745                 raise
1746             else:
1747                 if self.params.get('dump_single_json', False):
1748                     self.to_stdout(json.dumps(res))
1749
1750         return self._download_retcode
1751
1752     def download_with_info_file(self, info_filename):
1753         with contextlib.closing(fileinput.FileInput(
1754                 [info_filename], mode='r',
1755                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1756             # FileInput doesn't have a read method, we can't call json.load
1757             info = self.filter_requested_info(json.loads('\n'.join(f)))
1758         try:
1759             self.process_ie_result(info, download=True)
1760         except DownloadError:
1761             webpage_url = info.get('webpage_url')
1762             if webpage_url is not None:
1763                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1764                 return self.download([webpage_url])
1765             else:
1766                 raise
1767         return self._download_retcode
1768
1769     @staticmethod
1770     def filter_requested_info(info_dict):
1771         return dict(
1772             (k, v) for k, v in info_dict.items()
1773             if k not in ['requested_formats', 'requested_subtitles'])
1774
1775     def post_process(self, filename, ie_info):
1776         """Run all the postprocessors on the given file."""
1777         info = dict(ie_info)
1778         info['filepath'] = filename
1779         pps_chain = []
1780         if ie_info.get('__postprocessors') is not None:
1781             pps_chain.extend(ie_info['__postprocessors'])
1782         pps_chain.extend(self._pps)
1783         for pp in pps_chain:
1784             files_to_delete = []
1785             try:
1786                 files_to_delete, info = pp.run(info)
1787             except PostProcessingError as e:
1788                 self.report_error(e.msg)
1789             if files_to_delete and not self.params.get('keepvideo', False):
1790                 for old_filename in files_to_delete:
1791                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1792                     try:
1793                         os.remove(encodeFilename(old_filename))
1794                     except (IOError, OSError):
1795                         self.report_warning('Unable to remove downloaded original file')
1796
1797     def _make_archive_id(self, info_dict):
1798         # Future-proof against any change in case
1799         # and backwards compatibility with prior versions
1800         extractor = info_dict.get('extractor_key')
1801         if extractor is None:
1802             if 'id' in info_dict:
1803                 extractor = info_dict.get('ie_key')  # key in a playlist
1804         if extractor is None:
1805             return None  # Incomplete video information
1806         return extractor.lower() + ' ' + info_dict['id']
1807
1808     def in_download_archive(self, info_dict):
1809         fn = self.params.get('download_archive')
1810         if fn is None:
1811             return False
1812
1813         vid_id = self._make_archive_id(info_dict)
1814         if vid_id is None:
1815             return False  # Incomplete video information
1816
1817         try:
1818             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1819                 for line in archive_file:
1820                     if line.strip() == vid_id:
1821                         return True
1822         except IOError as ioe:
1823             if ioe.errno != errno.ENOENT:
1824                 raise
1825         return False
1826
1827     def record_download_archive(self, info_dict):
1828         fn = self.params.get('download_archive')
1829         if fn is None:
1830             return
1831         vid_id = self._make_archive_id(info_dict)
1832         assert vid_id
1833         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1834             archive_file.write(vid_id + '\n')
1835
1836     @staticmethod
1837     def format_resolution(format, default='unknown'):
1838         if format.get('vcodec') == 'none':
1839             return 'audio only'
1840         if format.get('resolution') is not None:
1841             return format['resolution']
1842         if format.get('height') is not None:
1843             if format.get('width') is not None:
1844                 res = '%sx%s' % (format['width'], format['height'])
1845             else:
1846                 res = '%sp' % format['height']
1847         elif format.get('width') is not None:
1848             res = '%dx?' % format['width']
1849         else:
1850             res = default
1851         return res
1852
1853     def _format_note(self, fdict):
1854         res = ''
1855         if fdict.get('ext') in ['f4f', 'f4m']:
1856             res += '(unsupported) '
1857         if fdict.get('language'):
1858             if res:
1859                 res += ' '
1860             res += '[%s] ' % fdict['language']
1861         if fdict.get('format_note') is not None:
1862             res += fdict['format_note'] + ' '
1863         if fdict.get('tbr') is not None:
1864             res += '%4dk ' % fdict['tbr']
1865         if fdict.get('container') is not None:
1866             if res:
1867                 res += ', '
1868             res += '%s container' % fdict['container']
1869         if (fdict.get('vcodec') is not None and
1870                 fdict.get('vcodec') != 'none'):
1871             if res:
1872                 res += ', '
1873             res += fdict['vcodec']
1874             if fdict.get('vbr') is not None:
1875                 res += '@'
1876         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1877             res += 'video@'
1878         if fdict.get('vbr') is not None:
1879             res += '%4dk' % fdict['vbr']
1880         if fdict.get('fps') is not None:
1881             if res:
1882                 res += ', '
1883             res += '%sfps' % fdict['fps']
1884         if fdict.get('acodec') is not None:
1885             if res:
1886                 res += ', '
1887             if fdict['acodec'] == 'none':
1888                 res += 'video only'
1889             else:
1890                 res += '%-5s' % fdict['acodec']
1891         elif fdict.get('abr') is not None:
1892             if res:
1893                 res += ', '
1894             res += 'audio'
1895         if fdict.get('abr') is not None:
1896             res += '@%3dk' % fdict['abr']
1897         if fdict.get('asr') is not None:
1898             res += ' (%5dHz)' % fdict['asr']
1899         if fdict.get('filesize') is not None:
1900             if res:
1901                 res += ', '
1902             res += format_bytes(fdict['filesize'])
1903         elif fdict.get('filesize_approx') is not None:
1904             if res:
1905                 res += ', '
1906             res += '~' + format_bytes(fdict['filesize_approx'])
1907         return res
1908
1909     def list_formats(self, info_dict):
1910         formats = info_dict.get('formats', [info_dict])
1911         table = [
1912             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1913             for f in formats
1914             if f.get('preference') is None or f['preference'] >= -1000]
1915         if len(formats) > 1:
1916             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1917
1918         header_line = ['format code', 'extension', 'resolution', 'note']
1919         self.to_screen(
1920             '[info] Available formats for %s:\n%s' %
1921             (info_dict['id'], render_table(header_line, table)))
1922
1923     def list_thumbnails(self, info_dict):
1924         thumbnails = info_dict.get('thumbnails')
1925         if not thumbnails:
1926             self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
1927             return
1928
1929         self.to_screen(
1930             '[info] Thumbnails for %s:' % info_dict['id'])
1931         self.to_screen(render_table(
1932             ['ID', 'width', 'height', 'URL'],
1933             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1934
1935     def list_subtitles(self, video_id, subtitles, name='subtitles'):
1936         if not subtitles:
1937             self.to_screen('%s has no %s' % (video_id, name))
1938             return
1939         self.to_screen(
1940             'Available %s for %s:' % (name, video_id))
1941         self.to_screen(render_table(
1942             ['Language', 'formats'],
1943             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1944                 for lang, formats in subtitles.items()]))
1945
1946     def urlopen(self, req):
1947         """ Start an HTTP download """
1948         if isinstance(req, compat_basestring):
1949             req = sanitized_Request(req)
1950         return self._opener.open(req, timeout=self._socket_timeout)
1951
1952     def print_debug_header(self):
1953         if not self.params.get('verbose'):
1954             return
1955
1956         if type('') is not compat_str:
1957             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1958             self.report_warning(
1959                 'Your Python is broken! Update to a newer and supported version')
1960
1961         stdout_encoding = getattr(
1962             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1963         encoding_str = (
1964             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1965                 locale.getpreferredencoding(),
1966                 sys.getfilesystemencoding(),
1967                 stdout_encoding,
1968                 self.get_encoding()))
1969         write_string(encoding_str, encoding=None)
1970
1971         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1972         if _LAZY_LOADER:
1973             self._write_string('[debug] Lazy loading extractors enabled' + '\n')
1974         try:
1975             sp = subprocess.Popen(
1976                 ['git', 'rev-parse', '--short', 'HEAD'],
1977                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1978                 cwd=os.path.dirname(os.path.abspath(__file__)))
1979             out, err = sp.communicate()
1980             out = out.decode().strip()
1981             if re.match('[0-9a-f]+', out):
1982                 self._write_string('[debug] Git HEAD: ' + out + '\n')
1983         except Exception:
1984             try:
1985                 sys.exc_clear()
1986             except Exception:
1987                 pass
1988         self._write_string('[debug] Python version %s - %s\n' % (
1989             platform.python_version(), platform_name()))
1990
1991         exe_versions = FFmpegPostProcessor.get_versions(self)
1992         exe_versions['rtmpdump'] = rtmpdump_version()
1993         exe_str = ', '.join(
1994             '%s %s' % (exe, v)
1995             for exe, v in sorted(exe_versions.items())
1996             if v
1997         )
1998         if not exe_str:
1999             exe_str = 'none'
2000         self._write_string('[debug] exe versions: %s\n' % exe_str)
2001
2002         proxy_map = {}
2003         for handler in self._opener.handlers:
2004             if hasattr(handler, 'proxies'):
2005                 proxy_map.update(handler.proxies)
2006         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2007
2008         if self.params.get('call_home', False):
2009             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2010             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2011             latest_version = self.urlopen(
2012                 'https://yt-dl.org/latest/version').read().decode('utf-8')
2013             if version_tuple(latest_version) > version_tuple(__version__):
2014                 self.report_warning(
2015                     'You are using an outdated version (newest version: %s)! '
2016                     'See https://yt-dl.org/update if you need help updating.' %
2017                     latest_version)
2018
2019     def _setup_opener(self):
2020         timeout_val = self.params.get('socket_timeout')
2021         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2022
2023         opts_cookiefile = self.params.get('cookiefile')
2024         opts_proxy = self.params.get('proxy')
2025
2026         if opts_cookiefile is None:
2027             self.cookiejar = compat_cookiejar.CookieJar()
2028         else:
2029             opts_cookiefile = compat_expanduser(opts_cookiefile)
2030             self.cookiejar = compat_cookiejar.MozillaCookieJar(
2031                 opts_cookiefile)
2032             if os.access(opts_cookiefile, os.R_OK):
2033                 self.cookiejar.load()
2034
2035         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2036         if opts_proxy is not None:
2037             if opts_proxy == '':
2038                 proxies = {}
2039             else:
2040                 proxies = {'http': opts_proxy, 'https': opts_proxy}
2041         else:
2042             proxies = compat_urllib_request.getproxies()
2043             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2044             if 'http' in proxies and 'https' not in proxies:
2045                 proxies['https'] = proxies['http']
2046         proxy_handler = PerRequestProxyHandler(proxies)
2047
2048         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2049         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2050         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2051         data_handler = compat_urllib_request_DataHandler()
2052
2053         # When passing our own FileHandler instance, build_opener won't add the
2054         # default FileHandler and allows us to disable the file protocol, which
2055         # can be used for malicious purposes (see
2056         # https://github.com/rg3/youtube-dl/issues/8227)
2057         file_handler = compat_urllib_request.FileHandler()
2058
2059         def file_open(*args, **kwargs):
2060             raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2061         file_handler.file_open = file_open
2062
2063         opener = compat_urllib_request.build_opener(
2064             proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2065
2066         # Delete the default user-agent header, which would otherwise apply in
2067         # cases where our custom HTTP handler doesn't come into play
2068         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2069         opener.addheaders = []
2070         self._opener = opener
2071
2072     def encode(self, s):
2073         if isinstance(s, bytes):
2074             return s  # Already encoded
2075
2076         try:
2077             return s.encode(self.get_encoding())
2078         except UnicodeEncodeError as err:
2079             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2080             raise
2081
2082     def get_encoding(self):
2083         encoding = self.params.get('encoding')
2084         if encoding is None:
2085             encoding = preferredencoding()
2086         return encoding
2087
2088     def _write_thumbnails(self, info_dict, filename):
2089         if self.params.get('writethumbnail', False):
2090             thumbnails = info_dict.get('thumbnails')
2091             if thumbnails:
2092                 thumbnails = [thumbnails[-1]]
2093         elif self.params.get('write_all_thumbnails', False):
2094             thumbnails = info_dict.get('thumbnails')
2095         else:
2096             return
2097
2098         if not thumbnails:
2099             # No thumbnails present, so return immediately
2100             return
2101
2102         for t in thumbnails:
2103             thumb_ext = determine_ext(t['url'], 'jpg')
2104             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2105             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2106             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2107
2108             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2109                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2110                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2111             else:
2112                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2113                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2114                 try:
2115                     uf = self.urlopen(t['url'])
2116                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2117                         shutil.copyfileobj(uf, thumbf)
2118                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2119                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2120                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2121                     self.report_warning('Unable to download thumbnail "%s": %s' %
2122                                         (t['url'], error_to_compat_str(err)))