_ Git - youtube-dl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import contextlib
   8 import datetime
   9 import errno
  10 import fileinput
  11 import io
  12 import itertools
  13 import json
  14 import locale
  15 import operator
  16 import os
  17 import platform
  18 import re
  19 import shutil
  20 import subprocess
  21 import socket
  22 import sys
  23 import time
  24 import tokenize
  25 import traceback
  26
  27 from .compat import (
  28     compat_basestring,
  29     compat_cookiejar,
  30     compat_expanduser,
  31     compat_get_terminal_size,
  32     compat_http_client,
  33     compat_kwargs,
  34     compat_os_name,
  35     compat_str,
  36     compat_tokenize_tokenize,
  37     compat_urllib_error,
  38     compat_urllib_request,
  39     compat_urllib_request_DataHandler,
  40 )
  41 from .utils import (
  42     age_restricted,
  43     args_to_str,
  44     ContentTooShortError,
  45     date_from_str,
  46     DateRange,
  47     DEFAULT_OUTTMPL,
  48     determine_ext,
  49     determine_protocol,
  50     DownloadError,
  51     encode_compat_str,
  52     encodeFilename,
  53     error_to_compat_str,
  54     ExtractorError,
  55     format_bytes,
  56     formatSeconds,
  57     locked_file,
  58     make_HTTPS_handler,
  59     MaxDownloadsReached,
  60     PagedList,
  61     parse_filesize,
  62     PerRequestProxyHandler,
  63     platform_name,
  64     PostProcessingError,
  65     preferredencoding,
  66     prepend_extension,
  67     register_socks_protocols,
  68     render_table,
  69     replace_extension,
  70     SameFileError,
  71     sanitize_filename,
  72     sanitize_path,
  73     sanitize_url,
  74     sanitized_Request,
  75     std_headers,
  76     subtitles_filename,
  77     UnavailableVideoError,
  78     url_basename,
  79     version_tuple,
  80     write_json_file,
  81     write_string,
  82     YoutubeDLCookieProcessor,
  83     YoutubeDLHandler,
  84 )
  85 from .cache import Cache
  86 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
  87 from .downloader import get_suitable_downloader
  88 from .downloader.rtmp import rtmpdump_version
  89 from .postprocessor import (
  90     FFmpegFixupM3u8PP,
  91     FFmpegFixupM4aPP,
  92     FFmpegFixupStretchedPP,
  93     FFmpegMergerPP,
  94     FFmpegPostProcessor,
  95     get_postprocessor,
  96 )
  97 from .version import __version__
  98
  99 if compat_os_name == 'nt':
 100     import ctypes
 101
 102
 103 class YoutubeDL(object):
 104     """YoutubeDL class.
 105
 106     YoutubeDL objects are the ones responsible of downloading the
 107     actual video file and writing it to disk if the user has requested
 108     it, among some other tasks. In most cases there should be one per
 109     program. As, given a video URL, the downloader doesn't know how to
 110     extract all the needed information, task that InfoExtractors do, it
 111     has to pass the URL to one of them.
 112
 113     For this, YoutubeDL objects have a method that allows
 114     InfoExtractors to be registered in a given order. When it is passed
 115     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 116     finds that reports being able to handle it. The InfoExtractor extracts
 117     all the information about the video or videos the URL refers to, and
 118     YoutubeDL process the extracted information, possibly using a File
 119     Downloader to download the video.
 120
 121     YoutubeDL objects accept a lot of parameters. In order not to saturate
 122     the object constructor with arguments, it receives a dictionary of
 123     options instead. These options are available through the params
 124     attribute for the InfoExtractors to use. The YoutubeDL also
 125     registers itself as the downloader in charge for the InfoExtractors
 126     that are added to it, so this is a "mutual registration".
 127
 128     Available options:
 129
 130     username:          Username for authentication purposes.
 131     password:          Password for authentication purposes.
 132     videopassword:     Password for accessing a video.
 133     usenetrc:          Use netrc for authentication instead.
 134     verbose:           Print additional info to stdout.
 135     quiet:             Do not print messages to stdout.
 136     no_warnings:       Do not print out anything for warnings.
 137     forceurl:          Force printing final URL.
 138     forcetitle:        Force printing title.
 139     forceid:           Force printing ID.
 140     forcethumbnail:    Force printing thumbnail URL.
 141     forcedescription:  Force printing description.
 142     forcefilename:     Force printing final filename.
 143     forceduration:     Force printing duration.
 144     forcejson:         Force printing info_dict as JSON.
 145     dump_single_json:  Force printing the info_dict of the whole playlist
 146                        (or video) as a single JSON line.
 147     simulate:          Do not download the video files.
 148     format:            Video format code. See options.py for more information.
 149     outtmpl:           Template for output names.
 150     restrictfilenames: Do not allow "&" and spaces in file names
 151     ignoreerrors:      Do not stop on download errors.
 152     force_generic_extractor: Force downloader to use the generic extractor
 153     nooverwrites:      Prevent overwriting files.
 154     playliststart:     Playlist item to start at.
 155     playlistend:       Playlist item to end at.
 156     playlist_items:    Specific indices of playlist to download.
 157     playlistreverse:   Download playlist items in reverse order.
 158     matchtitle:        Download only matching titles.
 159     rejecttitle:       Reject downloads for matching titles.
 160     logger:            Log messages to a logging.Logger instance.
 161     logtostderr:       Log messages to stderr instead of stdout.
 162     writedescription:  Write the video description to a .description file
 163     writeinfojson:     Write the video description to a .info.json file
 164     writeannotations:  Write the video annotations to a .annotations.xml file
 165     writethumbnail:    Write the thumbnail image to a file
 166     write_all_thumbnails:  Write all thumbnail formats to files
 167     writesubtitles:    Write the video subtitles to a file
 168     writeautomaticsub: Write the automatically generated subtitles to a file
 169     allsubtitles:      Downloads all the subtitles of the video
 170                        (requires writesubtitles or writeautomaticsub)
 171     listsubtitles:     Lists all available subtitles for the video
 172     subtitlesformat:   The format code for subtitles
 173     subtitleslangs:    List of languages of the subtitles to download
 174     keepvideo:         Keep the video file after post-processing
 175     daterange:         A DateRange object, download only if the upload_date is in the range.
 176     skip_download:     Skip the actual download of the video file
 177     cachedir:          Location of the cache files in the filesystem.
 178                        False to disable filesystem cache.
 179     noplaylist:        Download single video instead of a playlist if in doubt.
 180     age_limit:         An integer representing the user's age in years.
 181                        Unsuitable videos for the given age are skipped.
 182     min_views:         An integer representing the minimum view count the video
 183                        must have in order to not be skipped.
 184                        Videos without view count information are always
 185                        downloaded. None for no limit.
 186     max_views:         An integer representing the maximum view count.
 187                        Videos that are more popular than that are not
 188                        downloaded.
 189                        Videos without view count information are always
 190                        downloaded. None for no limit.
 191     download_archive:  File name of a file where all downloads are recorded.
 192                        Videos already present in the file are not downloaded
 193                        again.
 194     cookiefile:        File name where cookies should be read from and dumped to.
 195     nocheckcertificate:Do not verify SSL certificates
 196     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 197                        At the moment, this is only supported by YouTube.
 198     proxy:             URL of the proxy server to use
 199     cn_verification_proxy:  URL of the proxy to use for IP address verification
 200                        on Chinese sites. (Experimental)
 201     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 202     bidi_workaround:   Work around buggy terminals without bidirectional text
 203                        support, using fridibi
 204     debug_printtraffic:Print out sent and received HTTP traffic
 205     include_ads:       Download ads as well
 206     default_search:    Prepend this string if an input url is not valid.
 207                        'auto' for elaborate guessing
 208     encoding:          Use this encoding instead of the system-specified.
 209     extract_flat:      Do not resolve URLs, return the immediate result.
 210                        Pass in 'in_playlist' to only show this behavior for
 211                        playlist items.
 212     postprocessors:    A list of dictionaries, each with an entry
 213                        * key:  The name of the postprocessor. See
 214                                youtube_dl/postprocessor/__init__.py for a list.
 215                        as well as any further keyword arguments for the
 216                        postprocessor.
 217     progress_hooks:    A list of functions that get called on download
 218                        progress, with a dictionary with the entries
 219                        * status: One of "downloading", "error", or "finished".
 220                                  Check this first and ignore unknown values.
 221
 222                        If status is one of "downloading", or "finished", the
 223                        following properties may also be present:
 224                        * filename: The final filename (always present)
 225                        * tmpfilename: The filename we're currently writing to
 226                        * downloaded_bytes: Bytes on disk
 227                        * total_bytes: Size of the whole file, None if unknown
 228                        * total_bytes_estimate: Guess of the eventual file size,
 229                                                None if unavailable.
 230                        * elapsed: The number of seconds since download started.
 231                        * eta: The estimated time in seconds, None if unknown
 232                        * speed: The download speed in bytes/second, None if
 233                                 unknown
 234                        * fragment_index: The counter of the currently
 235                                          downloaded video fragment.
 236                        * fragment_count: The number of fragments (= individual
 237                                          files that will be merged)
 238
 239                        Progress hooks are guaranteed to be called at least once
 240                        (with status "finished") if the download is successful.
 241     merge_output_format: Extension to use when merging formats.
 242     fixup:             Automatically correct known faults of the file.
 243                        One of:
 244                        - "never": do nothing
 245                        - "warn": only emit a warning
 246                        - "detect_or_warn": check whether we can do anything
 247                                            about it, warn otherwise (default)
 248     source_address:    (Experimental) Client-side IP address to bind to.
 249     call_home:         Boolean, true iff we are allowed to contact the
 250                        youtube-dl servers for debugging.
 251     sleep_interval:    Number of seconds to sleep before each download.
 252     listformats:       Print an overview of available video formats and exit.
 253     list_thumbnails:   Print a table of all thumbnails and exit.
 254     match_filter:      A function that gets called with the info_dict of
 255                        every video.
 256                        If it returns a message, the video is ignored.
 257                        If it returns None, the video is downloaded.
 258                        match_filter_func in utils.py is one example for this.
 259     no_color:          Do not emit color codes in output.
 260
 261     The following options determine which downloader is picked:
 262     external_downloader: Executable of the external downloader to call.
 263                        None or unset for standard (built-in) downloader.
 264     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
 265                        if True, otherwise use ffmpeg/avconv if False, otherwise
 266                        use downloader suggested by extractor if None.
 267
 268     The following parameters are not used by YoutubeDL itself, they are used by
 269     the downloader (see youtube_dl/downloader/common.py):
 270     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 271     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 272     xattr_set_filesize, external_downloader_args, hls_use_mpegts.
 273
 274     The following options are used by the post processors:
 275     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 276                        otherwise prefer avconv.
 277     postprocessor_args: A list of additional command-line arguments for the
 278                         postprocessor.
 279     """
 280
 281     params = None
 282     _ies = []
 283     _pps = []
 284     _download_retcode = None
 285     _num_downloads = None
 286     _screen_file = None
 287
 288     def __init__(self, params=None, auto_init=True):
 289         """Create a FileDownloader object with the given options."""
 290         if params is None:
 291             params = {}
 292         self._ies = []
 293         self._ies_instances = {}
 294         self._pps = []
 295         self._progress_hooks = []
 296         self._download_retcode = 0
 297         self._num_downloads = 0
 298         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 299         self._err_file = sys.stderr
 300         self.params = {
 301             # Default parameters
 302             'nocheckcertificate': False,
 303         }
 304         self.params.update(params)
 305         self.cache = Cache(self)
 306
 307         if params.get('bidi_workaround', False):
 308             try:
 309                 import pty
 310                 master, slave = pty.openpty()
 311                 width = compat_get_terminal_size().columns
 312                 if width is None:
 313                     width_args = []
 314                 else:
 315                     width_args = ['-w', str(width)]
 316                 sp_kwargs = dict(
 317                     stdin=subprocess.PIPE,
 318                     stdout=slave,
 319                     stderr=self._err_file)
 320                 try:
 321                     self._output_process = subprocess.Popen(
 322                         ['bidiv'] + width_args, **sp_kwargs
 323                     )
 324                 except OSError:
 325                     self._output_process = subprocess.Popen(
 326                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 327                 self._output_channel = os.fdopen(master, 'rb')
 328             except OSError as ose:
 329                 if ose.errno == errno.ENOENT:
 330                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 331                 else:
 332                     raise
 333
 334         if (sys.version_info >= (3,) and sys.platform != 'win32' and
 335                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
 336                 not params.get('restrictfilenames', False)):
 337             # On Python 3, the Unicode filesystem API will throw errors (#1474)
 338             self.report_warning(
 339                 'Assuming --restrict-filenames since file system encoding '
 340                 'cannot encode all characters. '
 341                 'Set the LC_ALL environment variable to fix this.')
 342             self.params['restrictfilenames'] = True
 343
 344         if isinstance(params.get('outtmpl'), bytes):
 345             self.report_warning(
 346                 'Parameter outtmpl is bytes, but should be a unicode string. '
 347                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 348
 349         self._setup_opener()
 350
 351         if auto_init:
 352             self.print_debug_header()
 353             self.add_default_info_extractors()
 354
 355         for pp_def_raw in self.params.get('postprocessors', []):
 356             pp_class = get_postprocessor(pp_def_raw['key'])
 357             pp_def = dict(pp_def_raw)
 358             del pp_def['key']
 359             pp = pp_class(self, **compat_kwargs(pp_def))
 360             self.add_post_processor(pp)
 361
 362         for ph in self.params.get('progress_hooks', []):
 363             self.add_progress_hook(ph)
 364
 365         register_socks_protocols()
 366
 367     def warn_if_short_id(self, argv):
 368         # short YouTube ID starting with dash?
 369         idxs = [
 370             i for i, a in enumerate(argv)
 371             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 372         if idxs:
 373             correct_argv = (
 374                 ['youtube-dl'] +
 375                 [a for i, a in enumerate(argv) if i not in idxs] +
 376                 ['--'] + [argv[i] for i in idxs]
 377             )
 378             self.report_warning(
 379                 'Long argument string detected. '
 380                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 381                 args_to_str(correct_argv))
 382
 383     def add_info_extractor(self, ie):
 384         """Add an InfoExtractor object to the end of the list."""
 385         self._ies.append(ie)
 386         if not isinstance(ie, type):
 387             self._ies_instances[ie.ie_key()] = ie
 388             ie.set_downloader(self)
 389
 390     def get_info_extractor(self, ie_key):
 391         """
 392         Get an instance of an IE with name ie_key, it will try to get one from
 393         the _ies list, if there's no instance it will create a new one and add
 394         it to the extractor list.
 395         """
 396         ie = self._ies_instances.get(ie_key)
 397         if ie is None:
 398             ie = get_info_extractor(ie_key)()
 399             self.add_info_extractor(ie)
 400         return ie
 401
 402     def add_default_info_extractors(self):
 403         """
 404         Add the InfoExtractors returned by gen_extractors to the end of the list
 405         """
 406         for ie in gen_extractor_classes():
 407             self.add_info_extractor(ie)
 408
 409     def add_post_processor(self, pp):
 410         """Add a PostProcessor object to the end of the chain."""
 411         self._pps.append(pp)
 412         pp.set_downloader(self)
 413
 414     def add_progress_hook(self, ph):
 415         """Add the progress hook (currently only for the file downloader)"""
 416         self._progress_hooks.append(ph)
 417
 418     def _bidi_workaround(self, message):
 419         if not hasattr(self, '_output_channel'):
 420             return message
 421
 422         assert hasattr(self, '_output_process')
 423         assert isinstance(message, compat_str)
 424         line_count = message.count('\n') + 1
 425         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 426         self._output_process.stdin.flush()
 427         res = ''.join(self._output_channel.readline().decode('utf-8')
 428                       for _ in range(line_count))
 429         return res[:-len('\n')]
 430
 431     def to_screen(self, message, skip_eol=False):
 432         """Print message to stdout if not in quiet mode."""
 433         return self.to_stdout(message, skip_eol, check_quiet=True)
 434
 435     def _write_string(self, s, out=None):
 436         write_string(s, out=out, encoding=self.params.get('encoding'))
 437
 438     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 439         """Print message to stdout if not in quiet mode."""
 440         if self.params.get('logger'):
 441             self.params['logger'].debug(message)
 442         elif not check_quiet or not self.params.get('quiet', False):
 443             message = self._bidi_workaround(message)
 444             terminator = ['\n', ''][skip_eol]
 445             output = message + terminator
 446
 447             self._write_string(output, self._screen_file)
 448
 449     def to_stderr(self, message):
 450         """Print message to stderr."""
 451         assert isinstance(message, compat_str)
 452         if self.params.get('logger'):
 453             self.params['logger'].error(message)
 454         else:
 455             message = self._bidi_workaround(message)
 456             output = message + '\n'
 457             self._write_string(output, self._err_file)
 458
 459     def to_console_title(self, message):
 460         if not self.params.get('consoletitle', False):
 461             return
 462         if compat_os_name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 463             # c_wchar_p() might not be necessary if `message` is
 464             # already of type unicode()
 465             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 466         elif 'TERM' in os.environ:
 467             self._write_string('\033]0;%s\007' % message, self._screen_file)
 468
 469     def save_console_title(self):
 470         if not self.params.get('consoletitle', False):
 471             return
 472         if 'TERM' in os.environ:
 473             # Save the title on stack
 474             self._write_string('\033[22;0t', self._screen_file)
 475
 476     def restore_console_title(self):
 477         if not self.params.get('consoletitle', False):
 478             return
 479         if 'TERM' in os.environ:
 480             # Restore the title from stack
 481             self._write_string('\033[23;0t', self._screen_file)
 482
 483     def __enter__(self):
 484         self.save_console_title()
 485         return self
 486
 487     def __exit__(self, *args):
 488         self.restore_console_title()
 489
 490         if self.params.get('cookiefile') is not None:
 491             self.cookiejar.save()
 492
 493     def trouble(self, message=None, tb=None):
 494         """Determine action to take when a download problem appears.
 495
 496         Depending on if the downloader has been configured to ignore
 497         download errors or not, this method may throw an exception or
 498         not when errors are found, after printing the message.
 499
 500         tb, if given, is additional traceback information.
 501         """
 502         if message is not None:
 503             self.to_stderr(message)
 504         if self.params.get('verbose'):
 505             if tb is None:
 506                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 507                     tb = ''
 508                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 509                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 510                     tb += encode_compat_str(traceback.format_exc())
 511                 else:
 512                     tb_data = traceback.format_list(traceback.extract_stack())
 513                     tb = ''.join(tb_data)
 514             self.to_stderr(tb)
 515         if not self.params.get('ignoreerrors', False):
 516             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 517                 exc_info = sys.exc_info()[1].exc_info
 518             else:
 519                 exc_info = sys.exc_info()
 520             raise DownloadError(message, exc_info)
 521         self._download_retcode = 1
 522
 523     def report_warning(self, message):
 524         '''
 525         Print the message to stderr, it will be prefixed with 'WARNING:'
 526         If stderr is a tty file the 'WARNING:' will be colored
 527         '''
 528         if self.params.get('logger') is not None:
 529             self.params['logger'].warning(message)
 530         else:
 531             if self.params.get('no_warnings'):
 532                 return
 533             if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 534                 _msg_header = '\033[0;33mWARNING:\033[0m'
 535             else:
 536                 _msg_header = 'WARNING:'
 537             warning_message = '%s %s' % (_msg_header, message)
 538             self.to_stderr(warning_message)
 539
 540     def report_error(self, message, tb=None):
 541         '''
 542         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 543         in red if stderr is a tty file.
 544         '''
 545         if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
 546             _msg_header = '\033[0;31mERROR:\033[0m'
 547         else:
 548             _msg_header = 'ERROR:'
 549         error_message = '%s %s' % (_msg_header, message)
 550         self.trouble(error_message, tb)
 551
 552     def report_file_already_downloaded(self, file_name):
 553         """Report file has already been fully downloaded."""
 554         try:
 555             self.to_screen('[download] %s has already been downloaded' % file_name)
 556         except UnicodeEncodeError:
 557             self.to_screen('[download] The file has already been downloaded')
 558
 559     def prepare_filename(self, info_dict):
 560         """Generate the output filename."""
 561         try:
 562             template_dict = dict(info_dict)
 563
 564             template_dict['epoch'] = int(time.time())
 565             autonumber_size = self.params.get('autonumber_size')
 566             if autonumber_size is None:
 567                 autonumber_size = 5
 568             autonumber_templ = '%0' + str(autonumber_size) + 'd'
 569             template_dict['autonumber'] = autonumber_templ % self._num_downloads
 570             if template_dict.get('playlist_index') is not None:
 571                 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
 572             if template_dict.get('resolution') is None:
 573                 if template_dict.get('width') and template_dict.get('height'):
 574                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 575                 elif template_dict.get('height'):
 576                     template_dict['resolution'] = '%sp' % template_dict['height']
 577                 elif template_dict.get('width'):
 578                     template_dict['resolution'] = '%dx?' % template_dict['width']
 579
 580             sanitize = lambda k, v: sanitize_filename(
 581                 compat_str(v),
 582                 restricted=self.params.get('restrictfilenames'),
 583                 is_id=(k == 'id'))
 584             template_dict = dict((k, sanitize(k, v))
 585                                  for k, v in template_dict.items()
 586                                  if v is not None and not isinstance(v, (list, tuple, dict)))
 587             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 588
 589             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 590             tmpl = compat_expanduser(outtmpl)
 591             filename = tmpl % template_dict
 592             # Temporary fix for #4787
 593             # 'Treat' all problem characters by passing filename through preferredencoding
 594             # to workaround encoding issues with subprocess on python2 @ Windows
 595             if sys.version_info < (3, 0) and sys.platform == 'win32':
 596                 filename = encodeFilename(filename, True).decode(preferredencoding())
 597             return sanitize_path(filename)
 598         except ValueError as err:
 599             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 600             return None
 601
 602     def _match_entry(self, info_dict, incomplete):
 603         """ Returns None iff the file should be downloaded """
 604
 605         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 606         if 'title' in info_dict:
 607             # This can happen when we're just evaluating the playlist
 608             title = info_dict['title']
 609             matchtitle = self.params.get('matchtitle', False)
 610             if matchtitle:
 611                 if not re.search(matchtitle, title, re.IGNORECASE):
 612                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 613             rejecttitle = self.params.get('rejecttitle', False)
 614             if rejecttitle:
 615                 if re.search(rejecttitle, title, re.IGNORECASE):
 616                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 617         date = info_dict.get('upload_date')
 618         if date is not None:
 619             dateRange = self.params.get('daterange', DateRange())
 620             if date not in dateRange:
 621                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 622         view_count = info_dict.get('view_count')
 623         if view_count is not None:
 624             min_views = self.params.get('min_views')
 625             if min_views is not None and view_count < min_views:
 626                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 627             max_views = self.params.get('max_views')
 628             if max_views is not None and view_count > max_views:
 629                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 630         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 631             return 'Skipping "%s" because it is age restricted' % video_title
 632         if self.in_download_archive(info_dict):
 633             return '%s has already been recorded in archive' % video_title
 634
 635         if not incomplete:
 636             match_filter = self.params.get('match_filter')
 637             if match_filter is not None:
 638                 ret = match_filter(info_dict)
 639                 if ret is not None:
 640                     return ret
 641
 642         return None
 643
 644     @staticmethod
 645     def add_extra_info(info_dict, extra_info):
 646         '''Set the keys from extra_info in info dict if they are missing'''
 647         for key, value in extra_info.items():
 648             info_dict.setdefault(key, value)
 649
 650     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 651                      process=True, force_generic_extractor=False):
 652         '''
 653         Returns a list with a dictionary for each video we find.
 654         If 'download', also downloads the videos.
 655         extra_info is a dict containing the extra values to add to each result
 656         '''
 657
 658         if not ie_key and force_generic_extractor:
 659             ie_key = 'Generic'
 660
 661         if ie_key:
 662             ies = [self.get_info_extractor(ie_key)]
 663         else:
 664             ies = self._ies
 665
 666         for ie in ies:
 667             if not ie.suitable(url):
 668                 continue
 669
 670             ie = self.get_info_extractor(ie.ie_key())
 671             if not ie.working():
 672                 self.report_warning('The program functionality for this site has been marked as broken, '
 673                                     'and will probably not work.')
 674
 675             try:
 676                 ie_result = ie.extract(url)
 677                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 678                     break
 679                 if isinstance(ie_result, list):
 680                     # Backwards compatibility: old IE result format
 681                     ie_result = {
 682                         '_type': 'compat_list',
 683                         'entries': ie_result,
 684                     }
 685                 self.add_default_extra_info(ie_result, ie, url)
 686                 if process:
 687                     return self.process_ie_result(ie_result, download, extra_info)
 688                 else:
 689                     return ie_result
 690             except ExtractorError as e:  # An error we somewhat expected
 691                 self.report_error(compat_str(e), e.format_traceback())
 692                 break
 693             except MaxDownloadsReached:
 694                 raise
 695             except Exception as e:
 696                 if self.params.get('ignoreerrors', False):
 697                     self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
 698                     break
 699                 else:
 700                     raise
 701         else:
 702             self.report_error('no suitable InfoExtractor for URL %s' % url)
 703
 704     def add_default_extra_info(self, ie_result, ie, url):
 705         self.add_extra_info(ie_result, {
 706             'extractor': ie.IE_NAME,
 707             'webpage_url': url,
 708             'webpage_url_basename': url_basename(url),
 709             'extractor_key': ie.ie_key(),
 710         })
 711
 712     def process_ie_result(self, ie_result, download=True, extra_info={}):
 713         """
 714         Take the result of the ie(may be modified) and resolve all unresolved
 715         references (URLs, playlist items).
 716
 717         It will also download the videos if 'download'.
 718         Returns the resolved ie_result.
 719         """
 720         result_type = ie_result.get('_type', 'video')
 721
 722         if result_type in ('url', 'url_transparent'):
 723             ie_result['url'] = sanitize_url(ie_result['url'])
 724             extract_flat = self.params.get('extract_flat', False)
 725             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 726                     extract_flat is True):
 727                 if self.params.get('forcejson', False):
 728                     self.to_stdout(json.dumps(ie_result))
 729                 return ie_result
 730
 731         if result_type == 'video':
 732             self.add_extra_info(ie_result, extra_info)
 733             return self.process_video_result(ie_result, download=download)
 734         elif result_type == 'url':
 735             # We have to add extra_info to the results because it may be
 736             # contained in a playlist
 737             return self.extract_info(ie_result['url'],
 738                                      download,
 739                                      ie_key=ie_result.get('ie_key'),
 740                                      extra_info=extra_info)
 741         elif result_type == 'url_transparent':
 742             # Use the information from the embedding page
 743             info = self.extract_info(
 744                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 745                 extra_info=extra_info, download=False, process=False)
 746
 747             force_properties = dict(
 748                 (k, v) for k, v in ie_result.items() if v is not None)
 749             for f in ('_type', 'url', 'ie_key'):
 750                 if f in force_properties:
 751                     del force_properties[f]
 752             new_result = info.copy()
 753             new_result.update(force_properties)
 754
 755             assert new_result.get('_type') != 'url_transparent'
 756
 757             return self.process_ie_result(
 758                 new_result, download=download, extra_info=extra_info)
 759         elif result_type == 'playlist' or result_type == 'multi_video':
 760             # We process each entry in the playlist
 761             playlist = ie_result.get('title') or ie_result.get('id')
 762             self.to_screen('[download] Downloading playlist: %s' % playlist)
 763
 764             playlist_results = []
 765
 766             playliststart = self.params.get('playliststart', 1) - 1
 767             playlistend = self.params.get('playlistend')
 768             # For backwards compatibility, interpret -1 as whole list
 769             if playlistend == -1:
 770                 playlistend = None
 771
 772             playlistitems_str = self.params.get('playlist_items')
 773             playlistitems = None
 774             if playlistitems_str is not None:
 775                 def iter_playlistitems(format):
 776                     for string_segment in format.split(','):
 777                         if '-' in string_segment:
 778                             start, end = string_segment.split('-')
 779                             for item in range(int(start), int(end) + 1):
 780                                 yield int(item)
 781                         else:
 782                             yield int(string_segment)
 783                 playlistitems = iter_playlistitems(playlistitems_str)
 784
 785             ie_entries = ie_result['entries']
 786             if isinstance(ie_entries, list):
 787                 n_all_entries = len(ie_entries)
 788                 if playlistitems:
 789                     entries = [
 790                         ie_entries[i - 1] for i in playlistitems
 791                         if -n_all_entries <= i - 1 < n_all_entries]
 792                 else:
 793                     entries = ie_entries[playliststart:playlistend]
 794                 n_entries = len(entries)
 795                 self.to_screen(
 796                     '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
 797                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 798             elif isinstance(ie_entries, PagedList):
 799                 if playlistitems:
 800                     entries = []
 801                     for item in playlistitems:
 802                         entries.extend(ie_entries.getslice(
 803                             item - 1, item
 804                         ))
 805                 else:
 806                     entries = ie_entries.getslice(
 807                         playliststart, playlistend)
 808                 n_entries = len(entries)
 809                 self.to_screen(
 810                     '[%s] playlist %s: Downloading %d videos' %
 811                     (ie_result['extractor'], playlist, n_entries))
 812             else:  # iterable
 813                 if playlistitems:
 814                     entry_list = list(ie_entries)
 815                     entries = [entry_list[i - 1] for i in playlistitems]
 816                 else:
 817                     entries = list(itertools.islice(
 818                         ie_entries, playliststart, playlistend))
 819                 n_entries = len(entries)
 820                 self.to_screen(
 821                     '[%s] playlist %s: Downloading %d videos' %
 822                     (ie_result['extractor'], playlist, n_entries))
 823
 824             if self.params.get('playlistreverse', False):
 825                 entries = entries[::-1]
 826
 827             for i, entry in enumerate(entries, 1):
 828                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 829                 extra = {
 830                     'n_entries': n_entries,
 831                     'playlist': playlist,
 832                     'playlist_id': ie_result.get('id'),
 833                     'playlist_title': ie_result.get('title'),
 834                     'playlist_index': i + playliststart,
 835                     'extractor': ie_result['extractor'],
 836                     'webpage_url': ie_result['webpage_url'],
 837                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 838                     'extractor_key': ie_result['extractor_key'],
 839                 }
 840
 841                 reason = self._match_entry(entry, incomplete=True)
 842                 if reason is not None:
 843                     self.to_screen('[download] ' + reason)
 844                     continue
 845
 846                 entry_result = self.process_ie_result(entry,
 847                                                       download=download,
 848                                                       extra_info=extra)
 849                 playlist_results.append(entry_result)
 850             ie_result['entries'] = playlist_results
 851             self.to_screen('[download] Finished downloading playlist: %s' % playlist)
 852             return ie_result
 853         elif result_type == 'compat_list':
 854             self.report_warning(
 855                 'Extractor %s returned a compat_list result. '
 856                 'It needs to be updated.' % ie_result.get('extractor'))
 857
 858             def _fixup(r):
 859                 self.add_extra_info(
 860                     r,
 861                     {
 862                         'extractor': ie_result['extractor'],
 863                         'webpage_url': ie_result['webpage_url'],
 864                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 865                         'extractor_key': ie_result['extractor_key'],
 866                     }
 867                 )
 868                 return r
 869             ie_result['entries'] = [
 870                 self.process_ie_result(_fixup(r), download, extra_info)
 871                 for r in ie_result['entries']
 872             ]
 873             return ie_result
 874         else:
 875             raise Exception('Invalid result type: %s' % result_type)
 876
 877     def _build_format_filter(self, filter_spec):
 878         " Returns a function to filter the formats according to the filter_spec "
 879
 880         OPERATORS = {
 881             '<': operator.lt,
 882             '<=': operator.le,
 883             '>': operator.gt,
 884             '>=': operator.ge,
 885             '=': operator.eq,
 886             '!=': operator.ne,
 887         }
 888         operator_rex = re.compile(r'''(?x)\s*
 889             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
 890             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
 891             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
 892             $
 893             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
 894         m = operator_rex.search(filter_spec)
 895         if m:
 896             try:
 897                 comparison_value = int(m.group('value'))
 898             except ValueError:
 899                 comparison_value = parse_filesize(m.group('value'))
 900                 if comparison_value is None:
 901                     comparison_value = parse_filesize(m.group('value') + 'B')
 902                 if comparison_value is None:
 903                     raise ValueError(
 904                         'Invalid value %r in format specification %r' % (
 905                             m.group('value'), filter_spec))
 906             op = OPERATORS[m.group('op')]
 907
 908         if not m:
 909             STR_OPERATORS = {
 910                 '=': operator.eq,
 911                 '!=': operator.ne,
 912                 '^=': lambda attr, value: attr.startswith(value),
 913                 '$=': lambda attr, value: attr.endswith(value),
 914                 '*=': lambda attr, value: value in attr,
 915             }
 916             str_operator_rex = re.compile(r'''(?x)
 917                 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
 918                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
 919                 \s*(?P<value>[a-zA-Z0-9._-]+)
 920                 \s*$
 921                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
 922             m = str_operator_rex.search(filter_spec)
 923             if m:
 924                 comparison_value = m.group('value')
 925                 op = STR_OPERATORS[m.group('op')]
 926
 927         if not m:
 928             raise ValueError('Invalid filter specification %r' % filter_spec)
 929
 930         def _filter(f):
 931             actual_value = f.get(m.group('key'))
 932             if actual_value is None:
 933                 return m.group('none_inclusive')
 934             return op(actual_value, comparison_value)
 935         return _filter
 936
 937     def build_format_selector(self, format_spec):
 938         def syntax_error(note, start):
 939             message = (
 940                 'Invalid format specification: '
 941                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
 942             return SyntaxError(message)
 943
 944         PICKFIRST = 'PICKFIRST'
 945         MERGE = 'MERGE'
 946         SINGLE = 'SINGLE'
 947         GROUP = 'GROUP'
 948         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
 949
 950         def _parse_filter(tokens):
 951             filter_parts = []
 952             for type, string, start, _, _ in tokens:
 953                 if type == tokenize.OP and string == ']':
 954                     return ''.join(filter_parts)
 955                 else:
 956                     filter_parts.append(string)
 957
 958         def _remove_unused_ops(tokens):
 959             # Remove operators that we don't use and join them with the surrounding strings
 960             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
 961             ALLOWED_OPS = ('/', '+', ',', '(', ')')
 962             last_string, last_start, last_end, last_line = None, None, None, None
 963             for type, string, start, end, line in tokens:
 964                 if type == tokenize.OP and string == '[':
 965                     if last_string:
 966                         yield tokenize.NAME, last_string, last_start, last_end, last_line
 967                         last_string = None
 968                     yield type, string, start, end, line
 969                     # everything inside brackets will be handled by _parse_filter
 970                     for type, string, start, end, line in tokens:
 971                         yield type, string, start, end, line
 972                         if type == tokenize.OP and string == ']':
 973                             break
 974                 elif type == tokenize.OP and string in ALLOWED_OPS:
 975                     if last_string:
 976                         yield tokenize.NAME, last_string, last_start, last_end, last_line
 977                         last_string = None
 978                     yield type, string, start, end, line
 979                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
 980                     if not last_string:
 981                         last_string = string
 982                         last_start = start
 983                         last_end = end
 984                     else:
 985                         last_string += string
 986             if last_string:
 987                 yield tokenize.NAME, last_string, last_start, last_end, last_line
 988
 989         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
 990             selectors = []
 991             current_selector = None
 992             for type, string, start, _, _ in tokens:
 993                 # ENCODING is only defined in python 3.x
 994                 if type == getattr(tokenize, 'ENCODING', None):
 995                     continue
 996                 elif type in [tokenize.NAME, tokenize.NUMBER]:
 997                     current_selector = FormatSelector(SINGLE, string, [])
 998                 elif type == tokenize.OP:
 999                     if string == ')':
1000                         if not inside_group:
1001                             # ')' will be handled by the parentheses group
1002                             tokens.restore_last_token()
1003                         break
1004                     elif inside_merge and string in ['/', ',']:
1005                         tokens.restore_last_token()
1006                         break
1007                     elif inside_choice and string == ',':
1008                         tokens.restore_last_token()
1009                         break
1010                     elif string == ',':
1011                         if not current_selector:
1012                             raise syntax_error('"," must follow a format selector', start)
1013                         selectors.append(current_selector)
1014                         current_selector = None
1015                     elif string == '/':
1016                         if not current_selector:
1017                             raise syntax_error('"/" must follow a format selector', start)
1018                         first_choice = current_selector
1019                         second_choice = _parse_format_selection(tokens, inside_choice=True)
1020                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1021                     elif string == '[':
1022                         if not current_selector:
1023                             current_selector = FormatSelector(SINGLE, 'best', [])
1024                         format_filter = _parse_filter(tokens)
1025                         current_selector.filters.append(format_filter)
1026                     elif string == '(':
1027                         if current_selector:
1028                             raise syntax_error('Unexpected "("', start)
1029                         group = _parse_format_selection(tokens, inside_group=True)
1030                         current_selector = FormatSelector(GROUP, group, [])
1031                     elif string == '+':
1032                         video_selector = current_selector
1033                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
1034                         if not video_selector or not audio_selector:
1035                             raise syntax_error('"+" must be between two format selectors', start)
1036                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1037                     else:
1038                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1039                 elif type == tokenize.ENDMARKER:
1040                     break
1041             if current_selector:
1042                 selectors.append(current_selector)
1043             return selectors
1044
1045         def _build_selector_function(selector):
1046             if isinstance(selector, list):
1047                 fs = [_build_selector_function(s) for s in selector]
1048
1049                 def selector_function(formats):
1050                     for f in fs:
1051                         for format in f(formats):
1052                             yield format
1053                 return selector_function
1054             elif selector.type == GROUP:
1055                 selector_function = _build_selector_function(selector.selector)
1056             elif selector.type == PICKFIRST:
1057                 fs = [_build_selector_function(s) for s in selector.selector]
1058
1059                 def selector_function(formats):
1060                     for f in fs:
1061                         picked_formats = list(f(formats))
1062                         if picked_formats:
1063                             return picked_formats
1064                     return []
1065             elif selector.type == SINGLE:
1066                 format_spec = selector.selector
1067
1068                 def selector_function(formats):
1069                     formats = list(formats)
1070                     if not formats:
1071                         return
1072                     if format_spec == 'all':
1073                         for f in formats:
1074                             yield f
1075                     elif format_spec in ['best', 'worst', None]:
1076                         format_idx = 0 if format_spec == 'worst' else -1
1077                         audiovideo_formats = [
1078                             f for f in formats
1079                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1080                         if audiovideo_formats:
1081                             yield audiovideo_formats[format_idx]
1082                         # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
1083                         elif (all(f.get('acodec') != 'none' for f in formats) or
1084                               all(f.get('vcodec') != 'none' for f in formats)):
1085                             yield formats[format_idx]
1086                     elif format_spec == 'bestaudio':
1087                         audio_formats = [
1088                             f for f in formats
1089                             if f.get('vcodec') == 'none']
1090                         if audio_formats:
1091                             yield audio_formats[-1]
1092                     elif format_spec == 'worstaudio':
1093                         audio_formats = [
1094                             f for f in formats
1095                             if f.get('vcodec') == 'none']
1096                         if audio_formats:
1097                             yield audio_formats[0]
1098                     elif format_spec == 'bestvideo':
1099                         video_formats = [
1100                             f for f in formats
1101                             if f.get('acodec') == 'none']
1102                         if video_formats:
1103                             yield video_formats[-1]
1104                     elif format_spec == 'worstvideo':
1105                         video_formats = [
1106                             f for f in formats
1107                             if f.get('acodec') == 'none']
1108                         if video_formats:
1109                             yield video_formats[0]
1110                     else:
1111                         extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1112                         if format_spec in extensions:
1113                             filter_f = lambda f: f['ext'] == format_spec
1114                         else:
1115                             filter_f = lambda f: f['format_id'] == format_spec
1116                         matches = list(filter(filter_f, formats))
1117                         if matches:
1118                             yield matches[-1]
1119             elif selector.type == MERGE:
1120                 def _merge(formats_info):
1121                     format_1, format_2 = [f['format_id'] for f in formats_info]
1122                     # The first format must contain the video and the
1123                     # second the audio
1124                     if formats_info[0].get('vcodec') == 'none':
1125                         self.report_error('The first format must '
1126                                           'contain the video, try using '
1127                                           '"-f %s+%s"' % (format_2, format_1))
1128                         return
1129                     # Formats must be opposite (video+audio)
1130                     if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1131                         self.report_error(
1132                             'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1133                             % (format_1, format_2))
1134                         return
1135                     output_ext = (
1136                         formats_info[0]['ext']
1137                         if self.params.get('merge_output_format') is None
1138                         else self.params['merge_output_format'])
1139                     return {
1140                         'requested_formats': formats_info,
1141                         'format': '%s+%s' % (formats_info[0].get('format'),
1142                                              formats_info[1].get('format')),
1143                         'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1144                                                 formats_info[1].get('format_id')),
1145                         'width': formats_info[0].get('width'),
1146                         'height': formats_info[0].get('height'),
1147                         'resolution': formats_info[0].get('resolution'),
1148                         'fps': formats_info[0].get('fps'),
1149                         'vcodec': formats_info[0].get('vcodec'),
1150                         'vbr': formats_info[0].get('vbr'),
1151                         'stretched_ratio': formats_info[0].get('stretched_ratio'),
1152                         'acodec': formats_info[1].get('acodec'),
1153                         'abr': formats_info[1].get('abr'),
1154                         'ext': output_ext,
1155                     }
1156                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1157
1158                 def selector_function(formats):
1159                     formats = list(formats)
1160                     for pair in itertools.product(video_selector(formats), audio_selector(formats)):
1161                         yield _merge(pair)
1162
1163             filters = [self._build_format_filter(f) for f in selector.filters]
1164
1165             def final_selector(formats):
1166                 for _filter in filters:
1167                     formats = list(filter(_filter, formats))
1168                 return selector_function(formats)
1169             return final_selector
1170
1171         stream = io.BytesIO(format_spec.encode('utf-8'))
1172         try:
1173             tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1174         except tokenize.TokenError:
1175             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1176
1177         class TokenIterator(object):
1178             def __init__(self, tokens):
1179                 self.tokens = tokens
1180                 self.counter = 0
1181
1182             def __iter__(self):
1183                 return self
1184
1185             def __next__(self):
1186                 if self.counter >= len(self.tokens):
1187                     raise StopIteration()
1188                 value = self.tokens[self.counter]
1189                 self.counter += 1
1190                 return value
1191
1192             next = __next__
1193
1194             def restore_last_token(self):
1195                 self.counter -= 1
1196
1197         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1198         return _build_selector_function(parsed_selector)
1199
1200     def _calc_headers(self, info_dict):
1201         res = std_headers.copy()
1202
1203         add_headers = info_dict.get('http_headers')
1204         if add_headers:
1205             res.update(add_headers)
1206
1207         cookies = self._calc_cookies(info_dict)
1208         if cookies:
1209             res['Cookie'] = cookies
1210
1211         return res
1212
1213     def _calc_cookies(self, info_dict):
1214         pr = sanitized_Request(info_dict['url'])
1215         self.cookiejar.add_cookie_header(pr)
1216         return pr.get_header('Cookie')
1217
1218     def process_video_result(self, info_dict, download=True):
1219         assert info_dict.get('_type', 'video') == 'video'
1220
1221         if 'id' not in info_dict:
1222             raise ExtractorError('Missing "id" field in extractor result')
1223         if 'title' not in info_dict:
1224             raise ExtractorError('Missing "title" field in extractor result')
1225
1226         if 'playlist' not in info_dict:
1227             # It isn't part of a playlist
1228             info_dict['playlist'] = None
1229             info_dict['playlist_index'] = None
1230
1231         thumbnails = info_dict.get('thumbnails')
1232         if thumbnails is None:
1233             thumbnail = info_dict.get('thumbnail')
1234             if thumbnail:
1235                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1236         if thumbnails:
1237             thumbnails.sort(key=lambda t: (
1238                 t.get('preference'), t.get('width'), t.get('height'),
1239                 t.get('id'), t.get('url')))
1240             for i, t in enumerate(thumbnails):
1241                 t['url'] = sanitize_url(t['url'])
1242                 if t.get('width') and t.get('height'):
1243                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1244                 if t.get('id') is None:
1245                     t['id'] = '%d' % i
1246
1247         if self.params.get('list_thumbnails'):
1248             self.list_thumbnails(info_dict)
1249             return
1250
1251         thumbnail = info_dict.get('thumbnail')
1252         if thumbnail:
1253             info_dict['thumbnail'] = sanitize_url(thumbnail)
1254         elif thumbnails:
1255             info_dict['thumbnail'] = thumbnails[-1]['url']
1256
1257         if 'display_id' not in info_dict and 'id' in info_dict:
1258             info_dict['display_id'] = info_dict['id']
1259
1260         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1261             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1262             # see http://bugs.python.org/issue1646728)
1263             try:
1264                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1265                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1266             except (ValueError, OverflowError, OSError):
1267                 pass
1268
1269         # Auto generate title fields corresponding to the *_number fields when missing
1270         # in order to always have clean titles. This is very common for TV series.
1271         for field in ('chapter', 'season', 'episode'):
1272             if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1273                 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1274
1275         subtitles = info_dict.get('subtitles')
1276         if subtitles:
1277             for _, subtitle in subtitles.items():
1278                 for subtitle_format in subtitle:
1279                     if subtitle_format.get('url'):
1280                         subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1281                     if 'ext' not in subtitle_format:
1282                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1283
1284         if self.params.get('listsubtitles', False):
1285             if 'automatic_captions' in info_dict:
1286                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1287             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1288             return
1289         info_dict['requested_subtitles'] = self.process_subtitles(
1290             info_dict['id'], subtitles,
1291             info_dict.get('automatic_captions'))
1292
1293         # We now pick which formats have to be downloaded
1294         if info_dict.get('formats') is None:
1295             # There's only one format available
1296             formats = [info_dict]
1297         else:
1298             formats = info_dict['formats']
1299
1300         if not formats:
1301             raise ExtractorError('No video formats found!')
1302
1303         formats_dict = {}
1304
1305         # We check that all the formats have the format and format_id fields
1306         for i, format in enumerate(formats):
1307             if 'url' not in format:
1308                 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1309
1310             format['url'] = sanitize_url(format['url'])
1311
1312             if format.get('format_id') is None:
1313                 format['format_id'] = compat_str(i)
1314             else:
1315                 # Sanitize format_id from characters used in format selector expression
1316                 format['format_id'] = re.sub('[\s,/+\[\]()]', '_', format['format_id'])
1317             format_id = format['format_id']
1318             if format_id not in formats_dict:
1319                 formats_dict[format_id] = []
1320             formats_dict[format_id].append(format)
1321
1322         # Make sure all formats have unique format_id
1323         for format_id, ambiguous_formats in formats_dict.items():
1324             if len(ambiguous_formats) > 1:
1325                 for i, format in enumerate(ambiguous_formats):
1326                     format['format_id'] = '%s-%d' % (format_id, i)
1327
1328         for i, format in enumerate(formats):
1329             if format.get('format') is None:
1330                 format['format'] = '{id} - {res}{note}'.format(
1331                     id=format['format_id'],
1332                     res=self.format_resolution(format),
1333                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1334                 )
1335             # Automatically determine file extension if missing
1336             if 'ext' not in format:
1337                 format['ext'] = determine_ext(format['url']).lower()
1338             # Automatically determine protocol if missing (useful for format
1339             # selection purposes)
1340             if 'protocol' not in format:
1341                 format['protocol'] = determine_protocol(format)
1342             # Add HTTP headers, so that external programs can use them from the
1343             # json output
1344             full_format_info = info_dict.copy()
1345             full_format_info.update(format)
1346             format['http_headers'] = self._calc_headers(full_format_info)
1347
1348         # TODO Central sorting goes here
1349
1350         if formats[0] is not info_dict:
1351             # only set the 'formats' fields if the original info_dict list them
1352             # otherwise we end up with a circular reference, the first (and unique)
1353             # element in the 'formats' field in info_dict is info_dict itself,
1354             # which can't be exported to json
1355             info_dict['formats'] = formats
1356         if self.params.get('listformats'):
1357             self.list_formats(info_dict)
1358             return
1359
1360         req_format = self.params.get('format')
1361         if req_format is None:
1362             req_format_list = []
1363             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1364                     not info_dict.get('is_live')):
1365                 merger = FFmpegMergerPP(self)
1366                 if merger.available and merger.can_merge():
1367                     req_format_list.append('bestvideo+bestaudio')
1368             req_format_list.append('best')
1369             req_format = '/'.join(req_format_list)
1370         format_selector = self.build_format_selector(req_format)
1371         formats_to_download = list(format_selector(formats))
1372         if not formats_to_download:
1373             raise ExtractorError('requested format not available',
1374                                  expected=True)
1375
1376         if download:
1377             if len(formats_to_download) > 1:
1378                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1379             for format in formats_to_download:
1380                 new_info = dict(info_dict)
1381                 new_info.update(format)
1382                 self.process_info(new_info)
1383         # We update the info dict with the best quality format (backwards compatibility)
1384         info_dict.update(formats_to_download[-1])
1385         return info_dict
1386
1387     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1388         """Select the requested subtitles and their format"""
1389         available_subs = {}
1390         if normal_subtitles and self.params.get('writesubtitles'):
1391             available_subs.update(normal_subtitles)
1392         if automatic_captions and self.params.get('writeautomaticsub'):
1393             for lang, cap_info in automatic_captions.items():
1394                 if lang not in available_subs:
1395                     available_subs[lang] = cap_info
1396
1397         if (not self.params.get('writesubtitles') and not
1398                 self.params.get('writeautomaticsub') or not
1399                 available_subs):
1400             return None
1401
1402         if self.params.get('allsubtitles', False):
1403             requested_langs = available_subs.keys()
1404         else:
1405             if self.params.get('subtitleslangs', False):
1406                 requested_langs = self.params.get('subtitleslangs')
1407             elif 'en' in available_subs:
1408                 requested_langs = ['en']
1409             else:
1410                 requested_langs = [list(available_subs.keys())[0]]
1411
1412         formats_query = self.params.get('subtitlesformat', 'best')
1413         formats_preference = formats_query.split('/') if formats_query else []
1414         subs = {}
1415         for lang in requested_langs:
1416             formats = available_subs.get(lang)
1417             if formats is None:
1418                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1419                 continue
1420             for ext in formats_preference:
1421                 if ext == 'best':
1422                     f = formats[-1]
1423                     break
1424                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1425                 if matches:
1426                     f = matches[-1]
1427                     break
1428             else:
1429                 f = formats[-1]
1430                 self.report_warning(
1431                     'No subtitle format found matching "%s" for language %s, '
1432                     'using %s' % (formats_query, lang, f['ext']))
1433             subs[lang] = f
1434         return subs
1435
1436     def process_info(self, info_dict):
1437         """Process a single resolved IE result."""
1438
1439         assert info_dict.get('_type', 'video') == 'video'
1440
1441         max_downloads = self.params.get('max_downloads')
1442         if max_downloads is not None:
1443             if self._num_downloads >= int(max_downloads):
1444                 raise MaxDownloadsReached()
1445
1446         info_dict['fulltitle'] = info_dict['title']
1447         if len(info_dict['title']) > 200:
1448             info_dict['title'] = info_dict['title'][:197] + '...'
1449
1450         if 'format' not in info_dict:
1451             info_dict['format'] = info_dict['ext']
1452
1453         reason = self._match_entry(info_dict, incomplete=False)
1454         if reason is not None:
1455             self.to_screen('[download] ' + reason)
1456             return
1457
1458         self._num_downloads += 1
1459
1460         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1461
1462         # Forced printings
1463         if self.params.get('forcetitle', False):
1464             self.to_stdout(info_dict['fulltitle'])
1465         if self.params.get('forceid', False):
1466             self.to_stdout(info_dict['id'])
1467         if self.params.get('forceurl', False):
1468             if info_dict.get('requested_formats') is not None:
1469                 for f in info_dict['requested_formats']:
1470                     self.to_stdout(f['url'] + f.get('play_path', ''))
1471             else:
1472                 # For RTMP URLs, also include the playpath
1473                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1474         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1475             self.to_stdout(info_dict['thumbnail'])
1476         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1477             self.to_stdout(info_dict['description'])
1478         if self.params.get('forcefilename', False) and filename is not None:
1479             self.to_stdout(filename)
1480         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1481             self.to_stdout(formatSeconds(info_dict['duration']))
1482         if self.params.get('forceformat', False):
1483             self.to_stdout(info_dict['format'])
1484         if self.params.get('forcejson', False):
1485             self.to_stdout(json.dumps(info_dict))
1486
1487         # Do nothing else if in simulate mode
1488         if self.params.get('simulate', False):
1489             return
1490
1491         if filename is None:
1492             return
1493
1494         try:
1495             dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1496             if dn and not os.path.exists(dn):
1497                 os.makedirs(dn)
1498         except (OSError, IOError) as err:
1499             self.report_error('unable to create directory ' + error_to_compat_str(err))
1500             return
1501
1502         if self.params.get('writedescription', False):
1503             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1504             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1505                 self.to_screen('[info] Video description is already present')
1506             elif info_dict.get('description') is None:
1507                 self.report_warning('There\'s no description to write.')
1508             else:
1509                 try:
1510                     self.to_screen('[info] Writing video description to: ' + descfn)
1511                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1512                         descfile.write(info_dict['description'])
1513                 except (OSError, IOError):
1514                     self.report_error('Cannot write description file ' + descfn)
1515                     return
1516
1517         if self.params.get('writeannotations', False):
1518             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1519             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1520                 self.to_screen('[info] Video annotations are already present')
1521             else:
1522                 try:
1523                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1524                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1525                         annofile.write(info_dict['annotations'])
1526                 except (KeyError, TypeError):
1527                     self.report_warning('There are no annotations to write.')
1528                 except (OSError, IOError):
1529                     self.report_error('Cannot write annotations file: ' + annofn)
1530                     return
1531
1532         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1533                                        self.params.get('writeautomaticsub')])
1534
1535         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1536             # subtitles download errors are already managed as troubles in relevant IE
1537             # that way it will silently go on when used with unsupporting IE
1538             subtitles = info_dict['requested_subtitles']
1539             ie = self.get_info_extractor(info_dict['extractor_key'])
1540             for sub_lang, sub_info in subtitles.items():
1541                 sub_format = sub_info['ext']
1542                 if sub_info.get('data') is not None:
1543                     sub_data = sub_info['data']
1544                 else:
1545                     try:
1546                         sub_data = ie._download_webpage(
1547                             sub_info['url'], info_dict['id'], note=False)
1548                     except ExtractorError as err:
1549                         self.report_warning('Unable to download subtitle for "%s": %s' %
1550                                             (sub_lang, error_to_compat_str(err.cause)))
1551                         continue
1552                 try:
1553                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1554                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1555                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1556                     else:
1557                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1558                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1559                             subfile.write(sub_data)
1560                 except (OSError, IOError):
1561                     self.report_error('Cannot write subtitles file ' + sub_filename)
1562                     return
1563
1564         if self.params.get('writeinfojson', False):
1565             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1566             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1567                 self.to_screen('[info] Video description metadata is already present')
1568             else:
1569                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1570                 try:
1571                     write_json_file(self.filter_requested_info(info_dict), infofn)
1572                 except (OSError, IOError):
1573                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1574                     return
1575
1576         self._write_thumbnails(info_dict, filename)
1577
1578         if not self.params.get('skip_download', False):
1579             try:
1580                 def dl(name, info):
1581                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1582                     for ph in self._progress_hooks:
1583                         fd.add_progress_hook(ph)
1584                     if self.params.get('verbose'):
1585                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1586                     return fd.download(name, info)
1587
1588                 if info_dict.get('requested_formats') is not None:
1589                     downloaded = []
1590                     success = True
1591                     merger = FFmpegMergerPP(self)
1592                     if not merger.available:
1593                         postprocessors = []
1594                         self.report_warning('You have requested multiple '
1595                                             'formats but ffmpeg or avconv are not installed.'
1596                                             ' The formats won\'t be merged.')
1597                     else:
1598                         postprocessors = [merger]
1599
1600                     def compatible_formats(formats):
1601                         video, audio = formats
1602                         # Check extension
1603                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1604                         if video_ext and audio_ext:
1605                             COMPATIBLE_EXTS = (
1606                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1607                                 ('webm')
1608                             )
1609                             for exts in COMPATIBLE_EXTS:
1610                                 if video_ext in exts and audio_ext in exts:
1611                                     return True
1612                         # TODO: Check acodec/vcodec
1613                         return False
1614
1615                     filename_real_ext = os.path.splitext(filename)[1][1:]
1616                     filename_wo_ext = (
1617                         os.path.splitext(filename)[0]
1618                         if filename_real_ext == info_dict['ext']
1619                         else filename)
1620                     requested_formats = info_dict['requested_formats']
1621                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1622                         info_dict['ext'] = 'mkv'
1623                         self.report_warning(
1624                             'Requested formats are incompatible for merge and will be merged into mkv.')
1625                     # Ensure filename always has a correct extension for successful merge
1626                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1627                     if os.path.exists(encodeFilename(filename)):
1628                         self.to_screen(
1629                             '[download] %s has already been downloaded and '
1630                             'merged' % filename)
1631                     else:
1632                         for f in requested_formats:
1633                             new_info = dict(info_dict)
1634                             new_info.update(f)
1635                             fname = self.prepare_filename(new_info)
1636                             fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1637                             downloaded.append(fname)
1638                             partial_success = dl(fname, new_info)
1639                             success = success and partial_success
1640                         info_dict['__postprocessors'] = postprocessors
1641                         info_dict['__files_to_merge'] = downloaded
1642                 else:
1643                     # Just a single file
1644                     success = dl(filename, info_dict)
1645             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1646                 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1647                 return
1648             except (OSError, IOError) as err:
1649                 raise UnavailableVideoError(err)
1650             except (ContentTooShortError, ) as err:
1651                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1652                 return
1653
1654             if success and filename != '-':
1655                 # Fixup content
1656                 fixup_policy = self.params.get('fixup')
1657                 if fixup_policy is None:
1658                     fixup_policy = 'detect_or_warn'
1659
1660                 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1661
1662                 stretched_ratio = info_dict.get('stretched_ratio')
1663                 if stretched_ratio is not None and stretched_ratio != 1:
1664                     if fixup_policy == 'warn':
1665                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1666                             info_dict['id'], stretched_ratio))
1667                     elif fixup_policy == 'detect_or_warn':
1668                         stretched_pp = FFmpegFixupStretchedPP(self)
1669                         if stretched_pp.available:
1670                             info_dict.setdefault('__postprocessors', [])
1671                             info_dict['__postprocessors'].append(stretched_pp)
1672                         else:
1673                             self.report_warning(
1674                                 '%s: Non-uniform pixel ratio (%s). %s'
1675                                 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1676                     else:
1677                         assert fixup_policy in ('ignore', 'never')
1678
1679                 if (info_dict.get('requested_formats') is None and
1680                         info_dict.get('container') == 'm4a_dash'):
1681                     if fixup_policy == 'warn':
1682                         self.report_warning(
1683                             '%s: writing DASH m4a. '
1684                             'Only some players support this container.'
1685                             % info_dict['id'])
1686                     elif fixup_policy == 'detect_or_warn':
1687                         fixup_pp = FFmpegFixupM4aPP(self)
1688                         if fixup_pp.available:
1689                             info_dict.setdefault('__postprocessors', [])
1690                             info_dict['__postprocessors'].append(fixup_pp)
1691                         else:
1692                             self.report_warning(
1693                                 '%s: writing DASH m4a. '
1694                                 'Only some players support this container. %s'
1695                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1696                     else:
1697                         assert fixup_policy in ('ignore', 'never')
1698
1699                 if (info_dict.get('protocol') == 'm3u8_native' or
1700                         info_dict.get('protocol') == 'm3u8' and
1701                         self.params.get('hls_prefer_native')):
1702                     if fixup_policy == 'warn':
1703                         self.report_warning('%s: malformated aac bitstream.' % (
1704                             info_dict['id']))
1705                     elif fixup_policy == 'detect_or_warn':
1706                         fixup_pp = FFmpegFixupM3u8PP(self)
1707                         if fixup_pp.available:
1708                             info_dict.setdefault('__postprocessors', [])
1709                             info_dict['__postprocessors'].append(fixup_pp)
1710                         else:
1711                             self.report_warning(
1712                                 '%s: malformated aac bitstream. %s'
1713                                 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1714                     else:
1715                         assert fixup_policy in ('ignore', 'never')
1716
1717                 try:
1718                     self.post_process(filename, info_dict)
1719                 except (PostProcessingError) as err:
1720                     self.report_error('postprocessing: %s' % str(err))
1721                     return
1722                 self.record_download_archive(info_dict)
1723
1724     def download(self, url_list):
1725         """Download a given list of URLs."""
1726         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1727         if (len(url_list) > 1 and
1728                 '%' not in outtmpl and
1729                 self.params.get('max_downloads') != 1):
1730             raise SameFileError(outtmpl)
1731
1732         for url in url_list:
1733             try:
1734                 # It also downloads the videos
1735                 res = self.extract_info(
1736                     url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1737             except UnavailableVideoError:
1738                 self.report_error('unable to download video')
1739             except MaxDownloadsReached:
1740                 self.to_screen('[info] Maximum number of downloaded files reached.')
1741                 raise
1742             else:
1743                 if self.params.get('dump_single_json', False):
1744                     self.to_stdout(json.dumps(res))
1745
1746         return self._download_retcode
1747
1748     def download_with_info_file(self, info_filename):
1749         with contextlib.closing(fileinput.FileInput(
1750                 [info_filename], mode='r',
1751                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1752             # FileInput doesn't have a read method, we can't call json.load
1753             info = self.filter_requested_info(json.loads('\n'.join(f)))
1754         try:
1755             self.process_ie_result(info, download=True)
1756         except DownloadError:
1757             webpage_url = info.get('webpage_url')
1758             if webpage_url is not None:
1759                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1760                 return self.download([webpage_url])
1761             else:
1762                 raise
1763         return self._download_retcode
1764
1765     @staticmethod
1766     def filter_requested_info(info_dict):
1767         return dict(
1768             (k, v) for k, v in info_dict.items()
1769             if k not in ['requested_formats', 'requested_subtitles'])
1770
1771     def post_process(self, filename, ie_info):
1772         """Run all the postprocessors on the given file."""
1773         info = dict(ie_info)
1774         info['filepath'] = filename
1775         pps_chain = []
1776         if ie_info.get('__postprocessors') is not None:
1777             pps_chain.extend(ie_info['__postprocessors'])
1778         pps_chain.extend(self._pps)
1779         for pp in pps_chain:
1780             files_to_delete = []
1781             try:
1782                 files_to_delete, info = pp.run(info)
1783             except PostProcessingError as e:
1784                 self.report_error(e.msg)
1785             if files_to_delete and not self.params.get('keepvideo', False):
1786                 for old_filename in files_to_delete:
1787                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1788                     try:
1789                         os.remove(encodeFilename(old_filename))
1790                     except (IOError, OSError):
1791                         self.report_warning('Unable to remove downloaded original file')
1792
1793     def _make_archive_id(self, info_dict):
1794         # Future-proof against any change in case
1795         # and backwards compatibility with prior versions
1796         extractor = info_dict.get('extractor_key')
1797         if extractor is None:
1798             if 'id' in info_dict:
1799                 extractor = info_dict.get('ie_key')  # key in a playlist
1800         if extractor is None:
1801             return None  # Incomplete video information
1802         return extractor.lower() + ' ' + info_dict['id']
1803
1804     def in_download_archive(self, info_dict):
1805         fn = self.params.get('download_archive')
1806         if fn is None:
1807             return False
1808
1809         vid_id = self._make_archive_id(info_dict)
1810         if vid_id is None:
1811             return False  # Incomplete video information
1812
1813         try:
1814             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1815                 for line in archive_file:
1816                     if line.strip() == vid_id:
1817                         return True
1818         except IOError as ioe:
1819             if ioe.errno != errno.ENOENT:
1820                 raise
1821         return False
1822
1823     def record_download_archive(self, info_dict):
1824         fn = self.params.get('download_archive')
1825         if fn is None:
1826             return
1827         vid_id = self._make_archive_id(info_dict)
1828         assert vid_id
1829         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1830             archive_file.write(vid_id + '\n')
1831
1832     @staticmethod
1833     def format_resolution(format, default='unknown'):
1834         if format.get('vcodec') == 'none':
1835             return 'audio only'
1836         if format.get('resolution') is not None:
1837             return format['resolution']
1838         if format.get('height') is not None:
1839             if format.get('width') is not None:
1840                 res = '%sx%s' % (format['width'], format['height'])
1841             else:
1842                 res = '%sp' % format['height']
1843         elif format.get('width') is not None:
1844             res = '%dx?' % format['width']
1845         else:
1846             res = default
1847         return res
1848
1849     def _format_note(self, fdict):
1850         res = ''
1851         if fdict.get('ext') in ['f4f', 'f4m']:
1852             res += '(unsupported) '
1853         if fdict.get('language'):
1854             if res:
1855                 res += ' '
1856             res += '[%s] ' % fdict['language']
1857         if fdict.get('format_note') is not None:
1858             res += fdict['format_note'] + ' '
1859         if fdict.get('tbr') is not None:
1860             res += '%4dk ' % fdict['tbr']
1861         if fdict.get('container') is not None:
1862             if res:
1863                 res += ', '
1864             res += '%s container' % fdict['container']
1865         if (fdict.get('vcodec') is not None and
1866                 fdict.get('vcodec') != 'none'):
1867             if res:
1868                 res += ', '
1869             res += fdict['vcodec']
1870             if fdict.get('vbr') is not None:
1871                 res += '@'
1872         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1873             res += 'video@'
1874         if fdict.get('vbr') is not None:
1875             res += '%4dk' % fdict['vbr']
1876         if fdict.get('fps') is not None:
1877             if res:
1878                 res += ', '
1879             res += '%sfps' % fdict['fps']
1880         if fdict.get('acodec') is not None:
1881             if res:
1882                 res += ', '
1883             if fdict['acodec'] == 'none':
1884                 res += 'video only'
1885             else:
1886                 res += '%-5s' % fdict['acodec']
1887         elif fdict.get('abr') is not None:
1888             if res:
1889                 res += ', '
1890             res += 'audio'
1891         if fdict.get('abr') is not None:
1892             res += '@%3dk' % fdict['abr']
1893         if fdict.get('asr') is not None:
1894             res += ' (%5dHz)' % fdict['asr']
1895         if fdict.get('filesize') is not None:
1896             if res:
1897                 res += ', '
1898             res += format_bytes(fdict['filesize'])
1899         elif fdict.get('filesize_approx') is not None:
1900             if res:
1901                 res += ', '
1902             res += '~' + format_bytes(fdict['filesize_approx'])
1903         return res
1904
1905     def list_formats(self, info_dict):
1906         formats = info_dict.get('formats', [info_dict])
1907         table = [
1908             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1909             for f in formats
1910             if f.get('preference') is None or f['preference'] >= -1000]
1911         if len(formats) > 1:
1912             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1913
1914         header_line = ['format code', 'extension', 'resolution', 'note']
1915         self.to_screen(
1916             '[info] Available formats for %s:\n%s' %
1917             (info_dict['id'], render_table(header_line, table)))
1918
1919     def list_thumbnails(self, info_dict):
1920         thumbnails = info_dict.get('thumbnails')
1921         if not thumbnails:
1922             self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
1923             return
1924
1925         self.to_screen(
1926             '[info] Thumbnails for %s:' % info_dict['id'])
1927         self.to_screen(render_table(
1928             ['ID', 'width', 'height', 'URL'],
1929             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1930
1931     def list_subtitles(self, video_id, subtitles, name='subtitles'):
1932         if not subtitles:
1933             self.to_screen('%s has no %s' % (video_id, name))
1934             return
1935         self.to_screen(
1936             'Available %s for %s:' % (name, video_id))
1937         self.to_screen(render_table(
1938             ['Language', 'formats'],
1939             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1940                 for lang, formats in subtitles.items()]))
1941
1942     def urlopen(self, req):
1943         """ Start an HTTP download """
1944         if isinstance(req, compat_basestring):
1945             req = sanitized_Request(req)
1946         return self._opener.open(req, timeout=self._socket_timeout)
1947
1948     def print_debug_header(self):
1949         if not self.params.get('verbose'):
1950             return
1951
1952         if type('') is not compat_str:
1953             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1954             self.report_warning(
1955                 'Your Python is broken! Update to a newer and supported version')
1956
1957         stdout_encoding = getattr(
1958             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1959         encoding_str = (
1960             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1961                 locale.getpreferredencoding(),
1962                 sys.getfilesystemencoding(),
1963                 stdout_encoding,
1964                 self.get_encoding()))
1965         write_string(encoding_str, encoding=None)
1966
1967         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1968         if _LAZY_LOADER:
1969             self._write_string('[debug] Lazy loading extractors enabled' + '\n')
1970         try:
1971             sp = subprocess.Popen(
1972                 ['git', 'rev-parse', '--short', 'HEAD'],
1973                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1974                 cwd=os.path.dirname(os.path.abspath(__file__)))
1975             out, err = sp.communicate()
1976             out = out.decode().strip()
1977             if re.match('[0-9a-f]+', out):
1978                 self._write_string('[debug] Git HEAD: ' + out + '\n')
1979         except Exception:
1980             try:
1981                 sys.exc_clear()
1982             except Exception:
1983                 pass
1984         self._write_string('[debug] Python version %s - %s\n' % (
1985             platform.python_version(), platform_name()))
1986
1987         exe_versions = FFmpegPostProcessor.get_versions(self)
1988         exe_versions['rtmpdump'] = rtmpdump_version()
1989         exe_str = ', '.join(
1990             '%s %s' % (exe, v)
1991             for exe, v in sorted(exe_versions.items())
1992             if v
1993         )
1994         if not exe_str:
1995             exe_str = 'none'
1996         self._write_string('[debug] exe versions: %s\n' % exe_str)
1997
1998         proxy_map = {}
1999         for handler in self._opener.handlers:
2000             if hasattr(handler, 'proxies'):
2001                 proxy_map.update(handler.proxies)
2002         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2003
2004         if self.params.get('call_home', False):
2005             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2006             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2007             latest_version = self.urlopen(
2008                 'https://yt-dl.org/latest/version').read().decode('utf-8')
2009             if version_tuple(latest_version) > version_tuple(__version__):
2010                 self.report_warning(
2011                     'You are using an outdated version (newest version: %s)! '
2012                     'See https://yt-dl.org/update if you need help updating.' %
2013                     latest_version)
2014
2015     def _setup_opener(self):
2016         timeout_val = self.params.get('socket_timeout')
2017         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2018
2019         opts_cookiefile = self.params.get('cookiefile')
2020         opts_proxy = self.params.get('proxy')
2021
2022         if opts_cookiefile is None:
2023             self.cookiejar = compat_cookiejar.CookieJar()
2024         else:
2025             opts_cookiefile = compat_expanduser(opts_cookiefile)
2026             self.cookiejar = compat_cookiejar.MozillaCookieJar(
2027                 opts_cookiefile)
2028             if os.access(opts_cookiefile, os.R_OK):
2029                 self.cookiejar.load()
2030
2031         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2032         if opts_proxy is not None:
2033             if opts_proxy == '':
2034                 proxies = {}
2035             else:
2036                 proxies = {'http': opts_proxy, 'https': opts_proxy}
2037         else:
2038             proxies = compat_urllib_request.getproxies()
2039             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2040             if 'http' in proxies and 'https' not in proxies:
2041                 proxies['https'] = proxies['http']
2042         proxy_handler = PerRequestProxyHandler(proxies)
2043
2044         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2045         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2046         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2047         data_handler = compat_urllib_request_DataHandler()
2048
2049         # When passing our own FileHandler instance, build_opener won't add the
2050         # default FileHandler and allows us to disable the file protocol, which
2051         # can be used for malicious purposes (see
2052         # https://github.com/rg3/youtube-dl/issues/8227)
2053         file_handler = compat_urllib_request.FileHandler()
2054
2055         def file_open(*args, **kwargs):
2056             raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2057         file_handler.file_open = file_open
2058
2059         opener = compat_urllib_request.build_opener(
2060             proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2061
2062         # Delete the default user-agent header, which would otherwise apply in
2063         # cases where our custom HTTP handler doesn't come into play
2064         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2065         opener.addheaders = []
2066         self._opener = opener
2067
2068     def encode(self, s):
2069         if isinstance(s, bytes):
2070             return s  # Already encoded
2071
2072         try:
2073             return s.encode(self.get_encoding())
2074         except UnicodeEncodeError as err:
2075             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2076             raise
2077
2078     def get_encoding(self):
2079         encoding = self.params.get('encoding')
2080         if encoding is None:
2081             encoding = preferredencoding()
2082         return encoding
2083
2084     def _write_thumbnails(self, info_dict, filename):
2085         if self.params.get('writethumbnail', False):
2086             thumbnails = info_dict.get('thumbnails')
2087             if thumbnails:
2088                 thumbnails = [thumbnails[-1]]
2089         elif self.params.get('write_all_thumbnails', False):
2090             thumbnails = info_dict.get('thumbnails')
2091         else:
2092             return
2093
2094         if not thumbnails:
2095             # No thumbnails present, so return immediately
2096             return
2097
2098         for t in thumbnails:
2099             thumb_ext = determine_ext(t['url'], 'jpg')
2100             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2101             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2102             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2103
2104             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2105                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2106                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2107             else:
2108                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2109                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2110                 try:
2111                     uf = self.urlopen(t['url'])
2112                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2113                         shutil.copyfileobj(uf, thumbf)
2114                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2115                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2116                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2117                     self.report_warning('Unable to download thumbnail "%s": %s' %
2118                                         (t['url'], error_to_compat_str(err)))