_ Git - youtube-dl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import contextlib
   8 import datetime
   9 import errno
  10 import fileinput
  11 import io
  12 import itertools
  13 import json
  14 import locale
  15 import operator
  16 import os
  17 import platform
  18 import re
  19 import shutil
  20 import subprocess
  21 import socket
  22 import sys
  23 import time
  24 import tokenize
  25 import traceback
  26
  27 if os.name == 'nt':
  28     import ctypes
  29
  30 from .compat import (
  31     compat_basestring,
  32     compat_cookiejar,
  33     compat_expanduser,
  34     compat_get_terminal_size,
  35     compat_http_client,
  36     compat_kwargs,
  37     compat_str,
  38     compat_tokenize_tokenize,
  39     compat_urllib_error,
  40     compat_urllib_request,
  41     compat_urllib_request_DataHandler,
  42 )
  43 from .utils import (
  44     ContentTooShortError,
  45     date_from_str,
  46     DateRange,
  47     DEFAULT_OUTTMPL,
  48     determine_ext,
  49     determine_protocol,
  50     DownloadError,
  51     encode_compat_str,
  52     encodeFilename,
  53     error_to_compat_str,
  54     ExtractorError,
  55     format_bytes,
  56     formatSeconds,
  57     locked_file,
  58     make_HTTPS_handler,
  59     MaxDownloadsReached,
  60     PagedList,
  61     parse_filesize,
  62     PerRequestProxyHandler,
  63     PostProcessingError,
  64     platform_name,
  65     preferredencoding,
  66     render_table,
  67     SameFileError,
  68     sanitize_filename,
  69     sanitize_path,
  70     sanitized_Request,
  71     std_headers,
  72     subtitles_filename,
  73     UnavailableVideoError,
  74     url_basename,
  75     version_tuple,
  76     write_json_file,
  77     write_string,
  78     YoutubeDLCookieProcessor,
  79     YoutubeDLHandler,
  80     prepend_extension,
  81     replace_extension,
  82     args_to_str,
  83     age_restricted,
  84 )
  85 from .cache import Cache
  86 from .extractor import get_info_extractor, gen_extractors
  87 from .downloader import get_suitable_downloader
  88 from .downloader.rtmp import rtmpdump_version
  89 from .postprocessor import (
  90     FFmpegFixupM4aPP,
  91     FFmpegFixupStretchedPP,
  92     FFmpegMergerPP,
  93     FFmpegPostProcessor,
  94     get_postprocessor,
  95 )
  96 from .version import __version__
  97
  98
  99 class YoutubeDL(object):
 100     """YoutubeDL class.
 101
 102     YoutubeDL objects are the ones responsible of downloading the
 103     actual video file and writing it to disk if the user has requested
 104     it, among some other tasks. In most cases there should be one per
 105     program. As, given a video URL, the downloader doesn't know how to
 106     extract all the needed information, task that InfoExtractors do, it
 107     has to pass the URL to one of them.
 108
 109     For this, YoutubeDL objects have a method that allows
 110     InfoExtractors to be registered in a given order. When it is passed
 111     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 112     finds that reports being able to handle it. The InfoExtractor extracts
 113     all the information about the video or videos the URL refers to, and
 114     YoutubeDL process the extracted information, possibly using a File
 115     Downloader to download the video.
 116
 117     YoutubeDL objects accept a lot of parameters. In order not to saturate
 118     the object constructor with arguments, it receives a dictionary of
 119     options instead. These options are available through the params
 120     attribute for the InfoExtractors to use. The YoutubeDL also
 121     registers itself as the downloader in charge for the InfoExtractors
 122     that are added to it, so this is a "mutual registration".
 123
 124     Available options:
 125
 126     username:          Username for authentication purposes.
 127     password:          Password for authentication purposes.
 128     videopassword:     Password for accessing a video.
 129     usenetrc:          Use netrc for authentication instead.
 130     verbose:           Print additional info to stdout.
 131     quiet:             Do not print messages to stdout.
 132     no_warnings:       Do not print out anything for warnings.
 133     forceurl:          Force printing final URL.
 134     forcetitle:        Force printing title.
 135     forceid:           Force printing ID.
 136     forcethumbnail:    Force printing thumbnail URL.
 137     forcedescription:  Force printing description.
 138     forcefilename:     Force printing final filename.
 139     forceduration:     Force printing duration.
 140     forcejson:         Force printing info_dict as JSON.
 141     dump_single_json:  Force printing the info_dict of the whole playlist
 142                        (or video) as a single JSON line.
 143     simulate:          Do not download the video files.
 144     format:            Video format code. See options.py for more information.
 145     outtmpl:           Template for output names.
 146     restrictfilenames: Do not allow "&" and spaces in file names
 147     ignoreerrors:      Do not stop on download errors.
 148     force_generic_extractor: Force downloader to use the generic extractor
 149     nooverwrites:      Prevent overwriting files.
 150     playliststart:     Playlist item to start at.
 151     playlistend:       Playlist item to end at.
 152     playlist_items:    Specific indices of playlist to download.
 153     playlistreverse:   Download playlist items in reverse order.
 154     matchtitle:        Download only matching titles.
 155     rejecttitle:       Reject downloads for matching titles.
 156     logger:            Log messages to a logging.Logger instance.
 157     logtostderr:       Log messages to stderr instead of stdout.
 158     writedescription:  Write the video description to a .description file
 159     writeinfojson:     Write the video description to a .info.json file
 160     writeannotations:  Write the video annotations to a .annotations.xml file
 161     writethumbnail:    Write the thumbnail image to a file
 162     write_all_thumbnails:  Write all thumbnail formats to files
 163     writesubtitles:    Write the video subtitles to a file
 164     writeautomaticsub: Write the automatically generated subtitles to a file
 165     allsubtitles:      Downloads all the subtitles of the video
 166                        (requires writesubtitles or writeautomaticsub)
 167     listsubtitles:     Lists all available subtitles for the video
 168     subtitlesformat:   The format code for subtitles
 169     subtitleslangs:    List of languages of the subtitles to download
 170     keepvideo:         Keep the video file after post-processing
 171     daterange:         A DateRange object, download only if the upload_date is in the range.
 172     skip_download:     Skip the actual download of the video file
 173     cachedir:          Location of the cache files in the filesystem.
 174                        False to disable filesystem cache.
 175     noplaylist:        Download single video instead of a playlist if in doubt.
 176     age_limit:         An integer representing the user's age in years.
 177                        Unsuitable videos for the given age are skipped.
 178     min_views:         An integer representing the minimum view count the video
 179                        must have in order to not be skipped.
 180                        Videos without view count information are always
 181                        downloaded. None for no limit.
 182     max_views:         An integer representing the maximum view count.
 183                        Videos that are more popular than that are not
 184                        downloaded.
 185                        Videos without view count information are always
 186                        downloaded. None for no limit.
 187     download_archive:  File name of a file where all downloads are recorded.
 188                        Videos already present in the file are not downloaded
 189                        again.
 190     cookiefile:        File name where cookies should be read from and dumped to.
 191     nocheckcertificate:Do not verify SSL certificates
 192     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 193                        At the moment, this is only supported by YouTube.
 194     proxy:             URL of the proxy server to use
 195     cn_verification_proxy:  URL of the proxy to use for IP address verification
 196                        on Chinese sites. (Experimental)
 197     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 198     bidi_workaround:   Work around buggy terminals without bidirectional text
 199                        support, using fridibi
 200     debug_printtraffic:Print out sent and received HTTP traffic
 201     include_ads:       Download ads as well
 202     default_search:    Prepend this string if an input url is not valid.
 203                        'auto' for elaborate guessing
 204     encoding:          Use this encoding instead of the system-specified.
 205     extract_flat:      Do not resolve URLs, return the immediate result.
 206                        Pass in 'in_playlist' to only show this behavior for
 207                        playlist items.
 208     postprocessors:    A list of dictionaries, each with an entry
 209                        * key:  The name of the postprocessor. See
 210                                youtube_dl/postprocessor/__init__.py for a list.
 211                        as well as any further keyword arguments for the
 212                        postprocessor.
 213     progress_hooks:    A list of functions that get called on download
 214                        progress, with a dictionary with the entries
 215                        * status: One of "downloading", "error", or "finished".
 216                                  Check this first and ignore unknown values.
 217
 218                        If status is one of "downloading", or "finished", the
 219                        following properties may also be present:
 220                        * filename: The final filename (always present)
 221                        * tmpfilename: The filename we're currently writing to
 222                        * downloaded_bytes: Bytes on disk
 223                        * total_bytes: Size of the whole file, None if unknown
 224                        * total_bytes_estimate: Guess of the eventual file size,
 225                                                None if unavailable.
 226                        * elapsed: The number of seconds since download started.
 227                        * eta: The estimated time in seconds, None if unknown
 228                        * speed: The download speed in bytes/second, None if
 229                                 unknown
 230                        * fragment_index: The counter of the currently
 231                                          downloaded video fragment.
 232                        * fragment_count: The number of fragments (= individual
 233                                          files that will be merged)
 234
 235                        Progress hooks are guaranteed to be called at least once
 236                        (with status "finished") if the download is successful.
 237     merge_output_format: Extension to use when merging formats.
 238     fixup:             Automatically correct known faults of the file.
 239                        One of:
 240                        - "never": do nothing
 241                        - "warn": only emit a warning
 242                        - "detect_or_warn": check whether we can do anything
 243                                            about it, warn otherwise (default)
 244     source_address:    (Experimental) Client-side IP address to bind to.
 245     call_home:         Boolean, true iff we are allowed to contact the
 246                        youtube-dl servers for debugging.
 247     sleep_interval:    Number of seconds to sleep before each download.
 248     listformats:       Print an overview of available video formats and exit.
 249     list_thumbnails:   Print a table of all thumbnails and exit.
 250     match_filter:      A function that gets called with the info_dict of
 251                        every video.
 252                        If it returns a message, the video is ignored.
 253                        If it returns None, the video is downloaded.
 254                        match_filter_func in utils.py is one example for this.
 255     no_color:          Do not emit color codes in output.
 256
 257     The following options determine which downloader is picked:
 258     external_downloader: Executable of the external downloader to call.
 259                        None or unset for standard (built-in) downloader.
 260     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
 261
 262     The following parameters are not used by YoutubeDL itself, they are used by
 263     the downloader (see youtube_dl/downloader/common.py):
 264     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 265     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 266     xattr_set_filesize, external_downloader_args, hls_use_mpegts.
 267
 268     The following options are used by the post processors:
 269     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 270                        otherwise prefer avconv.
 271     postprocessor_args: A list of additional command-line arguments for the
 272                         postprocessor.
 273     """
 274
 275     params = None
 276     _ies = []
 277     _pps = []
 278     _download_retcode = None
 279     _num_downloads = None
 280     _screen_file = None
 281
 282     def __init__(self, params=None, auto_init=True):
 283         """Create a FileDownloader object with the given options."""
 284         if params is None:
 285             params = {}
 286         self._ies = []
 287         self._ies_instances = {}
 288         self._pps = []
 289         self._progress_hooks = []
 290         self._download_retcode = 0
 291         self._num_downloads = 0
 292         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 293         self._err_file = sys.stderr
 294         self.params = {
 295             # Default parameters
 296             'nocheckcertificate': False,
 297         }
 298         self.params.update(params)
 299         self.cache = Cache(self)
 300
 301         if params.get('bidi_workaround', False):
 302             try:
 303                 import pty
 304                 master, slave = pty.openpty()
 305                 width = compat_get_terminal_size().columns
 306                 if width is None:
 307                     width_args = []
 308                 else:
 309                     width_args = ['-w', str(width)]
 310                 sp_kwargs = dict(
 311                     stdin=subprocess.PIPE,
 312                     stdout=slave,
 313                     stderr=self._err_file)
 314                 try:
 315                     self._output_process = subprocess.Popen(
 316                         ['bidiv'] + width_args, **sp_kwargs
 317                     )
 318                 except OSError:
 319                     self._output_process = subprocess.Popen(
 320                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 321                 self._output_channel = os.fdopen(master, 'rb')
 322             except OSError as ose:
 323                 if ose.errno == 2:
 324                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 325                 else:
 326                     raise
 327
 328         if (sys.version_info >= (3,) and sys.platform != 'win32' and
 329                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
 330                 not params.get('restrictfilenames', False)):
 331             # On Python 3, the Unicode filesystem API will throw errors (#1474)
 332             self.report_warning(
 333                 'Assuming --restrict-filenames since file system encoding '
 334                 'cannot encode all characters. '
 335                 'Set the LC_ALL environment variable to fix this.')
 336             self.params['restrictfilenames'] = True
 337
 338         if isinstance(params.get('outtmpl'), bytes):
 339             self.report_warning(
 340                 'Parameter outtmpl is bytes, but should be a unicode string. '
 341                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 342
 343         self._setup_opener()
 344
 345         if auto_init:
 346             self.print_debug_header()
 347             self.add_default_info_extractors()
 348
 349         for pp_def_raw in self.params.get('postprocessors', []):
 350             pp_class = get_postprocessor(pp_def_raw['key'])
 351             pp_def = dict(pp_def_raw)
 352             del pp_def['key']
 353             pp = pp_class(self, **compat_kwargs(pp_def))
 354             self.add_post_processor(pp)
 355
 356         for ph in self.params.get('progress_hooks', []):
 357             self.add_progress_hook(ph)
 358
 359     def warn_if_short_id(self, argv):
 360         # short YouTube ID starting with dash?
 361         idxs = [
 362             i for i, a in enumerate(argv)
 363             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 364         if idxs:
 365             correct_argv = (
 366                 ['youtube-dl'] +
 367                 [a for i, a in enumerate(argv) if i not in idxs] +
 368                 ['--'] + [argv[i] for i in idxs]
 369             )
 370             self.report_warning(
 371                 'Long argument string detected. '
 372                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 373                 args_to_str(correct_argv))
 374
 375     def add_info_extractor(self, ie):
 376         """Add an InfoExtractor object to the end of the list."""
 377         self._ies.append(ie)
 378         self._ies_instances[ie.ie_key()] = ie
 379         ie.set_downloader(self)
 380
 381     def get_info_extractor(self, ie_key):
 382         """
 383         Get an instance of an IE with name ie_key, it will try to get one from
 384         the _ies list, if there's no instance it will create a new one and add
 385         it to the extractor list.
 386         """
 387         ie = self._ies_instances.get(ie_key)
 388         if ie is None:
 389             ie = get_info_extractor(ie_key)()
 390             self.add_info_extractor(ie)
 391         return ie
 392
 393     def add_default_info_extractors(self):
 394         """
 395         Add the InfoExtractors returned by gen_extractors to the end of the list
 396         """
 397         for ie in gen_extractors():
 398             self.add_info_extractor(ie)
 399
 400     def add_post_processor(self, pp):
 401         """Add a PostProcessor object to the end of the chain."""
 402         self._pps.append(pp)
 403         pp.set_downloader(self)
 404
 405     def add_progress_hook(self, ph):
 406         """Add the progress hook (currently only for the file downloader)"""
 407         self._progress_hooks.append(ph)
 408
 409     def _bidi_workaround(self, message):
 410         if not hasattr(self, '_output_channel'):
 411             return message
 412
 413         assert hasattr(self, '_output_process')
 414         assert isinstance(message, compat_str)
 415         line_count = message.count('\n') + 1
 416         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 417         self._output_process.stdin.flush()
 418         res = ''.join(self._output_channel.readline().decode('utf-8')
 419                       for _ in range(line_count))
 420         return res[:-len('\n')]
 421
 422     def to_screen(self, message, skip_eol=False):
 423         """Print message to stdout if not in quiet mode."""
 424         return self.to_stdout(message, skip_eol, check_quiet=True)
 425
 426     def _write_string(self, s, out=None):
 427         write_string(s, out=out, encoding=self.params.get('encoding'))
 428
 429     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 430         """Print message to stdout if not in quiet mode."""
 431         if self.params.get('logger'):
 432             self.params['logger'].debug(message)
 433         elif not check_quiet or not self.params.get('quiet', False):
 434             message = self._bidi_workaround(message)
 435             terminator = ['\n', ''][skip_eol]
 436             output = message + terminator
 437
 438             self._write_string(output, self._screen_file)
 439
 440     def to_stderr(self, message):
 441         """Print message to stderr."""
 442         assert isinstance(message, compat_str)
 443         if self.params.get('logger'):
 444             self.params['logger'].error(message)
 445         else:
 446             message = self._bidi_workaround(message)
 447             output = message + '\n'
 448             self._write_string(output, self._err_file)
 449
 450     def to_console_title(self, message):
 451         if not self.params.get('consoletitle', False):
 452             return
 453         if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 454             # c_wchar_p() might not be necessary if `message` is
 455             # already of type unicode()
 456             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 457         elif 'TERM' in os.environ:
 458             self._write_string('\033]0;%s\007' % message, self._screen_file)
 459
 460     def save_console_title(self):
 461         if not self.params.get('consoletitle', False):
 462             return
 463         if 'TERM' in os.environ:
 464             # Save the title on stack
 465             self._write_string('\033[22;0t', self._screen_file)
 466
 467     def restore_console_title(self):
 468         if not self.params.get('consoletitle', False):
 469             return
 470         if 'TERM' in os.environ:
 471             # Restore the title from stack
 472             self._write_string('\033[23;0t', self._screen_file)
 473
 474     def __enter__(self):
 475         self.save_console_title()
 476         return self
 477
 478     def __exit__(self, *args):
 479         self.restore_console_title()
 480
 481         if self.params.get('cookiefile') is not None:
 482             self.cookiejar.save()
 483
 484     def trouble(self, message=None, tb=None):
 485         """Determine action to take when a download problem appears.
 486
 487         Depending on if the downloader has been configured to ignore
 488         download errors or not, this method may throw an exception or
 489         not when errors are found, after printing the message.
 490
 491         tb, if given, is additional traceback information.
 492         """
 493         if message is not None:
 494             self.to_stderr(message)
 495         if self.params.get('verbose'):
 496             if tb is None:
 497                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 498                     tb = ''
 499                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 500                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 501                     tb += encode_compat_str(traceback.format_exc())
 502                 else:
 503                     tb_data = traceback.format_list(traceback.extract_stack())
 504                     tb = ''.join(tb_data)
 505             self.to_stderr(tb)
 506         if not self.params.get('ignoreerrors', False):
 507             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 508                 exc_info = sys.exc_info()[1].exc_info
 509             else:
 510                 exc_info = sys.exc_info()
 511             raise DownloadError(message, exc_info)
 512         self._download_retcode = 1
 513
 514     def report_warning(self, message):
 515         '''
 516         Print the message to stderr, it will be prefixed with 'WARNING:'
 517         If stderr is a tty file the 'WARNING:' will be colored
 518         '''
 519         if self.params.get('logger') is not None:
 520             self.params['logger'].warning(message)
 521         else:
 522             if self.params.get('no_warnings'):
 523                 return
 524             if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
 525                 _msg_header = '\033[0;33mWARNING:\033[0m'
 526             else:
 527                 _msg_header = 'WARNING:'
 528             warning_message = '%s %s' % (_msg_header, message)
 529             self.to_stderr(warning_message)
 530
 531     def report_error(self, message, tb=None):
 532         '''
 533         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 534         in red if stderr is a tty file.
 535         '''
 536         if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
 537             _msg_header = '\033[0;31mERROR:\033[0m'
 538         else:
 539             _msg_header = 'ERROR:'
 540         error_message = '%s %s' % (_msg_header, message)
 541         self.trouble(error_message, tb)
 542
 543     def report_file_already_downloaded(self, file_name):
 544         """Report file has already been fully downloaded."""
 545         try:
 546             self.to_screen('[download] %s has already been downloaded' % file_name)
 547         except UnicodeEncodeError:
 548             self.to_screen('[download] The file has already been downloaded')
 549
 550     def prepare_filename(self, info_dict):
 551         """Generate the output filename."""
 552         try:
 553             template_dict = dict(info_dict)
 554
 555             template_dict['epoch'] = int(time.time())
 556             autonumber_size = self.params.get('autonumber_size')
 557             if autonumber_size is None:
 558                 autonumber_size = 5
 559             autonumber_templ = '%0' + str(autonumber_size) + 'd'
 560             template_dict['autonumber'] = autonumber_templ % self._num_downloads
 561             if template_dict.get('playlist_index') is not None:
 562                 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
 563             if template_dict.get('resolution') is None:
 564                 if template_dict.get('width') and template_dict.get('height'):
 565                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 566                 elif template_dict.get('height'):
 567                     template_dict['resolution'] = '%sp' % template_dict['height']
 568                 elif template_dict.get('width'):
 569                     template_dict['resolution'] = '?x%d' % template_dict['width']
 570
 571             sanitize = lambda k, v: sanitize_filename(
 572                 compat_str(v),
 573                 restricted=self.params.get('restrictfilenames'),
 574                 is_id=(k == 'id'))
 575             template_dict = dict((k, sanitize(k, v))
 576                                  for k, v in template_dict.items()
 577                                  if v is not None)
 578             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 579
 580             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 581             tmpl = compat_expanduser(outtmpl)
 582             filename = tmpl % template_dict
 583             # Temporary fix for #4787
 584             # 'Treat' all problem characters by passing filename through preferredencoding
 585             # to workaround encoding issues with subprocess on python2 @ Windows
 586             if sys.version_info < (3, 0) and sys.platform == 'win32':
 587                 filename = encodeFilename(filename, True).decode(preferredencoding())
 588             return sanitize_path(filename)
 589         except ValueError as err:
 590             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 591             return None
 592
 593     def _match_entry(self, info_dict, incomplete):
 594         """ Returns None iff the file should be downloaded """
 595
 596         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 597         if 'title' in info_dict:
 598             # This can happen when we're just evaluating the playlist
 599             title = info_dict['title']
 600             matchtitle = self.params.get('matchtitle', False)
 601             if matchtitle:
 602                 if not re.search(matchtitle, title, re.IGNORECASE):
 603                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 604             rejecttitle = self.params.get('rejecttitle', False)
 605             if rejecttitle:
 606                 if re.search(rejecttitle, title, re.IGNORECASE):
 607                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 608         date = info_dict.get('upload_date', None)
 609         if date is not None:
 610             dateRange = self.params.get('daterange', DateRange())
 611             if date not in dateRange:
 612                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 613         view_count = info_dict.get('view_count', None)
 614         if view_count is not None:
 615             min_views = self.params.get('min_views')
 616             if min_views is not None and view_count < min_views:
 617                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 618             max_views = self.params.get('max_views')
 619             if max_views is not None and view_count > max_views:
 620                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 621         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 622             return 'Skipping "%s" because it is age restricted' % video_title
 623         if self.in_download_archive(info_dict):
 624             return '%s has already been recorded in archive' % video_title
 625
 626         if not incomplete:
 627             match_filter = self.params.get('match_filter')
 628             if match_filter is not None:
 629                 ret = match_filter(info_dict)
 630                 if ret is not None:
 631                     return ret
 632
 633         return None
 634
 635     @staticmethod
 636     def add_extra_info(info_dict, extra_info):
 637         '''Set the keys from extra_info in info dict if they are missing'''
 638         for key, value in extra_info.items():
 639             info_dict.setdefault(key, value)
 640
 641     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 642                      process=True, force_generic_extractor=False):
 643         '''
 644         Returns a list with a dictionary for each video we find.
 645         If 'download', also downloads the videos.
 646         extra_info is a dict containing the extra values to add to each result
 647         '''
 648
 649         if not ie_key and force_generic_extractor:
 650             ie_key = 'Generic'
 651
 652         if ie_key:
 653             ies = [self.get_info_extractor(ie_key)]
 654         else:
 655             ies = self._ies
 656
 657         for ie in ies:
 658             if not ie.suitable(url):
 659                 continue
 660
 661             if not ie.working():
 662                 self.report_warning('The program functionality for this site has been marked as broken, '
 663                                     'and will probably not work.')
 664
 665             try:
 666                 ie_result = ie.extract(url)
 667                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 668                     break
 669                 if isinstance(ie_result, list):
 670                     # Backwards compatibility: old IE result format
 671                     ie_result = {
 672                         '_type': 'compat_list',
 673                         'entries': ie_result,
 674                     }
 675                 self.add_default_extra_info(ie_result, ie, url)
 676                 if process:
 677                     return self.process_ie_result(ie_result, download, extra_info)
 678                 else:
 679                     return ie_result
 680             except ExtractorError as e:  # An error we somewhat expected
 681                 self.report_error(compat_str(e), e.format_traceback())
 682                 break
 683             except MaxDownloadsReached:
 684                 raise
 685             except Exception as e:
 686                 if self.params.get('ignoreerrors', False):
 687                     self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
 688                     break
 689                 else:
 690                     raise
 691         else:
 692             self.report_error('no suitable InfoExtractor for URL %s' % url)
 693
 694     def add_default_extra_info(self, ie_result, ie, url):
 695         self.add_extra_info(ie_result, {
 696             'extractor': ie.IE_NAME,
 697             'webpage_url': url,
 698             'webpage_url_basename': url_basename(url),
 699             'extractor_key': ie.ie_key(),
 700         })
 701
 702     def process_ie_result(self, ie_result, download=True, extra_info={}):
 703         """
 704         Take the result of the ie(may be modified) and resolve all unresolved
 705         references (URLs, playlist items).
 706
 707         It will also download the videos if 'download'.
 708         Returns the resolved ie_result.
 709         """
 710         result_type = ie_result.get('_type', 'video')
 711
 712         if result_type in ('url', 'url_transparent'):
 713             extract_flat = self.params.get('extract_flat', False)
 714             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 715                     extract_flat is True):
 716                 if self.params.get('forcejson', False):
 717                     self.to_stdout(json.dumps(ie_result))
 718                 return ie_result
 719
 720         if result_type == 'video':
 721             self.add_extra_info(ie_result, extra_info)
 722             return self.process_video_result(ie_result, download=download)
 723         elif result_type == 'url':
 724             # We have to add extra_info to the results because it may be
 725             # contained in a playlist
 726             return self.extract_info(ie_result['url'],
 727                                      download,
 728                                      ie_key=ie_result.get('ie_key'),
 729                                      extra_info=extra_info)
 730         elif result_type == 'url_transparent':
 731             # Use the information from the embedding page
 732             info = self.extract_info(
 733                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 734                 extra_info=extra_info, download=False, process=False)
 735
 736             force_properties = dict(
 737                 (k, v) for k, v in ie_result.items() if v is not None)
 738             for f in ('_type', 'url', 'ie_key'):
 739                 if f in force_properties:
 740                     del force_properties[f]
 741             new_result = info.copy()
 742             new_result.update(force_properties)
 743
 744             assert new_result.get('_type') != 'url_transparent'
 745
 746             return self.process_ie_result(
 747                 new_result, download=download, extra_info=extra_info)
 748         elif result_type == 'playlist' or result_type == 'multi_video':
 749             # We process each entry in the playlist
 750             playlist = ie_result.get('title', None) or ie_result.get('id', None)
 751             self.to_screen('[download] Downloading playlist: %s' % playlist)
 752
 753             playlist_results = []
 754
 755             playliststart = self.params.get('playliststart', 1) - 1
 756             playlistend = self.params.get('playlistend', None)
 757             # For backwards compatibility, interpret -1 as whole list
 758             if playlistend == -1:
 759                 playlistend = None
 760
 761             playlistitems_str = self.params.get('playlist_items', None)
 762             playlistitems = None
 763             if playlistitems_str is not None:
 764                 def iter_playlistitems(format):
 765                     for string_segment in format.split(','):
 766                         if '-' in string_segment:
 767                             start, end = string_segment.split('-')
 768                             for item in range(int(start), int(end) + 1):
 769                                 yield int(item)
 770                         else:
 771                             yield int(string_segment)
 772                 playlistitems = iter_playlistitems(playlistitems_str)
 773
 774             ie_entries = ie_result['entries']
 775             if isinstance(ie_entries, list):
 776                 n_all_entries = len(ie_entries)
 777                 if playlistitems:
 778                     entries = [
 779                         ie_entries[i - 1] for i in playlistitems
 780                         if -n_all_entries <= i - 1 < n_all_entries]
 781                 else:
 782                     entries = ie_entries[playliststart:playlistend]
 783                 n_entries = len(entries)
 784                 self.to_screen(
 785                     "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
 786                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 787             elif isinstance(ie_entries, PagedList):
 788                 if playlistitems:
 789                     entries = []
 790                     for item in playlistitems:
 791                         entries.extend(ie_entries.getslice(
 792                             item - 1, item
 793                         ))
 794                 else:
 795                     entries = ie_entries.getslice(
 796                         playliststart, playlistend)
 797                 n_entries = len(entries)
 798                 self.to_screen(
 799                     "[%s] playlist %s: Downloading %d videos" %
 800                     (ie_result['extractor'], playlist, n_entries))
 801             else:  # iterable
 802                 if playlistitems:
 803                     entry_list = list(ie_entries)
 804                     entries = [entry_list[i - 1] for i in playlistitems]
 805                 else:
 806                     entries = list(itertools.islice(
 807                         ie_entries, playliststart, playlistend))
 808                 n_entries = len(entries)
 809                 self.to_screen(
 810                     "[%s] playlist %s: Downloading %d videos" %
 811                     (ie_result['extractor'], playlist, n_entries))
 812
 813             if self.params.get('playlistreverse', False):
 814                 entries = entries[::-1]
 815
 816             for i, entry in enumerate(entries, 1):
 817                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 818                 extra = {
 819                     'n_entries': n_entries,
 820                     'playlist': playlist,
 821                     'playlist_id': ie_result.get('id'),
 822                     'playlist_title': ie_result.get('title'),
 823                     'playlist_index': i + playliststart,
 824                     'extractor': ie_result['extractor'],
 825                     'webpage_url': ie_result['webpage_url'],
 826                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 827                     'extractor_key': ie_result['extractor_key'],
 828                 }
 829
 830                 reason = self._match_entry(entry, incomplete=True)
 831                 if reason is not None:
 832                     self.to_screen('[download] ' + reason)
 833                     continue
 834
 835                 entry_result = self.process_ie_result(entry,
 836                                                       download=download,
 837                                                       extra_info=extra)
 838                 playlist_results.append(entry_result)
 839             ie_result['entries'] = playlist_results
 840             self.to_screen('[download] Finished downloading playlist: %s' % playlist)
 841             return ie_result
 842         elif result_type == 'compat_list':
 843             self.report_warning(
 844                 'Extractor %s returned a compat_list result. '
 845                 'It needs to be updated.' % ie_result.get('extractor'))
 846
 847             def _fixup(r):
 848                 self.add_extra_info(
 849                     r,
 850                     {
 851                         'extractor': ie_result['extractor'],
 852                         'webpage_url': ie_result['webpage_url'],
 853                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 854                         'extractor_key': ie_result['extractor_key'],
 855                     }
 856                 )
 857                 return r
 858             ie_result['entries'] = [
 859                 self.process_ie_result(_fixup(r), download, extra_info)
 860                 for r in ie_result['entries']
 861             ]
 862             return ie_result
 863         else:
 864             raise Exception('Invalid result type: %s' % result_type)
 865
 866     def _build_format_filter(self, filter_spec):
 867         " Returns a function to filter the formats according to the filter_spec "
 868
 869         OPERATORS = {
 870             '<': operator.lt,
 871             '<=': operator.le,
 872             '>': operator.gt,
 873             '>=': operator.ge,
 874             '=': operator.eq,
 875             '!=': operator.ne,
 876         }
 877         operator_rex = re.compile(r'''(?x)\s*
 878             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
 879             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
 880             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
 881             $
 882             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
 883         m = operator_rex.search(filter_spec)
 884         if m:
 885             try:
 886                 comparison_value = int(m.group('value'))
 887             except ValueError:
 888                 comparison_value = parse_filesize(m.group('value'))
 889                 if comparison_value is None:
 890                     comparison_value = parse_filesize(m.group('value') + 'B')
 891                 if comparison_value is None:
 892                     raise ValueError(
 893                         'Invalid value %r in format specification %r' % (
 894                             m.group('value'), filter_spec))
 895             op = OPERATORS[m.group('op')]
 896
 897         if not m:
 898             STR_OPERATORS = {
 899                 '=': operator.eq,
 900                 '!=': operator.ne,
 901                 '^=': lambda attr, value: attr.startswith(value),
 902                 '$=': lambda attr, value: attr.endswith(value),
 903                 '*=': lambda attr, value: value in attr,
 904             }
 905             str_operator_rex = re.compile(r'''(?x)
 906                 \s*(?P<key>ext|acodec|vcodec|container|protocol)
 907                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
 908                 \s*(?P<value>[a-zA-Z0-9._-]+)
 909                 \s*$
 910                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
 911             m = str_operator_rex.search(filter_spec)
 912             if m:
 913                 comparison_value = m.group('value')
 914                 op = STR_OPERATORS[m.group('op')]
 915
 916         if not m:
 917             raise ValueError('Invalid filter specification %r' % filter_spec)
 918
 919         def _filter(f):
 920             actual_value = f.get(m.group('key'))
 921             if actual_value is None:
 922                 return m.group('none_inclusive')
 923             return op(actual_value, comparison_value)
 924         return _filter
 925
 926     def build_format_selector(self, format_spec):
 927         def syntax_error(note, start):
 928             message = (
 929                 'Invalid format specification: '
 930                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
 931             return SyntaxError(message)
 932
 933         PICKFIRST = 'PICKFIRST'
 934         MERGE = 'MERGE'
 935         SINGLE = 'SINGLE'
 936         GROUP = 'GROUP'
 937         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
 938
 939         def _parse_filter(tokens):
 940             filter_parts = []
 941             for type, string, start, _, _ in tokens:
 942                 if type == tokenize.OP and string == ']':
 943                     return ''.join(filter_parts)
 944                 else:
 945                     filter_parts.append(string)
 946
 947         def _remove_unused_ops(tokens):
 948             # Remove operators that we don't use and join them with the surrounding strings
 949             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
 950             ALLOWED_OPS = ('/', '+', ',', '(', ')')
 951             last_string, last_start, last_end, last_line = None, None, None, None
 952             for type, string, start, end, line in tokens:
 953                 if type == tokenize.OP and string == '[':
 954                     if last_string:
 955                         yield tokenize.NAME, last_string, last_start, last_end, last_line
 956                         last_string = None
 957                     yield type, string, start, end, line
 958                     # everything inside brackets will be handled by _parse_filter
 959                     for type, string, start, end, line in tokens:
 960                         yield type, string, start, end, line
 961                         if type == tokenize.OP and string == ']':
 962                             break
 963                 elif type == tokenize.OP and string in ALLOWED_OPS:
 964                     if last_string:
 965                         yield tokenize.NAME, last_string, last_start, last_end, last_line
 966                         last_string = None
 967                     yield type, string, start, end, line
 968                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
 969                     if not last_string:
 970                         last_string = string
 971                         last_start = start
 972                         last_end = end
 973                     else:
 974                         last_string += string
 975             if last_string:
 976                 yield tokenize.NAME, last_string, last_start, last_end, last_line
 977
 978         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
 979             selectors = []
 980             current_selector = None
 981             for type, string, start, _, _ in tokens:
 982                 # ENCODING is only defined in python 3.x
 983                 if type == getattr(tokenize, 'ENCODING', None):
 984                     continue
 985                 elif type in [tokenize.NAME, tokenize.NUMBER]:
 986                     current_selector = FormatSelector(SINGLE, string, [])
 987                 elif type == tokenize.OP:
 988                     if string == ')':
 989                         if not inside_group:
 990                             # ')' will be handled by the parentheses group
 991                             tokens.restore_last_token()
 992                         break
 993                     elif inside_merge and string in ['/', ',']:
 994                         tokens.restore_last_token()
 995                         break
 996                     elif inside_choice and string == ',':
 997                         tokens.restore_last_token()
 998                         break
 999                     elif string == ',':
1000                         if not current_selector:
1001                             raise syntax_error('"," must follow a format selector', start)
1002                         selectors.append(current_selector)
1003                         current_selector = None
1004                     elif string == '/':
1005                         if not current_selector:
1006                             raise syntax_error('"/" must follow a format selector', start)
1007                         first_choice = current_selector
1008                         second_choice = _parse_format_selection(tokens, inside_choice=True)
1009                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1010                     elif string == '[':
1011                         if not current_selector:
1012                             current_selector = FormatSelector(SINGLE, 'best', [])
1013                         format_filter = _parse_filter(tokens)
1014                         current_selector.filters.append(format_filter)
1015                     elif string == '(':
1016                         if current_selector:
1017                             raise syntax_error('Unexpected "("', start)
1018                         group = _parse_format_selection(tokens, inside_group=True)
1019                         current_selector = FormatSelector(GROUP, group, [])
1020                     elif string == '+':
1021                         video_selector = current_selector
1022                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
1023                         if not video_selector or not audio_selector:
1024                             raise syntax_error('"+" must be between two format selectors', start)
1025                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1026                     else:
1027                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1028                 elif type == tokenize.ENDMARKER:
1029                     break
1030             if current_selector:
1031                 selectors.append(current_selector)
1032             return selectors
1033
1034         def _build_selector_function(selector):
1035             if isinstance(selector, list):
1036                 fs = [_build_selector_function(s) for s in selector]
1037
1038                 def selector_function(formats):
1039                     for f in fs:
1040                         for format in f(formats):
1041                             yield format
1042                 return selector_function
1043             elif selector.type == GROUP:
1044                 selector_function = _build_selector_function(selector.selector)
1045             elif selector.type == PICKFIRST:
1046                 fs = [_build_selector_function(s) for s in selector.selector]
1047
1048                 def selector_function(formats):
1049                     for f in fs:
1050                         picked_formats = list(f(formats))
1051                         if picked_formats:
1052                             return picked_formats
1053                     return []
1054             elif selector.type == SINGLE:
1055                 format_spec = selector.selector
1056
1057                 def selector_function(formats):
1058                     formats = list(formats)
1059                     if not formats:
1060                         return
1061                     if format_spec == 'all':
1062                         for f in formats:
1063                             yield f
1064                     elif format_spec in ['best', 'worst', None]:
1065                         format_idx = 0 if format_spec == 'worst' else -1
1066                         audiovideo_formats = [
1067                             f for f in formats
1068                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1069                         if audiovideo_formats:
1070                             yield audiovideo_formats[format_idx]
1071                         # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
1072                         elif (all(f.get('acodec') != 'none' for f in formats) or
1073                               all(f.get('vcodec') != 'none' for f in formats)):
1074                             yield formats[format_idx]
1075                     elif format_spec == 'bestaudio':
1076                         audio_formats = [
1077                             f for f in formats
1078                             if f.get('vcodec') == 'none']
1079                         if audio_formats:
1080                             yield audio_formats[-1]
1081                     elif format_spec == 'worstaudio':
1082                         audio_formats = [
1083                             f for f in formats
1084                             if f.get('vcodec') == 'none']
1085                         if audio_formats:
1086                             yield audio_formats[0]
1087                     elif format_spec == 'bestvideo':
1088                         video_formats = [
1089                             f for f in formats
1090                             if f.get('acodec') == 'none']
1091                         if video_formats:
1092                             yield video_formats[-1]
1093                     elif format_spec == 'worstvideo':
1094                         video_formats = [
1095                             f for f in formats
1096                             if f.get('acodec') == 'none']
1097                         if video_formats:
1098                             yield video_formats[0]
1099                     else:
1100                         extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1101                         if format_spec in extensions:
1102                             filter_f = lambda f: f['ext'] == format_spec
1103                         else:
1104                             filter_f = lambda f: f['format_id'] == format_spec
1105                         matches = list(filter(filter_f, formats))
1106                         if matches:
1107                             yield matches[-1]
1108             elif selector.type == MERGE:
1109                 def _merge(formats_info):
1110                     format_1, format_2 = [f['format_id'] for f in formats_info]
1111                     # The first format must contain the video and the
1112                     # second the audio
1113                     if formats_info[0].get('vcodec') == 'none':
1114                         self.report_error('The first format must '
1115                                           'contain the video, try using '
1116                                           '"-f %s+%s"' % (format_2, format_1))
1117                         return
1118                     # Formats must be opposite (video+audio)
1119                     if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1120                         self.report_error(
1121                             'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1122                             % (format_1, format_2))
1123                         return
1124                     output_ext = (
1125                         formats_info[0]['ext']
1126                         if self.params.get('merge_output_format') is None
1127                         else self.params['merge_output_format'])
1128                     return {
1129                         'requested_formats': formats_info,
1130                         'format': '%s+%s' % (formats_info[0].get('format'),
1131                                              formats_info[1].get('format')),
1132                         'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1133                                                 formats_info[1].get('format_id')),
1134                         'width': formats_info[0].get('width'),
1135                         'height': formats_info[0].get('height'),
1136                         'resolution': formats_info[0].get('resolution'),
1137                         'fps': formats_info[0].get('fps'),
1138                         'vcodec': formats_info[0].get('vcodec'),
1139                         'vbr': formats_info[0].get('vbr'),
1140                         'stretched_ratio': formats_info[0].get('stretched_ratio'),
1141                         'acodec': formats_info[1].get('acodec'),
1142                         'abr': formats_info[1].get('abr'),
1143                         'ext': output_ext,
1144                     }
1145                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1146
1147                 def selector_function(formats):
1148                     formats = list(formats)
1149                     for pair in itertools.product(video_selector(formats), audio_selector(formats)):
1150                         yield _merge(pair)
1151
1152             filters = [self._build_format_filter(f) for f in selector.filters]
1153
1154             def final_selector(formats):
1155                 for _filter in filters:
1156                     formats = list(filter(_filter, formats))
1157                 return selector_function(formats)
1158             return final_selector
1159
1160         stream = io.BytesIO(format_spec.encode('utf-8'))
1161         try:
1162             tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1163         except tokenize.TokenError:
1164             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1165
1166         class TokenIterator(object):
1167             def __init__(self, tokens):
1168                 self.tokens = tokens
1169                 self.counter = 0
1170
1171             def __iter__(self):
1172                 return self
1173
1174             def __next__(self):
1175                 if self.counter >= len(self.tokens):
1176                     raise StopIteration()
1177                 value = self.tokens[self.counter]
1178                 self.counter += 1
1179                 return value
1180
1181             next = __next__
1182
1183             def restore_last_token(self):
1184                 self.counter -= 1
1185
1186         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1187         return _build_selector_function(parsed_selector)
1188
1189     def _calc_headers(self, info_dict):
1190         res = std_headers.copy()
1191
1192         add_headers = info_dict.get('http_headers')
1193         if add_headers:
1194             res.update(add_headers)
1195
1196         cookies = self._calc_cookies(info_dict)
1197         if cookies:
1198             res['Cookie'] = cookies
1199
1200         return res
1201
1202     def _calc_cookies(self, info_dict):
1203         pr = sanitized_Request(info_dict['url'])
1204         self.cookiejar.add_cookie_header(pr)
1205         return pr.get_header('Cookie')
1206
1207     def process_video_result(self, info_dict, download=True):
1208         assert info_dict.get('_type', 'video') == 'video'
1209
1210         if 'id' not in info_dict:
1211             raise ExtractorError('Missing "id" field in extractor result')
1212         if 'title' not in info_dict:
1213             raise ExtractorError('Missing "title" field in extractor result')
1214
1215         if 'playlist' not in info_dict:
1216             # It isn't part of a playlist
1217             info_dict['playlist'] = None
1218             info_dict['playlist_index'] = None
1219
1220         thumbnails = info_dict.get('thumbnails')
1221         if thumbnails is None:
1222             thumbnail = info_dict.get('thumbnail')
1223             if thumbnail:
1224                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1225         if thumbnails:
1226             thumbnails.sort(key=lambda t: (
1227                 t.get('preference'), t.get('width'), t.get('height'),
1228                 t.get('id'), t.get('url')))
1229             for i, t in enumerate(thumbnails):
1230                 if t.get('width') and t.get('height'):
1231                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1232                 if t.get('id') is None:
1233                     t['id'] = '%d' % i
1234
1235         if thumbnails and 'thumbnail' not in info_dict:
1236             info_dict['thumbnail'] = thumbnails[-1]['url']
1237
1238         if 'display_id' not in info_dict and 'id' in info_dict:
1239             info_dict['display_id'] = info_dict['id']
1240
1241         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1242             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1243             # see http://bugs.python.org/issue1646728)
1244             try:
1245                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1246                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1247             except (ValueError, OverflowError, OSError):
1248                 pass
1249
1250         # Auto generate title fields corresponding to the *_number fields when missing
1251         # in order to always have clean titles. This is very common for TV series.
1252         for field in ('chapter', 'season', 'episode'):
1253             if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1254                 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1255
1256         subtitles = info_dict.get('subtitles')
1257         if subtitles:
1258             for _, subtitle in subtitles.items():
1259                 for subtitle_format in subtitle:
1260                     if 'ext' not in subtitle_format:
1261                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1262
1263         if self.params.get('listsubtitles', False):
1264             if 'automatic_captions' in info_dict:
1265                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1266             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1267             return
1268         info_dict['requested_subtitles'] = self.process_subtitles(
1269             info_dict['id'], subtitles,
1270             info_dict.get('automatic_captions'))
1271
1272         # We now pick which formats have to be downloaded
1273         if info_dict.get('formats') is None:
1274             # There's only one format available
1275             formats = [info_dict]
1276         else:
1277             formats = info_dict['formats']
1278
1279         if not formats:
1280             raise ExtractorError('No video formats found!')
1281
1282         formats_dict = {}
1283
1284         # We check that all the formats have the format and format_id fields
1285         for i, format in enumerate(formats):
1286             if 'url' not in format:
1287                 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1288
1289             if format.get('format_id') is None:
1290                 format['format_id'] = compat_str(i)
1291             else:
1292                 # Sanitize format_id from characters used in format selector expression
1293                 format['format_id'] = re.sub('[\s,/+\[\]()]', '_', format['format_id'])
1294             format_id = format['format_id']
1295             if format_id not in formats_dict:
1296                 formats_dict[format_id] = []
1297             formats_dict[format_id].append(format)
1298
1299         # Make sure all formats have unique format_id
1300         for format_id, ambiguous_formats in formats_dict.items():
1301             if len(ambiguous_formats) > 1:
1302                 for i, format in enumerate(ambiguous_formats):
1303                     format['format_id'] = '%s-%d' % (format_id, i)
1304
1305         for i, format in enumerate(formats):
1306             if format.get('format') is None:
1307                 format['format'] = '{id} - {res}{note}'.format(
1308                     id=format['format_id'],
1309                     res=self.format_resolution(format),
1310                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1311                 )
1312             # Automatically determine file extension if missing
1313             if 'ext' not in format:
1314                 format['ext'] = determine_ext(format['url']).lower()
1315             # Automatically determine protocol if missing (useful for format
1316             # selection purposes)
1317             if 'protocol' not in format:
1318                 format['protocol'] = determine_protocol(format)
1319             # Add HTTP headers, so that external programs can use them from the
1320             # json output
1321             full_format_info = info_dict.copy()
1322             full_format_info.update(format)
1323             format['http_headers'] = self._calc_headers(full_format_info)
1324
1325         # TODO Central sorting goes here
1326
1327         if formats[0] is not info_dict:
1328             # only set the 'formats' fields if the original info_dict list them
1329             # otherwise we end up with a circular reference, the first (and unique)
1330             # element in the 'formats' field in info_dict is info_dict itself,
1331             # which can't be exported to json
1332             info_dict['formats'] = formats
1333         if self.params.get('listformats'):
1334             self.list_formats(info_dict)
1335             return
1336         if self.params.get('list_thumbnails'):
1337             self.list_thumbnails(info_dict)
1338             return
1339
1340         req_format = self.params.get('format')
1341         if req_format is None:
1342             req_format_list = []
1343             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1344                     info_dict['extractor'] in ['youtube', 'ted'] and
1345                     not info_dict.get('is_live')):
1346                 merger = FFmpegMergerPP(self)
1347                 if merger.available and merger.can_merge():
1348                     req_format_list.append('bestvideo+bestaudio')
1349             req_format_list.append('best')
1350             req_format = '/'.join(req_format_list)
1351         format_selector = self.build_format_selector(req_format)
1352         formats_to_download = list(format_selector(formats))
1353         if not formats_to_download:
1354             raise ExtractorError('requested format not available',
1355                                  expected=True)
1356
1357         if download:
1358             if len(formats_to_download) > 1:
1359                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1360             for format in formats_to_download:
1361                 new_info = dict(info_dict)
1362                 new_info.update(format)
1363                 self.process_info(new_info)
1364         # We update the info dict with the best quality format (backwards compatibility)
1365         info_dict.update(formats_to_download[-1])
1366         return info_dict
1367
1368     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1369         """Select the requested subtitles and their format"""
1370         available_subs = {}
1371         if normal_subtitles and self.params.get('writesubtitles'):
1372             available_subs.update(normal_subtitles)
1373         if automatic_captions and self.params.get('writeautomaticsub'):
1374             for lang, cap_info in automatic_captions.items():
1375                 if lang not in available_subs:
1376                     available_subs[lang] = cap_info
1377
1378         if (not self.params.get('writesubtitles') and not
1379                 self.params.get('writeautomaticsub') or not
1380                 available_subs):
1381             return None
1382
1383         if self.params.get('allsubtitles', False):
1384             requested_langs = available_subs.keys()
1385         else:
1386             if self.params.get('subtitleslangs', False):
1387                 requested_langs = self.params.get('subtitleslangs')
1388             elif 'en' in available_subs:
1389                 requested_langs = ['en']
1390             else:
1391                 requested_langs = [list(available_subs.keys())[0]]
1392
1393         formats_query = self.params.get('subtitlesformat', 'best')
1394         formats_preference = formats_query.split('/') if formats_query else []
1395         subs = {}
1396         for lang in requested_langs:
1397             formats = available_subs.get(lang)
1398             if formats is None:
1399                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1400                 continue
1401             for ext in formats_preference:
1402                 if ext == 'best':
1403                     f = formats[-1]
1404                     break
1405                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1406                 if matches:
1407                     f = matches[-1]
1408                     break
1409             else:
1410                 f = formats[-1]
1411                 self.report_warning(
1412                     'No subtitle format found matching "%s" for language %s, '
1413                     'using %s' % (formats_query, lang, f['ext']))
1414             subs[lang] = f
1415         return subs
1416
1417     def process_info(self, info_dict):
1418         """Process a single resolved IE result."""
1419
1420         assert info_dict.get('_type', 'video') == 'video'
1421
1422         max_downloads = self.params.get('max_downloads')
1423         if max_downloads is not None:
1424             if self._num_downloads >= int(max_downloads):
1425                 raise MaxDownloadsReached()
1426
1427         info_dict['fulltitle'] = info_dict['title']
1428         if len(info_dict['title']) > 200:
1429             info_dict['title'] = info_dict['title'][:197] + '...'
1430
1431         if 'format' not in info_dict:
1432             info_dict['format'] = info_dict['ext']
1433
1434         reason = self._match_entry(info_dict, incomplete=False)
1435         if reason is not None:
1436             self.to_screen('[download] ' + reason)
1437             return
1438
1439         self._num_downloads += 1
1440
1441         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1442
1443         # Forced printings
1444         if self.params.get('forcetitle', False):
1445             self.to_stdout(info_dict['fulltitle'])
1446         if self.params.get('forceid', False):
1447             self.to_stdout(info_dict['id'])
1448         if self.params.get('forceurl', False):
1449             if info_dict.get('requested_formats') is not None:
1450                 for f in info_dict['requested_formats']:
1451                     self.to_stdout(f['url'] + f.get('play_path', ''))
1452             else:
1453                 # For RTMP URLs, also include the playpath
1454                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1455         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1456             self.to_stdout(info_dict['thumbnail'])
1457         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1458             self.to_stdout(info_dict['description'])
1459         if self.params.get('forcefilename', False) and filename is not None:
1460             self.to_stdout(filename)
1461         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1462             self.to_stdout(formatSeconds(info_dict['duration']))
1463         if self.params.get('forceformat', False):
1464             self.to_stdout(info_dict['format'])
1465         if self.params.get('forcejson', False):
1466             self.to_stdout(json.dumps(info_dict))
1467
1468         # Do nothing else if in simulate mode
1469         if self.params.get('simulate', False):
1470             return
1471
1472         if filename is None:
1473             return
1474
1475         try:
1476             dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1477             if dn and not os.path.exists(dn):
1478                 os.makedirs(dn)
1479         except (OSError, IOError) as err:
1480             self.report_error('unable to create directory ' + error_to_compat_str(err))
1481             return
1482
1483         if self.params.get('writedescription', False):
1484             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1485             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1486                 self.to_screen('[info] Video description is already present')
1487             elif info_dict.get('description') is None:
1488                 self.report_warning('There\'s no description to write.')
1489             else:
1490                 try:
1491                     self.to_screen('[info] Writing video description to: ' + descfn)
1492                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1493                         descfile.write(info_dict['description'])
1494                 except (OSError, IOError):
1495                     self.report_error('Cannot write description file ' + descfn)
1496                     return
1497
1498         if self.params.get('writeannotations', False):
1499             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1500             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1501                 self.to_screen('[info] Video annotations are already present')
1502             else:
1503                 try:
1504                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1505                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1506                         annofile.write(info_dict['annotations'])
1507                 except (KeyError, TypeError):
1508                     self.report_warning('There are no annotations to write.')
1509                 except (OSError, IOError):
1510                     self.report_error('Cannot write annotations file: ' + annofn)
1511                     return
1512
1513         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1514                                        self.params.get('writeautomaticsub')])
1515
1516         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1517             # subtitles download errors are already managed as troubles in relevant IE
1518             # that way it will silently go on when used with unsupporting IE
1519             subtitles = info_dict['requested_subtitles']
1520             ie = self.get_info_extractor(info_dict['extractor_key'])
1521             for sub_lang, sub_info in subtitles.items():
1522                 sub_format = sub_info['ext']
1523                 if sub_info.get('data') is not None:
1524                     sub_data = sub_info['data']
1525                 else:
1526                     try:
1527                         sub_data = ie._download_webpage(
1528                             sub_info['url'], info_dict['id'], note=False)
1529                     except ExtractorError as err:
1530                         self.report_warning('Unable to download subtitle for "%s": %s' %
1531                                             (sub_lang, error_to_compat_str(err.cause)))
1532                         continue
1533                 try:
1534                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1535                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1536                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1537                     else:
1538                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1539                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1540                             subfile.write(sub_data)
1541                 except (OSError, IOError):
1542                     self.report_error('Cannot write subtitles file ' + sub_filename)
1543                     return
1544
1545         if self.params.get('writeinfojson', False):
1546             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1547             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1548                 self.to_screen('[info] Video description metadata is already present')
1549             else:
1550                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1551                 try:
1552                     write_json_file(self.filter_requested_info(info_dict), infofn)
1553                 except (OSError, IOError):
1554                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1555                     return
1556
1557         self._write_thumbnails(info_dict, filename)
1558
1559         if not self.params.get('skip_download', False):
1560             try:
1561                 def dl(name, info):
1562                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1563                     for ph in self._progress_hooks:
1564                         fd.add_progress_hook(ph)
1565                     if self.params.get('verbose'):
1566                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1567                     return fd.download(name, info)
1568
1569                 if info_dict.get('requested_formats') is not None:
1570                     downloaded = []
1571                     success = True
1572                     merger = FFmpegMergerPP(self)
1573                     if not merger.available:
1574                         postprocessors = []
1575                         self.report_warning('You have requested multiple '
1576                                             'formats but ffmpeg or avconv are not installed.'
1577                                             ' The formats won\'t be merged.')
1578                     else:
1579                         postprocessors = [merger]
1580
1581                     def compatible_formats(formats):
1582                         video, audio = formats
1583                         # Check extension
1584                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1585                         if video_ext and audio_ext:
1586                             COMPATIBLE_EXTS = (
1587                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1588                                 ('webm')
1589                             )
1590                             for exts in COMPATIBLE_EXTS:
1591                                 if video_ext in exts and audio_ext in exts:
1592                                     return True
1593                         # TODO: Check acodec/vcodec
1594                         return False
1595
1596                     filename_real_ext = os.path.splitext(filename)[1][1:]
1597                     filename_wo_ext = (
1598                         os.path.splitext(filename)[0]
1599                         if filename_real_ext == info_dict['ext']
1600                         else filename)
1601                     requested_formats = info_dict['requested_formats']
1602                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1603                         info_dict['ext'] = 'mkv'
1604                         self.report_warning(
1605                             'Requested formats are incompatible for merge and will be merged into mkv.')
1606                     # Ensure filename always has a correct extension for successful merge
1607                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1608                     if os.path.exists(encodeFilename(filename)):
1609                         self.to_screen(
1610                             '[download] %s has already been downloaded and '
1611                             'merged' % filename)
1612                     else:
1613                         for f in requested_formats:
1614                             new_info = dict(info_dict)
1615                             new_info.update(f)
1616                             fname = self.prepare_filename(new_info)
1617                             fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1618                             downloaded.append(fname)
1619                             partial_success = dl(fname, new_info)
1620                             success = success and partial_success
1621                         info_dict['__postprocessors'] = postprocessors
1622                         info_dict['__files_to_merge'] = downloaded
1623                 else:
1624                     # Just a single file
1625                     success = dl(filename, info_dict)
1626             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1627                 self.report_error('unable to download video data: %s' % str(err))
1628                 return
1629             except (OSError, IOError) as err:
1630                 raise UnavailableVideoError(err)
1631             except (ContentTooShortError, ) as err:
1632                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1633                 return
1634
1635             if success:
1636                 # Fixup content
1637                 fixup_policy = self.params.get('fixup')
1638                 if fixup_policy is None:
1639                     fixup_policy = 'detect_or_warn'
1640
1641                 stretched_ratio = info_dict.get('stretched_ratio')
1642                 if stretched_ratio is not None and stretched_ratio != 1:
1643                     if fixup_policy == 'warn':
1644                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1645                             info_dict['id'], stretched_ratio))
1646                     elif fixup_policy == 'detect_or_warn':
1647                         stretched_pp = FFmpegFixupStretchedPP(self)
1648                         if stretched_pp.available:
1649                             info_dict.setdefault('__postprocessors', [])
1650                             info_dict['__postprocessors'].append(stretched_pp)
1651                         else:
1652                             self.report_warning(
1653                                 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1654                                     info_dict['id'], stretched_ratio))
1655                     else:
1656                         assert fixup_policy in ('ignore', 'never')
1657
1658                 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1659                     if fixup_policy == 'warn':
1660                         self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1661                             info_dict['id']))
1662                     elif fixup_policy == 'detect_or_warn':
1663                         fixup_pp = FFmpegFixupM4aPP(self)
1664                         if fixup_pp.available:
1665                             info_dict.setdefault('__postprocessors', [])
1666                             info_dict['__postprocessors'].append(fixup_pp)
1667                         else:
1668                             self.report_warning(
1669                                 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1670                                     info_dict['id']))
1671                     else:
1672                         assert fixup_policy in ('ignore', 'never')
1673
1674                 try:
1675                     self.post_process(filename, info_dict)
1676                 except (PostProcessingError) as err:
1677                     self.report_error('postprocessing: %s' % str(err))
1678                     return
1679                 self.record_download_archive(info_dict)
1680
1681     def download(self, url_list):
1682         """Download a given list of URLs."""
1683         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1684         if (len(url_list) > 1 and
1685                 '%' not in outtmpl and
1686                 self.params.get('max_downloads') != 1):
1687             raise SameFileError(outtmpl)
1688
1689         for url in url_list:
1690             try:
1691                 # It also downloads the videos
1692                 res = self.extract_info(
1693                     url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1694             except UnavailableVideoError:
1695                 self.report_error('unable to download video')
1696             except MaxDownloadsReached:
1697                 self.to_screen('[info] Maximum number of downloaded files reached.')
1698                 raise
1699             else:
1700                 if self.params.get('dump_single_json', False):
1701                     self.to_stdout(json.dumps(res))
1702
1703         return self._download_retcode
1704
1705     def download_with_info_file(self, info_filename):
1706         with contextlib.closing(fileinput.FileInput(
1707                 [info_filename], mode='r',
1708                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1709             # FileInput doesn't have a read method, we can't call json.load
1710             info = self.filter_requested_info(json.loads('\n'.join(f)))
1711         try:
1712             self.process_ie_result(info, download=True)
1713         except DownloadError:
1714             webpage_url = info.get('webpage_url')
1715             if webpage_url is not None:
1716                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1717                 return self.download([webpage_url])
1718             else:
1719                 raise
1720         return self._download_retcode
1721
1722     @staticmethod
1723     def filter_requested_info(info_dict):
1724         return dict(
1725             (k, v) for k, v in info_dict.items()
1726             if k not in ['requested_formats', 'requested_subtitles'])
1727
1728     def post_process(self, filename, ie_info):
1729         """Run all the postprocessors on the given file."""
1730         info = dict(ie_info)
1731         info['filepath'] = filename
1732         pps_chain = []
1733         if ie_info.get('__postprocessors') is not None:
1734             pps_chain.extend(ie_info['__postprocessors'])
1735         pps_chain.extend(self._pps)
1736         for pp in pps_chain:
1737             files_to_delete = []
1738             try:
1739                 files_to_delete, info = pp.run(info)
1740             except PostProcessingError as e:
1741                 self.report_error(e.msg)
1742             if files_to_delete and not self.params.get('keepvideo', False):
1743                 for old_filename in files_to_delete:
1744                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1745                     try:
1746                         os.remove(encodeFilename(old_filename))
1747                     except (IOError, OSError):
1748                         self.report_warning('Unable to remove downloaded original file')
1749
1750     def _make_archive_id(self, info_dict):
1751         # Future-proof against any change in case
1752         # and backwards compatibility with prior versions
1753         extractor = info_dict.get('extractor_key')
1754         if extractor is None:
1755             if 'id' in info_dict:
1756                 extractor = info_dict.get('ie_key')  # key in a playlist
1757         if extractor is None:
1758             return None  # Incomplete video information
1759         return extractor.lower() + ' ' + info_dict['id']
1760
1761     def in_download_archive(self, info_dict):
1762         fn = self.params.get('download_archive')
1763         if fn is None:
1764             return False
1765
1766         vid_id = self._make_archive_id(info_dict)
1767         if vid_id is None:
1768             return False  # Incomplete video information
1769
1770         try:
1771             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1772                 for line in archive_file:
1773                     if line.strip() == vid_id:
1774                         return True
1775         except IOError as ioe:
1776             if ioe.errno != errno.ENOENT:
1777                 raise
1778         return False
1779
1780     def record_download_archive(self, info_dict):
1781         fn = self.params.get('download_archive')
1782         if fn is None:
1783             return
1784         vid_id = self._make_archive_id(info_dict)
1785         assert vid_id
1786         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1787             archive_file.write(vid_id + '\n')
1788
1789     @staticmethod
1790     def format_resolution(format, default='unknown'):
1791         if format.get('vcodec') == 'none':
1792             return 'audio only'
1793         if format.get('resolution') is not None:
1794             return format['resolution']
1795         if format.get('height') is not None:
1796             if format.get('width') is not None:
1797                 res = '%sx%s' % (format['width'], format['height'])
1798             else:
1799                 res = '%sp' % format['height']
1800         elif format.get('width') is not None:
1801             res = '%dx?' % format['width']
1802         else:
1803             res = default
1804         return res
1805
1806     def _format_note(self, fdict):
1807         res = ''
1808         if fdict.get('ext') in ['f4f', 'f4m']:
1809             res += '(unsupported) '
1810         if fdict.get('language'):
1811             if res:
1812                 res += ' '
1813             res += '[%s]' % fdict['language']
1814         if fdict.get('format_note') is not None:
1815             res += fdict['format_note'] + ' '
1816         if fdict.get('tbr') is not None:
1817             res += '%4dk ' % fdict['tbr']
1818         if fdict.get('container') is not None:
1819             if res:
1820                 res += ', '
1821             res += '%s container' % fdict['container']
1822         if (fdict.get('vcodec') is not None and
1823                 fdict.get('vcodec') != 'none'):
1824             if res:
1825                 res += ', '
1826             res += fdict['vcodec']
1827             if fdict.get('vbr') is not None:
1828                 res += '@'
1829         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1830             res += 'video@'
1831         if fdict.get('vbr') is not None:
1832             res += '%4dk' % fdict['vbr']
1833         if fdict.get('fps') is not None:
1834             res += ', %sfps' % fdict['fps']
1835         if fdict.get('acodec') is not None:
1836             if res:
1837                 res += ', '
1838             if fdict['acodec'] == 'none':
1839                 res += 'video only'
1840             else:
1841                 res += '%-5s' % fdict['acodec']
1842         elif fdict.get('abr') is not None:
1843             if res:
1844                 res += ', '
1845             res += 'audio'
1846         if fdict.get('abr') is not None:
1847             res += '@%3dk' % fdict['abr']
1848         if fdict.get('asr') is not None:
1849             res += ' (%5dHz)' % fdict['asr']
1850         if fdict.get('filesize') is not None:
1851             if res:
1852                 res += ', '
1853             res += format_bytes(fdict['filesize'])
1854         elif fdict.get('filesize_approx') is not None:
1855             if res:
1856                 res += ', '
1857             res += '~' + format_bytes(fdict['filesize_approx'])
1858         return res
1859
1860     def list_formats(self, info_dict):
1861         formats = info_dict.get('formats', [info_dict])
1862         table = [
1863             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1864             for f in formats
1865             if f.get('preference') is None or f['preference'] >= -1000]
1866         if len(formats) > 1:
1867             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1868
1869         header_line = ['format code', 'extension', 'resolution', 'note']
1870         self.to_screen(
1871             '[info] Available formats for %s:\n%s' %
1872             (info_dict['id'], render_table(header_line, table)))
1873
1874     def list_thumbnails(self, info_dict):
1875         thumbnails = info_dict.get('thumbnails')
1876         if not thumbnails:
1877             tn_url = info_dict.get('thumbnail')
1878             if tn_url:
1879                 thumbnails = [{'id': '0', 'url': tn_url}]
1880             else:
1881                 self.to_screen(
1882                     '[info] No thumbnails present for %s' % info_dict['id'])
1883                 return
1884
1885         self.to_screen(
1886             '[info] Thumbnails for %s:' % info_dict['id'])
1887         self.to_screen(render_table(
1888             ['ID', 'width', 'height', 'URL'],
1889             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1890
1891     def list_subtitles(self, video_id, subtitles, name='subtitles'):
1892         if not subtitles:
1893             self.to_screen('%s has no %s' % (video_id, name))
1894             return
1895         self.to_screen(
1896             'Available %s for %s:' % (name, video_id))
1897         self.to_screen(render_table(
1898             ['Language', 'formats'],
1899             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1900                 for lang, formats in subtitles.items()]))
1901
1902     def urlopen(self, req):
1903         """ Start an HTTP download """
1904         if isinstance(req, compat_basestring):
1905             req = sanitized_Request(req)
1906         return self._opener.open(req, timeout=self._socket_timeout)
1907
1908     def print_debug_header(self):
1909         if not self.params.get('verbose'):
1910             return
1911
1912         if type('') is not compat_str:
1913             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1914             self.report_warning(
1915                 'Your Python is broken! Update to a newer and supported version')
1916
1917         stdout_encoding = getattr(
1918             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1919         encoding_str = (
1920             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1921                 locale.getpreferredencoding(),
1922                 sys.getfilesystemencoding(),
1923                 stdout_encoding,
1924                 self.get_encoding()))
1925         write_string(encoding_str, encoding=None)
1926
1927         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1928         try:
1929             sp = subprocess.Popen(
1930                 ['git', 'rev-parse', '--short', 'HEAD'],
1931                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1932                 cwd=os.path.dirname(os.path.abspath(__file__)))
1933             out, err = sp.communicate()
1934             out = out.decode().strip()
1935             if re.match('[0-9a-f]+', out):
1936                 self._write_string('[debug] Git HEAD: ' + out + '\n')
1937         except Exception:
1938             try:
1939                 sys.exc_clear()
1940             except Exception:
1941                 pass
1942         self._write_string('[debug] Python version %s - %s\n' % (
1943             platform.python_version(), platform_name()))
1944
1945         exe_versions = FFmpegPostProcessor.get_versions(self)
1946         exe_versions['rtmpdump'] = rtmpdump_version()
1947         exe_str = ', '.join(
1948             '%s %s' % (exe, v)
1949             for exe, v in sorted(exe_versions.items())
1950             if v
1951         )
1952         if not exe_str:
1953             exe_str = 'none'
1954         self._write_string('[debug] exe versions: %s\n' % exe_str)
1955
1956         proxy_map = {}
1957         for handler in self._opener.handlers:
1958             if hasattr(handler, 'proxies'):
1959                 proxy_map.update(handler.proxies)
1960         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1961
1962         if self.params.get('call_home', False):
1963             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1964             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1965             latest_version = self.urlopen(
1966                 'https://yt-dl.org/latest/version').read().decode('utf-8')
1967             if version_tuple(latest_version) > version_tuple(__version__):
1968                 self.report_warning(
1969                     'You are using an outdated version (newest version: %s)! '
1970                     'See https://yt-dl.org/update if you need help updating.' %
1971                     latest_version)
1972
1973     def _setup_opener(self):
1974         timeout_val = self.params.get('socket_timeout')
1975         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1976
1977         opts_cookiefile = self.params.get('cookiefile')
1978         opts_proxy = self.params.get('proxy')
1979
1980         if opts_cookiefile is None:
1981             self.cookiejar = compat_cookiejar.CookieJar()
1982         else:
1983             self.cookiejar = compat_cookiejar.MozillaCookieJar(
1984                 opts_cookiefile)
1985             if os.access(opts_cookiefile, os.R_OK):
1986                 self.cookiejar.load()
1987
1988         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
1989         if opts_proxy is not None:
1990             if opts_proxy == '':
1991                 proxies = {}
1992             else:
1993                 proxies = {'http': opts_proxy, 'https': opts_proxy}
1994         else:
1995             proxies = compat_urllib_request.getproxies()
1996             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1997             if 'http' in proxies and 'https' not in proxies:
1998                 proxies['https'] = proxies['http']
1999         proxy_handler = PerRequestProxyHandler(proxies)
2000
2001         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2002         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2003         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2004         data_handler = compat_urllib_request_DataHandler()
2005
2006         # When passing our own FileHandler instance, build_opener won't add the
2007         # default FileHandler and allows us to disable the file protocol, which
2008         # can be used for malicious purposes (see
2009         # https://github.com/rg3/youtube-dl/issues/8227)
2010         file_handler = compat_urllib_request.FileHandler()
2011
2012         def file_open(*args, **kwargs):
2013             raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2014         file_handler.file_open = file_open
2015
2016         opener = compat_urllib_request.build_opener(
2017             proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2018
2019         # Delete the default user-agent header, which would otherwise apply in
2020         # cases where our custom HTTP handler doesn't come into play
2021         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2022         opener.addheaders = []
2023         self._opener = opener
2024
2025     def encode(self, s):
2026         if isinstance(s, bytes):
2027             return s  # Already encoded
2028
2029         try:
2030             return s.encode(self.get_encoding())
2031         except UnicodeEncodeError as err:
2032             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2033             raise
2034
2035     def get_encoding(self):
2036         encoding = self.params.get('encoding')
2037         if encoding is None:
2038             encoding = preferredencoding()
2039         return encoding
2040
2041     def _write_thumbnails(self, info_dict, filename):
2042         if self.params.get('writethumbnail', False):
2043             thumbnails = info_dict.get('thumbnails')
2044             if thumbnails:
2045                 thumbnails = [thumbnails[-1]]
2046         elif self.params.get('write_all_thumbnails', False):
2047             thumbnails = info_dict.get('thumbnails')
2048         else:
2049             return
2050
2051         if not thumbnails:
2052             # No thumbnails present, so return immediately
2053             return
2054
2055         for t in thumbnails:
2056             thumb_ext = determine_ext(t['url'], 'jpg')
2057             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2058             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2059             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2060
2061             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2062                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2063                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2064             else:
2065                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2066                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2067                 try:
2068                     uf = self.urlopen(t['url'])
2069                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2070                         shutil.copyfileobj(uf, thumbf)
2071                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2072                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2073                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2074                     self.report_warning('Unable to download thumbnail "%s": %s' %
2075                                         (t['url'], error_to_compat_str(err)))