_ Git - youtube-dl/blob - youtube_dl/YoutubeDL.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from __future__ import absolute_import, unicode_literals
   5
   6 import collections
   7 import contextlib
   8 import datetime
   9 import errno
  10 import fileinput
  11 import io
  12 import itertools
  13 import json
  14 import locale
  15 import operator
  16 import os
  17 import platform
  18 import re
  19 import shutil
  20 import subprocess
  21 import socket
  22 import sys
  23 import time
  24 import tokenize
  25 import traceback
  26
  27 if os.name == 'nt':
  28     import ctypes
  29
  30 from .compat import (
  31     compat_basestring,
  32     compat_cookiejar,
  33     compat_expanduser,
  34     compat_get_terminal_size,
  35     compat_http_client,
  36     compat_kwargs,
  37     compat_str,
  38     compat_tokenize_tokenize,
  39     compat_urllib_error,
  40     compat_urllib_request,
  41     compat_urllib_request_DataHandler,
  42 )
  43 from .utils import (
  44     ContentTooShortError,
  45     date_from_str,
  46     DateRange,
  47     DEFAULT_OUTTMPL,
  48     determine_ext,
  49     DownloadError,
  50     encode_compat_str,
  51     encodeFilename,
  52     error_to_compat_str,
  53     ExtractorError,
  54     format_bytes,
  55     formatSeconds,
  56     locked_file,
  57     make_HTTPS_handler,
  58     MaxDownloadsReached,
  59     PagedList,
  60     parse_filesize,
  61     PerRequestProxyHandler,
  62     PostProcessingError,
  63     platform_name,
  64     preferredencoding,
  65     render_table,
  66     SameFileError,
  67     sanitize_filename,
  68     sanitize_path,
  69     sanitized_Request,
  70     std_headers,
  71     subtitles_filename,
  72     UnavailableVideoError,
  73     url_basename,
  74     version_tuple,
  75     write_json_file,
  76     write_string,
  77     YoutubeDLCookieProcessor,
  78     YoutubeDLHandler,
  79     prepend_extension,
  80     replace_extension,
  81     args_to_str,
  82     age_restricted,
  83 )
  84 from .cache import Cache
  85 from .extractor import get_info_extractor, gen_extractors
  86 from .downloader import get_suitable_downloader
  87 from .downloader.rtmp import rtmpdump_version
  88 from .postprocessor import (
  89     FFmpegFixupM4aPP,
  90     FFmpegFixupStretchedPP,
  91     FFmpegMergerPP,
  92     FFmpegPostProcessor,
  93     get_postprocessor,
  94 )
  95 from .version import __version__
  96
  97
  98 class YoutubeDL(object):
  99     """YoutubeDL class.
 100
 101     YoutubeDL objects are the ones responsible of downloading the
 102     actual video file and writing it to disk if the user has requested
 103     it, among some other tasks. In most cases there should be one per
 104     program. As, given a video URL, the downloader doesn't know how to
 105     extract all the needed information, task that InfoExtractors do, it
 106     has to pass the URL to one of them.
 107
 108     For this, YoutubeDL objects have a method that allows
 109     InfoExtractors to be registered in a given order. When it is passed
 110     a URL, the YoutubeDL object handles it to the first InfoExtractor it
 111     finds that reports being able to handle it. The InfoExtractor extracts
 112     all the information about the video or videos the URL refers to, and
 113     YoutubeDL process the extracted information, possibly using a File
 114     Downloader to download the video.
 115
 116     YoutubeDL objects accept a lot of parameters. In order not to saturate
 117     the object constructor with arguments, it receives a dictionary of
 118     options instead. These options are available through the params
 119     attribute for the InfoExtractors to use. The YoutubeDL also
 120     registers itself as the downloader in charge for the InfoExtractors
 121     that are added to it, so this is a "mutual registration".
 122
 123     Available options:
 124
 125     username:          Username for authentication purposes.
 126     password:          Password for authentication purposes.
 127     videopassword:     Password for accessing a video.
 128     usenetrc:          Use netrc for authentication instead.
 129     verbose:           Print additional info to stdout.
 130     quiet:             Do not print messages to stdout.
 131     no_warnings:       Do not print out anything for warnings.
 132     forceurl:          Force printing final URL.
 133     forcetitle:        Force printing title.
 134     forceid:           Force printing ID.
 135     forcethumbnail:    Force printing thumbnail URL.
 136     forcedescription:  Force printing description.
 137     forcefilename:     Force printing final filename.
 138     forceduration:     Force printing duration.
 139     forcejson:         Force printing info_dict as JSON.
 140     dump_single_json:  Force printing the info_dict of the whole playlist
 141                        (or video) as a single JSON line.
 142     simulate:          Do not download the video files.
 143     format:            Video format code. See options.py for more information.
 144     outtmpl:           Template for output names.
 145     restrictfilenames: Do not allow "&" and spaces in file names
 146     ignoreerrors:      Do not stop on download errors.
 147     force_generic_extractor: Force downloader to use the generic extractor
 148     nooverwrites:      Prevent overwriting files.
 149     playliststart:     Playlist item to start at.
 150     playlistend:       Playlist item to end at.
 151     playlist_items:    Specific indices of playlist to download.
 152     playlistreverse:   Download playlist items in reverse order.
 153     matchtitle:        Download only matching titles.
 154     rejecttitle:       Reject downloads for matching titles.
 155     logger:            Log messages to a logging.Logger instance.
 156     logtostderr:       Log messages to stderr instead of stdout.
 157     writedescription:  Write the video description to a .description file
 158     writeinfojson:     Write the video description to a .info.json file
 159     writeannotations:  Write the video annotations to a .annotations.xml file
 160     writethumbnail:    Write the thumbnail image to a file
 161     write_all_thumbnails:  Write all thumbnail formats to files
 162     writesubtitles:    Write the video subtitles to a file
 163     writeautomaticsub: Write the automatically generated subtitles to a file
 164     allsubtitles:      Downloads all the subtitles of the video
 165                        (requires writesubtitles or writeautomaticsub)
 166     listsubtitles:     Lists all available subtitles for the video
 167     subtitlesformat:   The format code for subtitles
 168     subtitleslangs:    List of languages of the subtitles to download
 169     keepvideo:         Keep the video file after post-processing
 170     daterange:         A DateRange object, download only if the upload_date is in the range.
 171     skip_download:     Skip the actual download of the video file
 172     cachedir:          Location of the cache files in the filesystem.
 173                        False to disable filesystem cache.
 174     noplaylist:        Download single video instead of a playlist if in doubt.
 175     age_limit:         An integer representing the user's age in years.
 176                        Unsuitable videos for the given age are skipped.
 177     min_views:         An integer representing the minimum view count the video
 178                        must have in order to not be skipped.
 179                        Videos without view count information are always
 180                        downloaded. None for no limit.
 181     max_views:         An integer representing the maximum view count.
 182                        Videos that are more popular than that are not
 183                        downloaded.
 184                        Videos without view count information are always
 185                        downloaded. None for no limit.
 186     download_archive:  File name of a file where all downloads are recorded.
 187                        Videos already present in the file are not downloaded
 188                        again.
 189     cookiefile:        File name where cookies should be read from and dumped to.
 190     nocheckcertificate:Do not verify SSL certificates
 191     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
 192                        At the moment, this is only supported by YouTube.
 193     proxy:             URL of the proxy server to use
 194     cn_verification_proxy:  URL of the proxy to use for IP address verification
 195                        on Chinese sites. (Experimental)
 196     socket_timeout:    Time to wait for unresponsive hosts, in seconds
 197     bidi_workaround:   Work around buggy terminals without bidirectional text
 198                        support, using fridibi
 199     debug_printtraffic:Print out sent and received HTTP traffic
 200     include_ads:       Download ads as well
 201     default_search:    Prepend this string if an input url is not valid.
 202                        'auto' for elaborate guessing
 203     encoding:          Use this encoding instead of the system-specified.
 204     extract_flat:      Do not resolve URLs, return the immediate result.
 205                        Pass in 'in_playlist' to only show this behavior for
 206                        playlist items.
 207     postprocessors:    A list of dictionaries, each with an entry
 208                        * key:  The name of the postprocessor. See
 209                                youtube_dl/postprocessor/__init__.py for a list.
 210                        as well as any further keyword arguments for the
 211                        postprocessor.
 212     progress_hooks:    A list of functions that get called on download
 213                        progress, with a dictionary with the entries
 214                        * status: One of "downloading", "error", or "finished".
 215                                  Check this first and ignore unknown values.
 216
 217                        If status is one of "downloading", or "finished", the
 218                        following properties may also be present:
 219                        * filename: The final filename (always present)
 220                        * tmpfilename: The filename we're currently writing to
 221                        * downloaded_bytes: Bytes on disk
 222                        * total_bytes: Size of the whole file, None if unknown
 223                        * total_bytes_estimate: Guess of the eventual file size,
 224                                                None if unavailable.
 225                        * elapsed: The number of seconds since download started.
 226                        * eta: The estimated time in seconds, None if unknown
 227                        * speed: The download speed in bytes/second, None if
 228                                 unknown
 229                        * fragment_index: The counter of the currently
 230                                          downloaded video fragment.
 231                        * fragment_count: The number of fragments (= individual
 232                                          files that will be merged)
 233
 234                        Progress hooks are guaranteed to be called at least once
 235                        (with status "finished") if the download is successful.
 236     merge_output_format: Extension to use when merging formats.
 237     fixup:             Automatically correct known faults of the file.
 238                        One of:
 239                        - "never": do nothing
 240                        - "warn": only emit a warning
 241                        - "detect_or_warn": check whether we can do anything
 242                                            about it, warn otherwise (default)
 243     source_address:    (Experimental) Client-side IP address to bind to.
 244     call_home:         Boolean, true iff we are allowed to contact the
 245                        youtube-dl servers for debugging.
 246     sleep_interval:    Number of seconds to sleep before each download.
 247     listformats:       Print an overview of available video formats and exit.
 248     list_thumbnails:   Print a table of all thumbnails and exit.
 249     match_filter:      A function that gets called with the info_dict of
 250                        every video.
 251                        If it returns a message, the video is ignored.
 252                        If it returns None, the video is downloaded.
 253                        match_filter_func in utils.py is one example for this.
 254     no_color:          Do not emit color codes in output.
 255
 256     The following options determine which downloader is picked:
 257     external_downloader: Executable of the external downloader to call.
 258                        None or unset for standard (built-in) downloader.
 259     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
 260
 261     The following parameters are not used by YoutubeDL itself, they are used by
 262     the downloader (see youtube_dl/downloader/common.py):
 263     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
 264     noresizebuffer, retries, continuedl, noprogress, consoletitle,
 265     xattr_set_filesize, external_downloader_args.
 266
 267     The following options are used by the post processors:
 268     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
 269                        otherwise prefer avconv.
 270     postprocessor_args: A list of additional command-line arguments for the
 271                         postprocessor.
 272     """
 273
 274     params = None
 275     _ies = []
 276     _pps = []
 277     _download_retcode = None
 278     _num_downloads = None
 279     _screen_file = None
 280
 281     def __init__(self, params=None, auto_init=True):
 282         """Create a FileDownloader object with the given options."""
 283         if params is None:
 284             params = {}
 285         self._ies = []
 286         self._ies_instances = {}
 287         self._pps = []
 288         self._progress_hooks = []
 289         self._download_retcode = 0
 290         self._num_downloads = 0
 291         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
 292         self._err_file = sys.stderr
 293         self.params = {
 294             # Default parameters
 295             'nocheckcertificate': False,
 296         }
 297         self.params.update(params)
 298         self.cache = Cache(self)
 299
 300         if params.get('bidi_workaround', False):
 301             try:
 302                 import pty
 303                 master, slave = pty.openpty()
 304                 width = compat_get_terminal_size().columns
 305                 if width is None:
 306                     width_args = []
 307                 else:
 308                     width_args = ['-w', str(width)]
 309                 sp_kwargs = dict(
 310                     stdin=subprocess.PIPE,
 311                     stdout=slave,
 312                     stderr=self._err_file)
 313                 try:
 314                     self._output_process = subprocess.Popen(
 315                         ['bidiv'] + width_args, **sp_kwargs
 316                     )
 317                 except OSError:
 318                     self._output_process = subprocess.Popen(
 319                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
 320                 self._output_channel = os.fdopen(master, 'rb')
 321             except OSError as ose:
 322                 if ose.errno == 2:
 323                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
 324                 else:
 325                     raise
 326
 327         if (sys.version_info >= (3,) and sys.platform != 'win32' and
 328                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
 329                 not params.get('restrictfilenames', False)):
 330             # On Python 3, the Unicode filesystem API will throw errors (#1474)
 331             self.report_warning(
 332                 'Assuming --restrict-filenames since file system encoding '
 333                 'cannot encode all characters. '
 334                 'Set the LC_ALL environment variable to fix this.')
 335             self.params['restrictfilenames'] = True
 336
 337         if isinstance(params.get('outtmpl'), bytes):
 338             self.report_warning(
 339                 'Parameter outtmpl is bytes, but should be a unicode string. '
 340                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 341
 342         self._setup_opener()
 343
 344         if auto_init:
 345             self.print_debug_header()
 346             self.add_default_info_extractors()
 347
 348         for pp_def_raw in self.params.get('postprocessors', []):
 349             pp_class = get_postprocessor(pp_def_raw['key'])
 350             pp_def = dict(pp_def_raw)
 351             del pp_def['key']
 352             pp = pp_class(self, **compat_kwargs(pp_def))
 353             self.add_post_processor(pp)
 354
 355         for ph in self.params.get('progress_hooks', []):
 356             self.add_progress_hook(ph)
 357
 358     def warn_if_short_id(self, argv):
 359         # short YouTube ID starting with dash?
 360         idxs = [
 361             i for i, a in enumerate(argv)
 362             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
 363         if idxs:
 364             correct_argv = (
 365                 ['youtube-dl'] +
 366                 [a for i, a in enumerate(argv) if i not in idxs] +
 367                 ['--'] + [argv[i] for i in idxs]
 368             )
 369             self.report_warning(
 370                 'Long argument string detected. '
 371                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
 372                 args_to_str(correct_argv))
 373
 374     def add_info_extractor(self, ie):
 375         """Add an InfoExtractor object to the end of the list."""
 376         self._ies.append(ie)
 377         self._ies_instances[ie.ie_key()] = ie
 378         ie.set_downloader(self)
 379
 380     def get_info_extractor(self, ie_key):
 381         """
 382         Get an instance of an IE with name ie_key, it will try to get one from
 383         the _ies list, if there's no instance it will create a new one and add
 384         it to the extractor list.
 385         """
 386         ie = self._ies_instances.get(ie_key)
 387         if ie is None:
 388             ie = get_info_extractor(ie_key)()
 389             self.add_info_extractor(ie)
 390         return ie
 391
 392     def add_default_info_extractors(self):
 393         """
 394         Add the InfoExtractors returned by gen_extractors to the end of the list
 395         """
 396         for ie in gen_extractors():
 397             self.add_info_extractor(ie)
 398
 399     def add_post_processor(self, pp):
 400         """Add a PostProcessor object to the end of the chain."""
 401         self._pps.append(pp)
 402         pp.set_downloader(self)
 403
 404     def add_progress_hook(self, ph):
 405         """Add the progress hook (currently only for the file downloader)"""
 406         self._progress_hooks.append(ph)
 407
 408     def _bidi_workaround(self, message):
 409         if not hasattr(self, '_output_channel'):
 410             return message
 411
 412         assert hasattr(self, '_output_process')
 413         assert isinstance(message, compat_str)
 414         line_count = message.count('\n') + 1
 415         self._output_process.stdin.write((message + '\n').encode('utf-8'))
 416         self._output_process.stdin.flush()
 417         res = ''.join(self._output_channel.readline().decode('utf-8')
 418                       for _ in range(line_count))
 419         return res[:-len('\n')]
 420
 421     def to_screen(self, message, skip_eol=False):
 422         """Print message to stdout if not in quiet mode."""
 423         return self.to_stdout(message, skip_eol, check_quiet=True)
 424
 425     def _write_string(self, s, out=None):
 426         write_string(s, out=out, encoding=self.params.get('encoding'))
 427
 428     def to_stdout(self, message, skip_eol=False, check_quiet=False):
 429         """Print message to stdout if not in quiet mode."""
 430         if self.params.get('logger'):
 431             self.params['logger'].debug(message)
 432         elif not check_quiet or not self.params.get('quiet', False):
 433             message = self._bidi_workaround(message)
 434             terminator = ['\n', ''][skip_eol]
 435             output = message + terminator
 436
 437             self._write_string(output, self._screen_file)
 438
 439     def to_stderr(self, message):
 440         """Print message to stderr."""
 441         assert isinstance(message, compat_str)
 442         if self.params.get('logger'):
 443             self.params['logger'].error(message)
 444         else:
 445             message = self._bidi_workaround(message)
 446             output = message + '\n'
 447             self._write_string(output, self._err_file)
 448
 449     def to_console_title(self, message):
 450         if not self.params.get('consoletitle', False):
 451             return
 452         if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
 453             # c_wchar_p() might not be necessary if `message` is
 454             # already of type unicode()
 455             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
 456         elif 'TERM' in os.environ:
 457             self._write_string('\033]0;%s\007' % message, self._screen_file)
 458
 459     def save_console_title(self):
 460         if not self.params.get('consoletitle', False):
 461             return
 462         if 'TERM' in os.environ:
 463             # Save the title on stack
 464             self._write_string('\033[22;0t', self._screen_file)
 465
 466     def restore_console_title(self):
 467         if not self.params.get('consoletitle', False):
 468             return
 469         if 'TERM' in os.environ:
 470             # Restore the title from stack
 471             self._write_string('\033[23;0t', self._screen_file)
 472
 473     def __enter__(self):
 474         self.save_console_title()
 475         return self
 476
 477     def __exit__(self, *args):
 478         self.restore_console_title()
 479
 480         if self.params.get('cookiefile') is not None:
 481             self.cookiejar.save()
 482
 483     def trouble(self, message=None, tb=None):
 484         """Determine action to take when a download problem appears.
 485
 486         Depending on if the downloader has been configured to ignore
 487         download errors or not, this method may throw an exception or
 488         not when errors are found, after printing the message.
 489
 490         tb, if given, is additional traceback information.
 491         """
 492         if message is not None:
 493             self.to_stderr(message)
 494         if self.params.get('verbose'):
 495             if tb is None:
 496                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
 497                     tb = ''
 498                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 499                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
 500                     tb += encode_compat_str(traceback.format_exc())
 501                 else:
 502                     tb_data = traceback.format_list(traceback.extract_stack())
 503                     tb = ''.join(tb_data)
 504             self.to_stderr(tb)
 505         if not self.params.get('ignoreerrors', False):
 506             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
 507                 exc_info = sys.exc_info()[1].exc_info
 508             else:
 509                 exc_info = sys.exc_info()
 510             raise DownloadError(message, exc_info)
 511         self._download_retcode = 1
 512
 513     def report_warning(self, message):
 514         '''
 515         Print the message to stderr, it will be prefixed with 'WARNING:'
 516         If stderr is a tty file the 'WARNING:' will be colored
 517         '''
 518         if self.params.get('logger') is not None:
 519             self.params['logger'].warning(message)
 520         else:
 521             if self.params.get('no_warnings'):
 522                 return
 523             if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
 524                 _msg_header = '\033[0;33mWARNING:\033[0m'
 525             else:
 526                 _msg_header = 'WARNING:'
 527             warning_message = '%s %s' % (_msg_header, message)
 528             self.to_stderr(warning_message)
 529
 530     def report_error(self, message, tb=None):
 531         '''
 532         Do the same as trouble, but prefixes the message with 'ERROR:', colored
 533         in red if stderr is a tty file.
 534         '''
 535         if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
 536             _msg_header = '\033[0;31mERROR:\033[0m'
 537         else:
 538             _msg_header = 'ERROR:'
 539         error_message = '%s %s' % (_msg_header, message)
 540         self.trouble(error_message, tb)
 541
 542     def report_file_already_downloaded(self, file_name):
 543         """Report file has already been fully downloaded."""
 544         try:
 545             self.to_screen('[download] %s has already been downloaded' % file_name)
 546         except UnicodeEncodeError:
 547             self.to_screen('[download] The file has already been downloaded')
 548
 549     def prepare_filename(self, info_dict):
 550         """Generate the output filename."""
 551         try:
 552             template_dict = dict(info_dict)
 553
 554             template_dict['epoch'] = int(time.time())
 555             autonumber_size = self.params.get('autonumber_size')
 556             if autonumber_size is None:
 557                 autonumber_size = 5
 558             autonumber_templ = '%0' + str(autonumber_size) + 'd'
 559             template_dict['autonumber'] = autonumber_templ % self._num_downloads
 560             if template_dict.get('playlist_index') is not None:
 561                 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
 562             if template_dict.get('resolution') is None:
 563                 if template_dict.get('width') and template_dict.get('height'):
 564                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
 565                 elif template_dict.get('height'):
 566                     template_dict['resolution'] = '%sp' % template_dict['height']
 567                 elif template_dict.get('width'):
 568                     template_dict['resolution'] = '?x%d' % template_dict['width']
 569
 570             sanitize = lambda k, v: sanitize_filename(
 571                 compat_str(v),
 572                 restricted=self.params.get('restrictfilenames'),
 573                 is_id=(k == 'id'))
 574             template_dict = dict((k, sanitize(k, v))
 575                                  for k, v in template_dict.items()
 576                                  if v is not None)
 577             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 578
 579             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 580             tmpl = compat_expanduser(outtmpl)
 581             filename = tmpl % template_dict
 582             # Temporary fix for #4787
 583             # 'Treat' all problem characters by passing filename through preferredencoding
 584             # to workaround encoding issues with subprocess on python2 @ Windows
 585             if sys.version_info < (3, 0) and sys.platform == 'win32':
 586                 filename = encodeFilename(filename, True).decode(preferredencoding())
 587             return sanitize_path(filename)
 588         except ValueError as err:
 589             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
 590             return None
 591
 592     def _match_entry(self, info_dict, incomplete):
 593         """ Returns None iff the file should be downloaded """
 594
 595         video_title = info_dict.get('title', info_dict.get('id', 'video'))
 596         if 'title' in info_dict:
 597             # This can happen when we're just evaluating the playlist
 598             title = info_dict['title']
 599             matchtitle = self.params.get('matchtitle', False)
 600             if matchtitle:
 601                 if not re.search(matchtitle, title, re.IGNORECASE):
 602                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
 603             rejecttitle = self.params.get('rejecttitle', False)
 604             if rejecttitle:
 605                 if re.search(rejecttitle, title, re.IGNORECASE):
 606                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
 607         date = info_dict.get('upload_date', None)
 608         if date is not None:
 609             dateRange = self.params.get('daterange', DateRange())
 610             if date not in dateRange:
 611                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
 612         view_count = info_dict.get('view_count', None)
 613         if view_count is not None:
 614             min_views = self.params.get('min_views')
 615             if min_views is not None and view_count < min_views:
 616                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
 617             max_views = self.params.get('max_views')
 618             if max_views is not None and view_count > max_views:
 619                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
 620         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
 621             return 'Skipping "%s" because it is age restricted' % video_title
 622         if self.in_download_archive(info_dict):
 623             return '%s has already been recorded in archive' % video_title
 624
 625         if not incomplete:
 626             match_filter = self.params.get('match_filter')
 627             if match_filter is not None:
 628                 ret = match_filter(info_dict)
 629                 if ret is not None:
 630                     return ret
 631
 632         return None
 633
 634     @staticmethod
 635     def add_extra_info(info_dict, extra_info):
 636         '''Set the keys from extra_info in info dict if they are missing'''
 637         for key, value in extra_info.items():
 638             info_dict.setdefault(key, value)
 639
 640     def extract_info(self, url, download=True, ie_key=None, extra_info={},
 641                      process=True, force_generic_extractor=False):
 642         '''
 643         Returns a list with a dictionary for each video we find.
 644         If 'download', also downloads the videos.
 645         extra_info is a dict containing the extra values to add to each result
 646         '''
 647
 648         if not ie_key and force_generic_extractor:
 649             ie_key = 'Generic'
 650
 651         if ie_key:
 652             ies = [self.get_info_extractor(ie_key)]
 653         else:
 654             ies = self._ies
 655
 656         for ie in ies:
 657             if not ie.suitable(url):
 658                 continue
 659
 660             if not ie.working():
 661                 self.report_warning('The program functionality for this site has been marked as broken, '
 662                                     'and will probably not work.')
 663
 664             try:
 665                 ie_result = ie.extract(url)
 666                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
 667                     break
 668                 if isinstance(ie_result, list):
 669                     # Backwards compatibility: old IE result format
 670                     ie_result = {
 671                         '_type': 'compat_list',
 672                         'entries': ie_result,
 673                     }
 674                 self.add_default_extra_info(ie_result, ie, url)
 675                 if process:
 676                     return self.process_ie_result(ie_result, download, extra_info)
 677                 else:
 678                     return ie_result
 679             except ExtractorError as e:  # An error we somewhat expected
 680                 self.report_error(compat_str(e), e.format_traceback())
 681                 break
 682             except MaxDownloadsReached:
 683                 raise
 684             except Exception as e:
 685                 if self.params.get('ignoreerrors', False):
 686                     self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
 687                     break
 688                 else:
 689                     raise
 690         else:
 691             self.report_error('no suitable InfoExtractor for URL %s' % url)
 692
 693     def add_default_extra_info(self, ie_result, ie, url):
 694         self.add_extra_info(ie_result, {
 695             'extractor': ie.IE_NAME,
 696             'webpage_url': url,
 697             'webpage_url_basename': url_basename(url),
 698             'extractor_key': ie.ie_key(),
 699         })
 700
 701     def process_ie_result(self, ie_result, download=True, extra_info={}):
 702         """
 703         Take the result of the ie(may be modified) and resolve all unresolved
 704         references (URLs, playlist items).
 705
 706         It will also download the videos if 'download'.
 707         Returns the resolved ie_result.
 708         """
 709
 710         result_type = ie_result.get('_type', 'video')
 711
 712         if result_type in ('url', 'url_transparent'):
 713             extract_flat = self.params.get('extract_flat', False)
 714             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
 715                     extract_flat is True):
 716                 if self.params.get('forcejson', False):
 717                     self.to_stdout(json.dumps(ie_result))
 718                 return ie_result
 719
 720         if result_type == 'video':
 721             self.add_extra_info(ie_result, extra_info)
 722             return self.process_video_result(ie_result, download=download)
 723         elif result_type == 'url':
 724             # We have to add extra_info to the results because it may be
 725             # contained in a playlist
 726             return self.extract_info(ie_result['url'],
 727                                      download,
 728                                      ie_key=ie_result.get('ie_key'),
 729                                      extra_info=extra_info)
 730         elif result_type == 'url_transparent':
 731             # Use the information from the embedding page
 732             info = self.extract_info(
 733                 ie_result['url'], ie_key=ie_result.get('ie_key'),
 734                 extra_info=extra_info, download=False, process=False)
 735
 736             force_properties = dict(
 737                 (k, v) for k, v in ie_result.items() if v is not None)
 738             for f in ('_type', 'url'):
 739                 if f in force_properties:
 740                     del force_properties[f]
 741             new_result = info.copy()
 742             new_result.update(force_properties)
 743
 744             assert new_result.get('_type') != 'url_transparent'
 745
 746             return self.process_ie_result(
 747                 new_result, download=download, extra_info=extra_info)
 748         elif result_type == 'playlist' or result_type == 'multi_video':
 749             # We process each entry in the playlist
 750             playlist = ie_result.get('title', None) or ie_result.get('id', None)
 751             self.to_screen('[download] Downloading playlist: %s' % playlist)
 752
 753             playlist_results = []
 754
 755             playliststart = self.params.get('playliststart', 1) - 1
 756             playlistend = self.params.get('playlistend', None)
 757             # For backwards compatibility, interpret -1 as whole list
 758             if playlistend == -1:
 759                 playlistend = None
 760
 761             playlistitems_str = self.params.get('playlist_items', None)
 762             playlistitems = None
 763             if playlistitems_str is not None:
 764                 def iter_playlistitems(format):
 765                     for string_segment in format.split(','):
 766                         if '-' in string_segment:
 767                             start, end = string_segment.split('-')
 768                             for item in range(int(start), int(end) + 1):
 769                                 yield int(item)
 770                         else:
 771                             yield int(string_segment)
 772                 playlistitems = iter_playlistitems(playlistitems_str)
 773
 774             ie_entries = ie_result['entries']
 775             if isinstance(ie_entries, list):
 776                 n_all_entries = len(ie_entries)
 777                 if playlistitems:
 778                     entries = [
 779                         ie_entries[i - 1] for i in playlistitems
 780                         if -n_all_entries <= i - 1 < n_all_entries]
 781                 else:
 782                     entries = ie_entries[playliststart:playlistend]
 783                 n_entries = len(entries)
 784                 self.to_screen(
 785                     "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
 786                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
 787             elif isinstance(ie_entries, PagedList):
 788                 if playlistitems:
 789                     entries = []
 790                     for item in playlistitems:
 791                         entries.extend(ie_entries.getslice(
 792                             item - 1, item
 793                         ))
 794                 else:
 795                     entries = ie_entries.getslice(
 796                         playliststart, playlistend)
 797                 n_entries = len(entries)
 798                 self.to_screen(
 799                     "[%s] playlist %s: Downloading %d videos" %
 800                     (ie_result['extractor'], playlist, n_entries))
 801             else:  # iterable
 802                 if playlistitems:
 803                     entry_list = list(ie_entries)
 804                     entries = [entry_list[i - 1] for i in playlistitems]
 805                 else:
 806                     entries = list(itertools.islice(
 807                         ie_entries, playliststart, playlistend))
 808                 n_entries = len(entries)
 809                 self.to_screen(
 810                     "[%s] playlist %s: Downloading %d videos" %
 811                     (ie_result['extractor'], playlist, n_entries))
 812
 813             if self.params.get('playlistreverse', False):
 814                 entries = entries[::-1]
 815
 816             for i, entry in enumerate(entries, 1):
 817                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
 818                 extra = {
 819                     'n_entries': n_entries,
 820                     'playlist': playlist,
 821                     'playlist_id': ie_result.get('id'),
 822                     'playlist_title': ie_result.get('title'),
 823                     'playlist_index': i + playliststart,
 824                     'extractor': ie_result['extractor'],
 825                     'webpage_url': ie_result['webpage_url'],
 826                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
 827                     'extractor_key': ie_result['extractor_key'],
 828                 }
 829
 830                 reason = self._match_entry(entry, incomplete=True)
 831                 if reason is not None:
 832                     self.to_screen('[download] ' + reason)
 833                     continue
 834
 835                 entry_result = self.process_ie_result(entry,
 836                                                       download=download,
 837                                                       extra_info=extra)
 838                 playlist_results.append(entry_result)
 839             ie_result['entries'] = playlist_results
 840             self.to_screen('[download] Finished downloading playlist: %s' % playlist)
 841             return ie_result
 842         elif result_type == 'compat_list':
 843             self.report_warning(
 844                 'Extractor %s returned a compat_list result. '
 845                 'It needs to be updated.' % ie_result.get('extractor'))
 846
 847             def _fixup(r):
 848                 self.add_extra_info(
 849                     r,
 850                     {
 851                         'extractor': ie_result['extractor'],
 852                         'webpage_url': ie_result['webpage_url'],
 853                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
 854                         'extractor_key': ie_result['extractor_key'],
 855                     }
 856                 )
 857                 return r
 858             ie_result['entries'] = [
 859                 self.process_ie_result(_fixup(r), download, extra_info)
 860                 for r in ie_result['entries']
 861             ]
 862             return ie_result
 863         else:
 864             raise Exception('Invalid result type: %s' % result_type)
 865
 866     def _build_format_filter(self, filter_spec):
 867         " Returns a function to filter the formats according to the filter_spec "
 868
 869         OPERATORS = {
 870             '<': operator.lt,
 871             '<=': operator.le,
 872             '>': operator.gt,
 873             '>=': operator.ge,
 874             '=': operator.eq,
 875             '!=': operator.ne,
 876         }
 877         operator_rex = re.compile(r'''(?x)\s*
 878             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
 879             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
 880             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
 881             $
 882             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
 883         m = operator_rex.search(filter_spec)
 884         if m:
 885             try:
 886                 comparison_value = int(m.group('value'))
 887             except ValueError:
 888                 comparison_value = parse_filesize(m.group('value'))
 889                 if comparison_value is None:
 890                     comparison_value = parse_filesize(m.group('value') + 'B')
 891                 if comparison_value is None:
 892                     raise ValueError(
 893                         'Invalid value %r in format specification %r' % (
 894                             m.group('value'), filter_spec))
 895             op = OPERATORS[m.group('op')]
 896
 897         if not m:
 898             STR_OPERATORS = {
 899                 '=': operator.eq,
 900                 '!=': operator.ne,
 901                 '^=': lambda attr, value: attr.startswith(value),
 902                 '$=': lambda attr, value: attr.endswith(value),
 903                 '*=': lambda attr, value: value in attr,
 904             }
 905             str_operator_rex = re.compile(r'''(?x)
 906                 \s*(?P<key>ext|acodec|vcodec|container|protocol)
 907                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
 908                 \s*(?P<value>[a-zA-Z0-9_-]+)
 909                 \s*$
 910                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
 911             m = str_operator_rex.search(filter_spec)
 912             if m:
 913                 comparison_value = m.group('value')
 914                 op = STR_OPERATORS[m.group('op')]
 915
 916         if not m:
 917             raise ValueError('Invalid filter specification %r' % filter_spec)
 918
 919         def _filter(f):
 920             actual_value = f.get(m.group('key'))
 921             if actual_value is None:
 922                 return m.group('none_inclusive')
 923             return op(actual_value, comparison_value)
 924         return _filter
 925
 926     def build_format_selector(self, format_spec):
 927         def syntax_error(note, start):
 928             message = (
 929                 'Invalid format specification: '
 930                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
 931             return SyntaxError(message)
 932
 933         PICKFIRST = 'PICKFIRST'
 934         MERGE = 'MERGE'
 935         SINGLE = 'SINGLE'
 936         GROUP = 'GROUP'
 937         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
 938
 939         def _parse_filter(tokens):
 940             filter_parts = []
 941             for type, string, start, _, _ in tokens:
 942                 if type == tokenize.OP and string == ']':
 943                     return ''.join(filter_parts)
 944                 else:
 945                     filter_parts.append(string)
 946
 947         def _remove_unused_ops(tokens):
 948             # Remove operators that we don't use and join them with the surrounding strings
 949             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
 950             ALLOWED_OPS = ('/', '+', ',', '(', ')')
 951             last_string, last_start, last_end, last_line = None, None, None, None
 952             for type, string, start, end, line in tokens:
 953                 if type == tokenize.OP and string == '[':
 954                     if last_string:
 955                         yield tokenize.NAME, last_string, last_start, last_end, last_line
 956                         last_string = None
 957                     yield type, string, start, end, line
 958                     # everything inside brackets will be handled by _parse_filter
 959                     for type, string, start, end, line in tokens:
 960                         yield type, string, start, end, line
 961                         if type == tokenize.OP and string == ']':
 962                             break
 963                 elif type == tokenize.OP and string in ALLOWED_OPS:
 964                     if last_string:
 965                         yield tokenize.NAME, last_string, last_start, last_end, last_line
 966                         last_string = None
 967                     yield type, string, start, end, line
 968                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
 969                     if not last_string:
 970                         last_string = string
 971                         last_start = start
 972                         last_end = end
 973                     else:
 974                         last_string += string
 975             if last_string:
 976                 yield tokenize.NAME, last_string, last_start, last_end, last_line
 977
 978         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
 979             selectors = []
 980             current_selector = None
 981             for type, string, start, _, _ in tokens:
 982                 # ENCODING is only defined in python 3.x
 983                 if type == getattr(tokenize, 'ENCODING', None):
 984                     continue
 985                 elif type in [tokenize.NAME, tokenize.NUMBER]:
 986                     current_selector = FormatSelector(SINGLE, string, [])
 987                 elif type == tokenize.OP:
 988                     if string == ')':
 989                         if not inside_group:
 990                             # ')' will be handled by the parentheses group
 991                             tokens.restore_last_token()
 992                         break
 993                     elif inside_merge and string in ['/', ',']:
 994                         tokens.restore_last_token()
 995                         break
 996                     elif inside_choice and string == ',':
 997                         tokens.restore_last_token()
 998                         break
 999                     elif string == ',':
1000                         if not current_selector:
1001                             raise syntax_error('"," must follow a format selector', start)
1002                         selectors.append(current_selector)
1003                         current_selector = None
1004                     elif string == '/':
1005                         if not current_selector:
1006                             raise syntax_error('"/" must follow a format selector', start)
1007                         first_choice = current_selector
1008                         second_choice = _parse_format_selection(tokens, inside_choice=True)
1009                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1010                     elif string == '[':
1011                         if not current_selector:
1012                             current_selector = FormatSelector(SINGLE, 'best', [])
1013                         format_filter = _parse_filter(tokens)
1014                         current_selector.filters.append(format_filter)
1015                     elif string == '(':
1016                         if current_selector:
1017                             raise syntax_error('Unexpected "("', start)
1018                         group = _parse_format_selection(tokens, inside_group=True)
1019                         current_selector = FormatSelector(GROUP, group, [])
1020                     elif string == '+':
1021                         video_selector = current_selector
1022                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
1023                         if not video_selector or not audio_selector:
1024                             raise syntax_error('"+" must be between two format selectors', start)
1025                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1026                     else:
1027                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1028                 elif type == tokenize.ENDMARKER:
1029                     break
1030             if current_selector:
1031                 selectors.append(current_selector)
1032             return selectors
1033
1034         def _build_selector_function(selector):
1035             if isinstance(selector, list):
1036                 fs = [_build_selector_function(s) for s in selector]
1037
1038                 def selector_function(formats):
1039                     for f in fs:
1040                         for format in f(formats):
1041                             yield format
1042                 return selector_function
1043             elif selector.type == GROUP:
1044                 selector_function = _build_selector_function(selector.selector)
1045             elif selector.type == PICKFIRST:
1046                 fs = [_build_selector_function(s) for s in selector.selector]
1047
1048                 def selector_function(formats):
1049                     for f in fs:
1050                         picked_formats = list(f(formats))
1051                         if picked_formats:
1052                             return picked_formats
1053                     return []
1054             elif selector.type == SINGLE:
1055                 format_spec = selector.selector
1056
1057                 def selector_function(formats):
1058                     formats = list(formats)
1059                     if not formats:
1060                         return
1061                     if format_spec == 'all':
1062                         for f in formats:
1063                             yield f
1064                     elif format_spec in ['best', 'worst', None]:
1065                         format_idx = 0 if format_spec == 'worst' else -1
1066                         audiovideo_formats = [
1067                             f for f in formats
1068                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1069                         if audiovideo_formats:
1070                             yield audiovideo_formats[format_idx]
1071                         # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
1072                         elif (all(f.get('acodec') != 'none' for f in formats) or
1073                               all(f.get('vcodec') != 'none' for f in formats)):
1074                             yield formats[format_idx]
1075                     elif format_spec == 'bestaudio':
1076                         audio_formats = [
1077                             f for f in formats
1078                             if f.get('vcodec') == 'none']
1079                         if audio_formats:
1080                             yield audio_formats[-1]
1081                     elif format_spec == 'worstaudio':
1082                         audio_formats = [
1083                             f for f in formats
1084                             if f.get('vcodec') == 'none']
1085                         if audio_formats:
1086                             yield audio_formats[0]
1087                     elif format_spec == 'bestvideo':
1088                         video_formats = [
1089                             f for f in formats
1090                             if f.get('acodec') == 'none']
1091                         if video_formats:
1092                             yield video_formats[-1]
1093                     elif format_spec == 'worstvideo':
1094                         video_formats = [
1095                             f for f in formats
1096                             if f.get('acodec') == 'none']
1097                         if video_formats:
1098                             yield video_formats[0]
1099                     else:
1100                         extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1101                         if format_spec in extensions:
1102                             filter_f = lambda f: f['ext'] == format_spec
1103                         else:
1104                             filter_f = lambda f: f['format_id'] == format_spec
1105                         matches = list(filter(filter_f, formats))
1106                         if matches:
1107                             yield matches[-1]
1108             elif selector.type == MERGE:
1109                 def _merge(formats_info):
1110                     format_1, format_2 = [f['format_id'] for f in formats_info]
1111                     # The first format must contain the video and the
1112                     # second the audio
1113                     if formats_info[0].get('vcodec') == 'none':
1114                         self.report_error('The first format must '
1115                                           'contain the video, try using '
1116                                           '"-f %s+%s"' % (format_2, format_1))
1117                         return
1118                     # Formats must be opposite (video+audio)
1119                     if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1120                         self.report_error(
1121                             'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1122                             % (format_1, format_2))
1123                         return
1124                     output_ext = (
1125                         formats_info[0]['ext']
1126                         if self.params.get('merge_output_format') is None
1127                         else self.params['merge_output_format'])
1128                     return {
1129                         'requested_formats': formats_info,
1130                         'format': '%s+%s' % (formats_info[0].get('format'),
1131                                              formats_info[1].get('format')),
1132                         'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1133                                                 formats_info[1].get('format_id')),
1134                         'width': formats_info[0].get('width'),
1135                         'height': formats_info[0].get('height'),
1136                         'resolution': formats_info[0].get('resolution'),
1137                         'fps': formats_info[0].get('fps'),
1138                         'vcodec': formats_info[0].get('vcodec'),
1139                         'vbr': formats_info[0].get('vbr'),
1140                         'stretched_ratio': formats_info[0].get('stretched_ratio'),
1141                         'acodec': formats_info[1].get('acodec'),
1142                         'abr': formats_info[1].get('abr'),
1143                         'ext': output_ext,
1144                     }
1145                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1146
1147                 def selector_function(formats):
1148                     formats = list(formats)
1149                     for pair in itertools.product(video_selector(formats), audio_selector(formats)):
1150                         yield _merge(pair)
1151
1152             filters = [self._build_format_filter(f) for f in selector.filters]
1153
1154             def final_selector(formats):
1155                 for _filter in filters:
1156                     formats = list(filter(_filter, formats))
1157                 return selector_function(formats)
1158             return final_selector
1159
1160         stream = io.BytesIO(format_spec.encode('utf-8'))
1161         try:
1162             tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1163         except tokenize.TokenError:
1164             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1165
1166         class TokenIterator(object):
1167             def __init__(self, tokens):
1168                 self.tokens = tokens
1169                 self.counter = 0
1170
1171             def __iter__(self):
1172                 return self
1173
1174             def __next__(self):
1175                 if self.counter >= len(self.tokens):
1176                     raise StopIteration()
1177                 value = self.tokens[self.counter]
1178                 self.counter += 1
1179                 return value
1180
1181             next = __next__
1182
1183             def restore_last_token(self):
1184                 self.counter -= 1
1185
1186         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1187         return _build_selector_function(parsed_selector)
1188
1189     def _calc_headers(self, info_dict):
1190         res = std_headers.copy()
1191
1192         add_headers = info_dict.get('http_headers')
1193         if add_headers:
1194             res.update(add_headers)
1195
1196         cookies = self._calc_cookies(info_dict)
1197         if cookies:
1198             res['Cookie'] = cookies
1199
1200         return res
1201
1202     def _calc_cookies(self, info_dict):
1203         pr = sanitized_Request(info_dict['url'])
1204         self.cookiejar.add_cookie_header(pr)
1205         return pr.get_header('Cookie')
1206
1207     def process_video_result(self, info_dict, download=True):
1208         assert info_dict.get('_type', 'video') == 'video'
1209
1210         if 'id' not in info_dict:
1211             raise ExtractorError('Missing "id" field in extractor result')
1212         if 'title' not in info_dict:
1213             raise ExtractorError('Missing "title" field in extractor result')
1214
1215         if 'playlist' not in info_dict:
1216             # It isn't part of a playlist
1217             info_dict['playlist'] = None
1218             info_dict['playlist_index'] = None
1219
1220         thumbnails = info_dict.get('thumbnails')
1221         if thumbnails is None:
1222             thumbnail = info_dict.get('thumbnail')
1223             if thumbnail:
1224                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1225         if thumbnails:
1226             thumbnails.sort(key=lambda t: (
1227                 t.get('preference'), t.get('width'), t.get('height'),
1228                 t.get('id'), t.get('url')))
1229             for i, t in enumerate(thumbnails):
1230                 if t.get('width') and t.get('height'):
1231                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1232                 if t.get('id') is None:
1233                     t['id'] = '%d' % i
1234
1235         if thumbnails and 'thumbnail' not in info_dict:
1236             info_dict['thumbnail'] = thumbnails[-1]['url']
1237
1238         if 'display_id' not in info_dict and 'id' in info_dict:
1239             info_dict['display_id'] = info_dict['id']
1240
1241         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1242             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1243             # see http://bugs.python.org/issue1646728)
1244             try:
1245                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1246                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1247             except (ValueError, OverflowError, OSError):
1248                 pass
1249
1250         # Auto generate title fields corresponding to the *_number fields when missing
1251         # in order to always have clean titles. This is very common for TV series.
1252         for field in ('chapter', 'season', 'episode'):
1253             if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1254                 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1255
1256         subtitles = info_dict.get('subtitles')
1257         if subtitles:
1258             for _, subtitle in subtitles.items():
1259                 for subtitle_format in subtitle:
1260                     if 'ext' not in subtitle_format:
1261                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1262
1263         if self.params.get('listsubtitles', False):
1264             if 'automatic_captions' in info_dict:
1265                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1266             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1267             return
1268         info_dict['requested_subtitles'] = self.process_subtitles(
1269             info_dict['id'], subtitles,
1270             info_dict.get('automatic_captions'))
1271
1272         # We now pick which formats have to be downloaded
1273         if info_dict.get('formats') is None:
1274             # There's only one format available
1275             formats = [info_dict]
1276         else:
1277             formats = info_dict['formats']
1278
1279         if not formats:
1280             raise ExtractorError('No video formats found!')
1281
1282         formats_dict = {}
1283
1284         # We check that all the formats have the format and format_id fields
1285         for i, format in enumerate(formats):
1286             if 'url' not in format:
1287                 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1288
1289             if format.get('format_id') is None:
1290                 format['format_id'] = compat_str(i)
1291             format_id = format['format_id']
1292             if format_id not in formats_dict:
1293                 formats_dict[format_id] = []
1294             formats_dict[format_id].append(format)
1295
1296         # Make sure all formats have unique format_id
1297         for format_id, ambiguous_formats in formats_dict.items():
1298             if len(ambiguous_formats) > 1:
1299                 for i, format in enumerate(ambiguous_formats):
1300                     format['format_id'] = '%s-%d' % (format_id, i)
1301
1302         for i, format in enumerate(formats):
1303             if format.get('format') is None:
1304                 format['format'] = '{id} - {res}{note}'.format(
1305                     id=format['format_id'],
1306                     res=self.format_resolution(format),
1307                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1308                 )
1309             # Automatically determine file extension if missing
1310             if 'ext' not in format:
1311                 format['ext'] = determine_ext(format['url']).lower()
1312             # Add HTTP headers, so that external programs can use them from the
1313             # json output
1314             full_format_info = info_dict.copy()
1315             full_format_info.update(format)
1316             format['http_headers'] = self._calc_headers(full_format_info)
1317
1318         # TODO Central sorting goes here
1319
1320         if formats[0] is not info_dict:
1321             # only set the 'formats' fields if the original info_dict list them
1322             # otherwise we end up with a circular reference, the first (and unique)
1323             # element in the 'formats' field in info_dict is info_dict itself,
1324             # which can't be exported to json
1325             info_dict['formats'] = formats
1326         if self.params.get('listformats'):
1327             self.list_formats(info_dict)
1328             return
1329         if self.params.get('list_thumbnails'):
1330             self.list_thumbnails(info_dict)
1331             return
1332
1333         req_format = self.params.get('format')
1334         if req_format is None:
1335             req_format_list = []
1336             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1337                     info_dict['extractor'] in ['youtube', 'ted'] and
1338                     not info_dict.get('is_live')):
1339                 merger = FFmpegMergerPP(self)
1340                 if merger.available and merger.can_merge():
1341                     req_format_list.append('bestvideo+bestaudio')
1342             req_format_list.append('best')
1343             req_format = '/'.join(req_format_list)
1344         format_selector = self.build_format_selector(req_format)
1345         formats_to_download = list(format_selector(formats))
1346         if not formats_to_download:
1347             raise ExtractorError('requested format not available',
1348                                  expected=True)
1349
1350         if download:
1351             if len(formats_to_download) > 1:
1352                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1353             for format in formats_to_download:
1354                 new_info = dict(info_dict)
1355                 new_info.update(format)
1356                 self.process_info(new_info)
1357         # We update the info dict with the best quality format (backwards compatibility)
1358         info_dict.update(formats_to_download[-1])
1359         return info_dict
1360
1361     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1362         """Select the requested subtitles and their format"""
1363         available_subs = {}
1364         if normal_subtitles and self.params.get('writesubtitles'):
1365             available_subs.update(normal_subtitles)
1366         if automatic_captions and self.params.get('writeautomaticsub'):
1367             for lang, cap_info in automatic_captions.items():
1368                 if lang not in available_subs:
1369                     available_subs[lang] = cap_info
1370
1371         if (not self.params.get('writesubtitles') and not
1372                 self.params.get('writeautomaticsub') or not
1373                 available_subs):
1374             return None
1375
1376         if self.params.get('allsubtitles', False):
1377             requested_langs = available_subs.keys()
1378         else:
1379             if self.params.get('subtitleslangs', False):
1380                 requested_langs = self.params.get('subtitleslangs')
1381             elif 'en' in available_subs:
1382                 requested_langs = ['en']
1383             else:
1384                 requested_langs = [list(available_subs.keys())[0]]
1385
1386         formats_query = self.params.get('subtitlesformat', 'best')
1387         formats_preference = formats_query.split('/') if formats_query else []
1388         subs = {}
1389         for lang in requested_langs:
1390             formats = available_subs.get(lang)
1391             if formats is None:
1392                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1393                 continue
1394             for ext in formats_preference:
1395                 if ext == 'best':
1396                     f = formats[-1]
1397                     break
1398                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1399                 if matches:
1400                     f = matches[-1]
1401                     break
1402             else:
1403                 f = formats[-1]
1404                 self.report_warning(
1405                     'No subtitle format found matching "%s" for language %s, '
1406                     'using %s' % (formats_query, lang, f['ext']))
1407             subs[lang] = f
1408         return subs
1409
1410     def process_info(self, info_dict):
1411         """Process a single resolved IE result."""
1412
1413         assert info_dict.get('_type', 'video') == 'video'
1414
1415         max_downloads = self.params.get('max_downloads')
1416         if max_downloads is not None:
1417             if self._num_downloads >= int(max_downloads):
1418                 raise MaxDownloadsReached()
1419
1420         info_dict['fulltitle'] = info_dict['title']
1421         if len(info_dict['title']) > 200:
1422             info_dict['title'] = info_dict['title'][:197] + '...'
1423
1424         if 'format' not in info_dict:
1425             info_dict['format'] = info_dict['ext']
1426
1427         reason = self._match_entry(info_dict, incomplete=False)
1428         if reason is not None:
1429             self.to_screen('[download] ' + reason)
1430             return
1431
1432         self._num_downloads += 1
1433
1434         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1435
1436         # Forced printings
1437         if self.params.get('forcetitle', False):
1438             self.to_stdout(info_dict['fulltitle'])
1439         if self.params.get('forceid', False):
1440             self.to_stdout(info_dict['id'])
1441         if self.params.get('forceurl', False):
1442             if info_dict.get('requested_formats') is not None:
1443                 for f in info_dict['requested_formats']:
1444                     self.to_stdout(f['url'] + f.get('play_path', ''))
1445             else:
1446                 # For RTMP URLs, also include the playpath
1447                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1448         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1449             self.to_stdout(info_dict['thumbnail'])
1450         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1451             self.to_stdout(info_dict['description'])
1452         if self.params.get('forcefilename', False) and filename is not None:
1453             self.to_stdout(filename)
1454         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1455             self.to_stdout(formatSeconds(info_dict['duration']))
1456         if self.params.get('forceformat', False):
1457             self.to_stdout(info_dict['format'])
1458         if self.params.get('forcejson', False):
1459             self.to_stdout(json.dumps(info_dict))
1460
1461         # Do nothing else if in simulate mode
1462         if self.params.get('simulate', False):
1463             return
1464
1465         if filename is None:
1466             return
1467
1468         try:
1469             dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1470             if dn and not os.path.exists(dn):
1471                 os.makedirs(dn)
1472         except (OSError, IOError) as err:
1473             self.report_error('unable to create directory ' + error_to_compat_str(err))
1474             return
1475
1476         if self.params.get('writedescription', False):
1477             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1478             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1479                 self.to_screen('[info] Video description is already present')
1480             elif info_dict.get('description') is None:
1481                 self.report_warning('There\'s no description to write.')
1482             else:
1483                 try:
1484                     self.to_screen('[info] Writing video description to: ' + descfn)
1485                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1486                         descfile.write(info_dict['description'])
1487                 except (OSError, IOError):
1488                     self.report_error('Cannot write description file ' + descfn)
1489                     return
1490
1491         if self.params.get('writeannotations', False):
1492             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1493             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1494                 self.to_screen('[info] Video annotations are already present')
1495             else:
1496                 try:
1497                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1498                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1499                         annofile.write(info_dict['annotations'])
1500                 except (KeyError, TypeError):
1501                     self.report_warning('There are no annotations to write.')
1502                 except (OSError, IOError):
1503                     self.report_error('Cannot write annotations file: ' + annofn)
1504                     return
1505
1506         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1507                                        self.params.get('writeautomaticsub')])
1508
1509         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1510             # subtitles download errors are already managed as troubles in relevant IE
1511             # that way it will silently go on when used with unsupporting IE
1512             subtitles = info_dict['requested_subtitles']
1513             ie = self.get_info_extractor(info_dict['extractor_key'])
1514             for sub_lang, sub_info in subtitles.items():
1515                 sub_format = sub_info['ext']
1516                 if sub_info.get('data') is not None:
1517                     sub_data = sub_info['data']
1518                 else:
1519                     try:
1520                         sub_data = ie._download_webpage(
1521                             sub_info['url'], info_dict['id'], note=False)
1522                     except ExtractorError as err:
1523                         self.report_warning('Unable to download subtitle for "%s": %s' %
1524                                             (sub_lang, error_to_compat_str(err.cause)))
1525                         continue
1526                 try:
1527                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1528                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1529                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1530                     else:
1531                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1532                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1533                             subfile.write(sub_data)
1534                 except (OSError, IOError):
1535                     self.report_error('Cannot write subtitles file ' + sub_filename)
1536                     return
1537
1538         if self.params.get('writeinfojson', False):
1539             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1540             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1541                 self.to_screen('[info] Video description metadata is already present')
1542             else:
1543                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1544                 try:
1545                     write_json_file(self.filter_requested_info(info_dict), infofn)
1546                 except (OSError, IOError):
1547                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1548                     return
1549
1550         self._write_thumbnails(info_dict, filename)
1551
1552         if not self.params.get('skip_download', False):
1553             try:
1554                 def dl(name, info):
1555                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1556                     for ph in self._progress_hooks:
1557                         fd.add_progress_hook(ph)
1558                     if self.params.get('verbose'):
1559                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1560                     return fd.download(name, info)
1561
1562                 if info_dict.get('requested_formats') is not None:
1563                     downloaded = []
1564                     success = True
1565                     merger = FFmpegMergerPP(self)
1566                     if not merger.available:
1567                         postprocessors = []
1568                         self.report_warning('You have requested multiple '
1569                                             'formats but ffmpeg or avconv are not installed.'
1570                                             ' The formats won\'t be merged.')
1571                     else:
1572                         postprocessors = [merger]
1573
1574                     def compatible_formats(formats):
1575                         video, audio = formats
1576                         # Check extension
1577                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1578                         if video_ext and audio_ext:
1579                             COMPATIBLE_EXTS = (
1580                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1581                                 ('webm')
1582                             )
1583                             for exts in COMPATIBLE_EXTS:
1584                                 if video_ext in exts and audio_ext in exts:
1585                                     return True
1586                         # TODO: Check acodec/vcodec
1587                         return False
1588
1589                     filename_real_ext = os.path.splitext(filename)[1][1:]
1590                     filename_wo_ext = (
1591                         os.path.splitext(filename)[0]
1592                         if filename_real_ext == info_dict['ext']
1593                         else filename)
1594                     requested_formats = info_dict['requested_formats']
1595                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1596                         info_dict['ext'] = 'mkv'
1597                         self.report_warning(
1598                             'Requested formats are incompatible for merge and will be merged into mkv.')
1599                     # Ensure filename always has a correct extension for successful merge
1600                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1601                     if os.path.exists(encodeFilename(filename)):
1602                         self.to_screen(
1603                             '[download] %s has already been downloaded and '
1604                             'merged' % filename)
1605                     else:
1606                         for f in requested_formats:
1607                             new_info = dict(info_dict)
1608                             new_info.update(f)
1609                             fname = self.prepare_filename(new_info)
1610                             fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1611                             downloaded.append(fname)
1612                             partial_success = dl(fname, new_info)
1613                             success = success and partial_success
1614                         info_dict['__postprocessors'] = postprocessors
1615                         info_dict['__files_to_merge'] = downloaded
1616                 else:
1617                     # Just a single file
1618                     success = dl(filename, info_dict)
1619             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1620                 self.report_error('unable to download video data: %s' % str(err))
1621                 return
1622             except (OSError, IOError) as err:
1623                 raise UnavailableVideoError(err)
1624             except (ContentTooShortError, ) as err:
1625                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1626                 return
1627
1628             if success:
1629                 # Fixup content
1630                 fixup_policy = self.params.get('fixup')
1631                 if fixup_policy is None:
1632                     fixup_policy = 'detect_or_warn'
1633
1634                 stretched_ratio = info_dict.get('stretched_ratio')
1635                 if stretched_ratio is not None and stretched_ratio != 1:
1636                     if fixup_policy == 'warn':
1637                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1638                             info_dict['id'], stretched_ratio))
1639                     elif fixup_policy == 'detect_or_warn':
1640                         stretched_pp = FFmpegFixupStretchedPP(self)
1641                         if stretched_pp.available:
1642                             info_dict.setdefault('__postprocessors', [])
1643                             info_dict['__postprocessors'].append(stretched_pp)
1644                         else:
1645                             self.report_warning(
1646                                 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1647                                     info_dict['id'], stretched_ratio))
1648                     else:
1649                         assert fixup_policy in ('ignore', 'never')
1650
1651                 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1652                     if fixup_policy == 'warn':
1653                         self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1654                             info_dict['id']))
1655                     elif fixup_policy == 'detect_or_warn':
1656                         fixup_pp = FFmpegFixupM4aPP(self)
1657                         if fixup_pp.available:
1658                             info_dict.setdefault('__postprocessors', [])
1659                             info_dict['__postprocessors'].append(fixup_pp)
1660                         else:
1661                             self.report_warning(
1662                                 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1663                                     info_dict['id']))
1664                     else:
1665                         assert fixup_policy in ('ignore', 'never')
1666
1667                 try:
1668                     self.post_process(filename, info_dict)
1669                 except (PostProcessingError) as err:
1670                     self.report_error('postprocessing: %s' % str(err))
1671                     return
1672                 self.record_download_archive(info_dict)
1673
1674     def download(self, url_list):
1675         """Download a given list of URLs."""
1676         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1677         if (len(url_list) > 1 and
1678                 '%' not in outtmpl and
1679                 self.params.get('max_downloads') != 1):
1680             raise SameFileError(outtmpl)
1681
1682         for url in url_list:
1683             try:
1684                 # It also downloads the videos
1685                 res = self.extract_info(
1686                     url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1687             except UnavailableVideoError:
1688                 self.report_error('unable to download video')
1689             except MaxDownloadsReached:
1690                 self.to_screen('[info] Maximum number of downloaded files reached.')
1691                 raise
1692             else:
1693                 if self.params.get('dump_single_json', False):
1694                     self.to_stdout(json.dumps(res))
1695
1696         return self._download_retcode
1697
1698     def download_with_info_file(self, info_filename):
1699         with contextlib.closing(fileinput.FileInput(
1700                 [info_filename], mode='r',
1701                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1702             # FileInput doesn't have a read method, we can't call json.load
1703             info = self.filter_requested_info(json.loads('\n'.join(f)))
1704         try:
1705             self.process_ie_result(info, download=True)
1706         except DownloadError:
1707             webpage_url = info.get('webpage_url')
1708             if webpage_url is not None:
1709                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1710                 return self.download([webpage_url])
1711             else:
1712                 raise
1713         return self._download_retcode
1714
1715     @staticmethod
1716     def filter_requested_info(info_dict):
1717         return dict(
1718             (k, v) for k, v in info_dict.items()
1719             if k not in ['requested_formats', 'requested_subtitles'])
1720
1721     def post_process(self, filename, ie_info):
1722         """Run all the postprocessors on the given file."""
1723         info = dict(ie_info)
1724         info['filepath'] = filename
1725         pps_chain = []
1726         if ie_info.get('__postprocessors') is not None:
1727             pps_chain.extend(ie_info['__postprocessors'])
1728         pps_chain.extend(self._pps)
1729         for pp in pps_chain:
1730             files_to_delete = []
1731             try:
1732                 files_to_delete, info = pp.run(info)
1733             except PostProcessingError as e:
1734                 self.report_error(e.msg)
1735             if files_to_delete and not self.params.get('keepvideo', False):
1736                 for old_filename in files_to_delete:
1737                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1738                     try:
1739                         os.remove(encodeFilename(old_filename))
1740                     except (IOError, OSError):
1741                         self.report_warning('Unable to remove downloaded original file')
1742
1743     def _make_archive_id(self, info_dict):
1744         # Future-proof against any change in case
1745         # and backwards compatibility with prior versions
1746         extractor = info_dict.get('extractor_key')
1747         if extractor is None:
1748             if 'id' in info_dict:
1749                 extractor = info_dict.get('ie_key')  # key in a playlist
1750         if extractor is None:
1751             return None  # Incomplete video information
1752         return extractor.lower() + ' ' + info_dict['id']
1753
1754     def in_download_archive(self, info_dict):
1755         fn = self.params.get('download_archive')
1756         if fn is None:
1757             return False
1758
1759         vid_id = self._make_archive_id(info_dict)
1760         if vid_id is None:
1761             return False  # Incomplete video information
1762
1763         try:
1764             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1765                 for line in archive_file:
1766                     if line.strip() == vid_id:
1767                         return True
1768         except IOError as ioe:
1769             if ioe.errno != errno.ENOENT:
1770                 raise
1771         return False
1772
1773     def record_download_archive(self, info_dict):
1774         fn = self.params.get('download_archive')
1775         if fn is None:
1776             return
1777         vid_id = self._make_archive_id(info_dict)
1778         assert vid_id
1779         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1780             archive_file.write(vid_id + '\n')
1781
1782     @staticmethod
1783     def format_resolution(format, default='unknown'):
1784         if format.get('vcodec') == 'none':
1785             return 'audio only'
1786         if format.get('resolution') is not None:
1787             return format['resolution']
1788         if format.get('height') is not None:
1789             if format.get('width') is not None:
1790                 res = '%sx%s' % (format['width'], format['height'])
1791             else:
1792                 res = '%sp' % format['height']
1793         elif format.get('width') is not None:
1794             res = '?x%d' % format['width']
1795         else:
1796             res = default
1797         return res
1798
1799     def _format_note(self, fdict):
1800         res = ''
1801         if fdict.get('ext') in ['f4f', 'f4m']:
1802             res += '(unsupported) '
1803         if fdict.get('language'):
1804             if res:
1805                 res += ' '
1806             res += '[%s]' % fdict['language']
1807         if fdict.get('format_note') is not None:
1808             res += fdict['format_note'] + ' '
1809         if fdict.get('tbr') is not None:
1810             res += '%4dk ' % fdict['tbr']
1811         if fdict.get('container') is not None:
1812             if res:
1813                 res += ', '
1814             res += '%s container' % fdict['container']
1815         if (fdict.get('vcodec') is not None and
1816                 fdict.get('vcodec') != 'none'):
1817             if res:
1818                 res += ', '
1819             res += fdict['vcodec']
1820             if fdict.get('vbr') is not None:
1821                 res += '@'
1822         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1823             res += 'video@'
1824         if fdict.get('vbr') is not None:
1825             res += '%4dk' % fdict['vbr']
1826         if fdict.get('fps') is not None:
1827             res += ', %sfps' % fdict['fps']
1828         if fdict.get('acodec') is not None:
1829             if res:
1830                 res += ', '
1831             if fdict['acodec'] == 'none':
1832                 res += 'video only'
1833             else:
1834                 res += '%-5s' % fdict['acodec']
1835         elif fdict.get('abr') is not None:
1836             if res:
1837                 res += ', '
1838             res += 'audio'
1839         if fdict.get('abr') is not None:
1840             res += '@%3dk' % fdict['abr']
1841         if fdict.get('asr') is not None:
1842             res += ' (%5dHz)' % fdict['asr']
1843         if fdict.get('filesize') is not None:
1844             if res:
1845                 res += ', '
1846             res += format_bytes(fdict['filesize'])
1847         elif fdict.get('filesize_approx') is not None:
1848             if res:
1849                 res += ', '
1850             res += '~' + format_bytes(fdict['filesize_approx'])
1851         return res
1852
1853     def list_formats(self, info_dict):
1854         formats = info_dict.get('formats', [info_dict])
1855         table = [
1856             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1857             for f in formats
1858             if f.get('preference') is None or f['preference'] >= -1000]
1859         if len(formats) > 1:
1860             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1861
1862         header_line = ['format code', 'extension', 'resolution', 'note']
1863         self.to_screen(
1864             '[info] Available formats for %s:\n%s' %
1865             (info_dict['id'], render_table(header_line, table)))
1866
1867     def list_thumbnails(self, info_dict):
1868         thumbnails = info_dict.get('thumbnails')
1869         if not thumbnails:
1870             tn_url = info_dict.get('thumbnail')
1871             if tn_url:
1872                 thumbnails = [{'id': '0', 'url': tn_url}]
1873             else:
1874                 self.to_screen(
1875                     '[info] No thumbnails present for %s' % info_dict['id'])
1876                 return
1877
1878         self.to_screen(
1879             '[info] Thumbnails for %s:' % info_dict['id'])
1880         self.to_screen(render_table(
1881             ['ID', 'width', 'height', 'URL'],
1882             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1883
1884     def list_subtitles(self, video_id, subtitles, name='subtitles'):
1885         if not subtitles:
1886             self.to_screen('%s has no %s' % (video_id, name))
1887             return
1888         self.to_screen(
1889             'Available %s for %s:' % (name, video_id))
1890         self.to_screen(render_table(
1891             ['Language', 'formats'],
1892             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1893                 for lang, formats in subtitles.items()]))
1894
1895     def urlopen(self, req):
1896         """ Start an HTTP download """
1897         if isinstance(req, compat_basestring):
1898             req = sanitized_Request(req)
1899         return self._opener.open(req, timeout=self._socket_timeout)
1900
1901     def print_debug_header(self):
1902         if not self.params.get('verbose'):
1903             return
1904
1905         if type('') is not compat_str:
1906             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1907             self.report_warning(
1908                 'Your Python is broken! Update to a newer and supported version')
1909
1910         stdout_encoding = getattr(
1911             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1912         encoding_str = (
1913             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1914                 locale.getpreferredencoding(),
1915                 sys.getfilesystemencoding(),
1916                 stdout_encoding,
1917                 self.get_encoding()))
1918         write_string(encoding_str, encoding=None)
1919
1920         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1921         try:
1922             sp = subprocess.Popen(
1923                 ['git', 'rev-parse', '--short', 'HEAD'],
1924                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1925                 cwd=os.path.dirname(os.path.abspath(__file__)))
1926             out, err = sp.communicate()
1927             out = out.decode().strip()
1928             if re.match('[0-9a-f]+', out):
1929                 self._write_string('[debug] Git HEAD: ' + out + '\n')
1930         except Exception:
1931             try:
1932                 sys.exc_clear()
1933             except Exception:
1934                 pass
1935         self._write_string('[debug] Python version %s - %s\n' % (
1936             platform.python_version(), platform_name()))
1937
1938         exe_versions = FFmpegPostProcessor.get_versions(self)
1939         exe_versions['rtmpdump'] = rtmpdump_version()
1940         exe_str = ', '.join(
1941             '%s %s' % (exe, v)
1942             for exe, v in sorted(exe_versions.items())
1943             if v
1944         )
1945         if not exe_str:
1946             exe_str = 'none'
1947         self._write_string('[debug] exe versions: %s\n' % exe_str)
1948
1949         proxy_map = {}
1950         for handler in self._opener.handlers:
1951             if hasattr(handler, 'proxies'):
1952                 proxy_map.update(handler.proxies)
1953         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1954
1955         if self.params.get('call_home', False):
1956             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1957             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1958             latest_version = self.urlopen(
1959                 'https://yt-dl.org/latest/version').read().decode('utf-8')
1960             if version_tuple(latest_version) > version_tuple(__version__):
1961                 self.report_warning(
1962                     'You are using an outdated version (newest version: %s)! '
1963                     'See https://yt-dl.org/update if you need help updating.' %
1964                     latest_version)
1965
1966     def _setup_opener(self):
1967         timeout_val = self.params.get('socket_timeout')
1968         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1969
1970         opts_cookiefile = self.params.get('cookiefile')
1971         opts_proxy = self.params.get('proxy')
1972
1973         if opts_cookiefile is None:
1974             self.cookiejar = compat_cookiejar.CookieJar()
1975         else:
1976             self.cookiejar = compat_cookiejar.MozillaCookieJar(
1977                 opts_cookiefile)
1978             if os.access(opts_cookiefile, os.R_OK):
1979                 self.cookiejar.load()
1980
1981         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
1982         if opts_proxy is not None:
1983             if opts_proxy == '':
1984                 proxies = {}
1985             else:
1986                 proxies = {'http': opts_proxy, 'https': opts_proxy}
1987         else:
1988             proxies = compat_urllib_request.getproxies()
1989             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1990             if 'http' in proxies and 'https' not in proxies:
1991                 proxies['https'] = proxies['http']
1992         proxy_handler = PerRequestProxyHandler(proxies)
1993
1994         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1995         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1996         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1997         data_handler = compat_urllib_request_DataHandler()
1998
1999         # When passing our own FileHandler instance, build_opener won't add the
2000         # default FileHandler and allows us to disable the file protocol, which
2001         # can be used for malicious purposes (see
2002         # https://github.com/rg3/youtube-dl/issues/8227)
2003         file_handler = compat_urllib_request.FileHandler()
2004
2005         def file_open(*args, **kwargs):
2006             raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2007         file_handler.file_open = file_open
2008
2009         opener = compat_urllib_request.build_opener(
2010             proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2011
2012         # Delete the default user-agent header, which would otherwise apply in
2013         # cases where our custom HTTP handler doesn't come into play
2014         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2015         opener.addheaders = []
2016         self._opener = opener
2017
2018     def encode(self, s):
2019         if isinstance(s, bytes):
2020             return s  # Already encoded
2021
2022         try:
2023             return s.encode(self.get_encoding())
2024         except UnicodeEncodeError as err:
2025             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2026             raise
2027
2028     def get_encoding(self):
2029         encoding = self.params.get('encoding')
2030         if encoding is None:
2031             encoding = preferredencoding()
2032         return encoding
2033
2034     def _write_thumbnails(self, info_dict, filename):
2035         if self.params.get('writethumbnail', False):
2036             thumbnails = info_dict.get('thumbnails')
2037             if thumbnails:
2038                 thumbnails = [thumbnails[-1]]
2039         elif self.params.get('write_all_thumbnails', False):
2040             thumbnails = info_dict.get('thumbnails')
2041         else:
2042             return
2043
2044         if not thumbnails:
2045             # No thumbnails present, so return immediately
2046             return
2047
2048         for t in thumbnails:
2049             thumb_ext = determine_ext(t['url'], 'jpg')
2050             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2051             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2052             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2053
2054             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2055                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2056                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2057             else:
2058                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2059                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2060                 try:
2061                     uf = self.urlopen(t['url'])
2062                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2063                         shutil.copyfileobj(uf, thumbf)
2064                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2065                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2066                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2067                     self.report_warning('Unable to download thumbnail "%s": %s' %
2068                                         (t['url'], error_to_compat_str(err)))