]> git.bitcoin.ninja Git - youtube-dl/blob - youtube_dl/YoutubeDL.py
Merge branch 'bleacherreport' of github.com:remitamine/youtube-dl into remitamine...
[youtube-dl] / youtube_dl / YoutubeDL.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import absolute_import, unicode_literals
5
6 import collections
7 import contextlib
8 import datetime
9 import errno
10 import fileinput
11 import io
12 import itertools
13 import json
14 import locale
15 import operator
16 import os
17 import platform
18 import re
19 import shutil
20 import subprocess
21 import socket
22 import sys
23 import time
24 import tokenize
25 import traceback
26
27 if os.name == 'nt':
28     import ctypes
29
30 from .compat import (
31     compat_basestring,
32     compat_cookiejar,
33     compat_expanduser,
34     compat_get_terminal_size,
35     compat_http_client,
36     compat_kwargs,
37     compat_str,
38     compat_tokenize_tokenize,
39     compat_urllib_error,
40     compat_urllib_request,
41     compat_urllib_request_DataHandler,
42 )
43 from .utils import (
44     ContentTooShortError,
45     date_from_str,
46     DateRange,
47     DEFAULT_OUTTMPL,
48     determine_ext,
49     DownloadError,
50     encode_compat_str,
51     encodeFilename,
52     error_to_compat_str,
53     ExtractorError,
54     format_bytes,
55     formatSeconds,
56     locked_file,
57     make_HTTPS_handler,
58     MaxDownloadsReached,
59     PagedList,
60     parse_filesize,
61     PerRequestProxyHandler,
62     PostProcessingError,
63     platform_name,
64     preferredencoding,
65     render_table,
66     SameFileError,
67     sanitize_filename,
68     sanitize_path,
69     sanitized_Request,
70     std_headers,
71     subtitles_filename,
72     UnavailableVideoError,
73     url_basename,
74     version_tuple,
75     write_json_file,
76     write_string,
77     YoutubeDLCookieProcessor,
78     YoutubeDLHandler,
79     prepend_extension,
80     replace_extension,
81     args_to_str,
82     age_restricted,
83 )
84 from .cache import Cache
85 from .extractor import get_info_extractor, gen_extractors
86 from .downloader import get_suitable_downloader
87 from .downloader.rtmp import rtmpdump_version
88 from .postprocessor import (
89     FFmpegFixupM4aPP,
90     FFmpegFixupStretchedPP,
91     FFmpegMergerPP,
92     FFmpegPostProcessor,
93     get_postprocessor,
94 )
95 from .version import __version__
96
97
98 class YoutubeDL(object):
99     """YoutubeDL class.
100
101     YoutubeDL objects are the ones responsible of downloading the
102     actual video file and writing it to disk if the user has requested
103     it, among some other tasks. In most cases there should be one per
104     program. As, given a video URL, the downloader doesn't know how to
105     extract all the needed information, task that InfoExtractors do, it
106     has to pass the URL to one of them.
107
108     For this, YoutubeDL objects have a method that allows
109     InfoExtractors to be registered in a given order. When it is passed
110     a URL, the YoutubeDL object handles it to the first InfoExtractor it
111     finds that reports being able to handle it. The InfoExtractor extracts
112     all the information about the video or videos the URL refers to, and
113     YoutubeDL process the extracted information, possibly using a File
114     Downloader to download the video.
115
116     YoutubeDL objects accept a lot of parameters. In order not to saturate
117     the object constructor with arguments, it receives a dictionary of
118     options instead. These options are available through the params
119     attribute for the InfoExtractors to use. The YoutubeDL also
120     registers itself as the downloader in charge for the InfoExtractors
121     that are added to it, so this is a "mutual registration".
122
123     Available options:
124
125     username:          Username for authentication purposes.
126     password:          Password for authentication purposes.
127     videopassword:     Password for accessing a video.
128     usenetrc:          Use netrc for authentication instead.
129     verbose:           Print additional info to stdout.
130     quiet:             Do not print messages to stdout.
131     no_warnings:       Do not print out anything for warnings.
132     forceurl:          Force printing final URL.
133     forcetitle:        Force printing title.
134     forceid:           Force printing ID.
135     forcethumbnail:    Force printing thumbnail URL.
136     forcedescription:  Force printing description.
137     forcefilename:     Force printing final filename.
138     forceduration:     Force printing duration.
139     forcejson:         Force printing info_dict as JSON.
140     dump_single_json:  Force printing the info_dict of the whole playlist
141                        (or video) as a single JSON line.
142     simulate:          Do not download the video files.
143     format:            Video format code. See options.py for more information.
144     outtmpl:           Template for output names.
145     restrictfilenames: Do not allow "&" and spaces in file names
146     ignoreerrors:      Do not stop on download errors.
147     force_generic_extractor: Force downloader to use the generic extractor
148     nooverwrites:      Prevent overwriting files.
149     playliststart:     Playlist item to start at.
150     playlistend:       Playlist item to end at.
151     playlist_items:    Specific indices of playlist to download.
152     playlistreverse:   Download playlist items in reverse order.
153     matchtitle:        Download only matching titles.
154     rejecttitle:       Reject downloads for matching titles.
155     logger:            Log messages to a logging.Logger instance.
156     logtostderr:       Log messages to stderr instead of stdout.
157     writedescription:  Write the video description to a .description file
158     writeinfojson:     Write the video description to a .info.json file
159     writeannotations:  Write the video annotations to a .annotations.xml file
160     writethumbnail:    Write the thumbnail image to a file
161     write_all_thumbnails:  Write all thumbnail formats to files
162     writesubtitles:    Write the video subtitles to a file
163     writeautomaticsub: Write the automatically generated subtitles to a file
164     allsubtitles:      Downloads all the subtitles of the video
165                        (requires writesubtitles or writeautomaticsub)
166     listsubtitles:     Lists all available subtitles for the video
167     subtitlesformat:   The format code for subtitles
168     subtitleslangs:    List of languages of the subtitles to download
169     keepvideo:         Keep the video file after post-processing
170     daterange:         A DateRange object, download only if the upload_date is in the range.
171     skip_download:     Skip the actual download of the video file
172     cachedir:          Location of the cache files in the filesystem.
173                        False to disable filesystem cache.
174     noplaylist:        Download single video instead of a playlist if in doubt.
175     age_limit:         An integer representing the user's age in years.
176                        Unsuitable videos for the given age are skipped.
177     min_views:         An integer representing the minimum view count the video
178                        must have in order to not be skipped.
179                        Videos without view count information are always
180                        downloaded. None for no limit.
181     max_views:         An integer representing the maximum view count.
182                        Videos that are more popular than that are not
183                        downloaded.
184                        Videos without view count information are always
185                        downloaded. None for no limit.
186     download_archive:  File name of a file where all downloads are recorded.
187                        Videos already present in the file are not downloaded
188                        again.
189     cookiefile:        File name where cookies should be read from and dumped to.
190     nocheckcertificate:Do not verify SSL certificates
191     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
192                        At the moment, this is only supported by YouTube.
193     proxy:             URL of the proxy server to use
194     cn_verification_proxy:  URL of the proxy to use for IP address verification
195                        on Chinese sites. (Experimental)
196     socket_timeout:    Time to wait for unresponsive hosts, in seconds
197     bidi_workaround:   Work around buggy terminals without bidirectional text
198                        support, using fridibi
199     debug_printtraffic:Print out sent and received HTTP traffic
200     include_ads:       Download ads as well
201     default_search:    Prepend this string if an input url is not valid.
202                        'auto' for elaborate guessing
203     encoding:          Use this encoding instead of the system-specified.
204     extract_flat:      Do not resolve URLs, return the immediate result.
205                        Pass in 'in_playlist' to only show this behavior for
206                        playlist items.
207     postprocessors:    A list of dictionaries, each with an entry
208                        * key:  The name of the postprocessor. See
209                                youtube_dl/postprocessor/__init__.py for a list.
210                        as well as any further keyword arguments for the
211                        postprocessor.
212     progress_hooks:    A list of functions that get called on download
213                        progress, with a dictionary with the entries
214                        * status: One of "downloading", "error", or "finished".
215                                  Check this first and ignore unknown values.
216
217                        If status is one of "downloading", or "finished", the
218                        following properties may also be present:
219                        * filename: The final filename (always present)
220                        * tmpfilename: The filename we're currently writing to
221                        * downloaded_bytes: Bytes on disk
222                        * total_bytes: Size of the whole file, None if unknown
223                        * total_bytes_estimate: Guess of the eventual file size,
224                                                None if unavailable.
225                        * elapsed: The number of seconds since download started.
226                        * eta: The estimated time in seconds, None if unknown
227                        * speed: The download speed in bytes/second, None if
228                                 unknown
229                        * fragment_index: The counter of the currently
230                                          downloaded video fragment.
231                        * fragment_count: The number of fragments (= individual
232                                          files that will be merged)
233
234                        Progress hooks are guaranteed to be called at least once
235                        (with status "finished") if the download is successful.
236     merge_output_format: Extension to use when merging formats.
237     fixup:             Automatically correct known faults of the file.
238                        One of:
239                        - "never": do nothing
240                        - "warn": only emit a warning
241                        - "detect_or_warn": check whether we can do anything
242                                            about it, warn otherwise (default)
243     source_address:    (Experimental) Client-side IP address to bind to.
244     call_home:         Boolean, true iff we are allowed to contact the
245                        youtube-dl servers for debugging.
246     sleep_interval:    Number of seconds to sleep before each download.
247     listformats:       Print an overview of available video formats and exit.
248     list_thumbnails:   Print a table of all thumbnails and exit.
249     match_filter:      A function that gets called with the info_dict of
250                        every video.
251                        If it returns a message, the video is ignored.
252                        If it returns None, the video is downloaded.
253                        match_filter_func in utils.py is one example for this.
254     no_color:          Do not emit color codes in output.
255
256     The following options determine which downloader is picked:
257     external_downloader: Executable of the external downloader to call.
258                        None or unset for standard (built-in) downloader.
259     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
260
261     The following parameters are not used by YoutubeDL itself, they are used by
262     the downloader (see youtube_dl/downloader/common.py):
263     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
264     noresizebuffer, retries, continuedl, noprogress, consoletitle,
265     xattr_set_filesize, external_downloader_args.
266
267     The following options are used by the post processors:
268     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
269                        otherwise prefer avconv.
270     postprocessor_args: A list of additional command-line arguments for the
271                         postprocessor.
272     """
273
274     params = None
275     _ies = []
276     _pps = []
277     _download_retcode = None
278     _num_downloads = None
279     _screen_file = None
280
281     def __init__(self, params=None, auto_init=True):
282         """Create a FileDownloader object with the given options."""
283         if params is None:
284             params = {}
285         self._ies = []
286         self._ies_instances = {}
287         self._pps = []
288         self._progress_hooks = []
289         self._download_retcode = 0
290         self._num_downloads = 0
291         self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
292         self._err_file = sys.stderr
293         self.params = {
294             # Default parameters
295             'nocheckcertificate': False,
296         }
297         self.params.update(params)
298         self.cache = Cache(self)
299
300         if params.get('bidi_workaround', False):
301             try:
302                 import pty
303                 master, slave = pty.openpty()
304                 width = compat_get_terminal_size().columns
305                 if width is None:
306                     width_args = []
307                 else:
308                     width_args = ['-w', str(width)]
309                 sp_kwargs = dict(
310                     stdin=subprocess.PIPE,
311                     stdout=slave,
312                     stderr=self._err_file)
313                 try:
314                     self._output_process = subprocess.Popen(
315                         ['bidiv'] + width_args, **sp_kwargs
316                     )
317                 except OSError:
318                     self._output_process = subprocess.Popen(
319                         ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
320                 self._output_channel = os.fdopen(master, 'rb')
321             except OSError as ose:
322                 if ose.errno == 2:
323                     self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that  fribidi  is an executable file in one of the directories in your $PATH.')
324                 else:
325                     raise
326
327         if (sys.version_info >= (3,) and sys.platform != 'win32' and
328                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
329                 not params.get('restrictfilenames', False)):
330             # On Python 3, the Unicode filesystem API will throw errors (#1474)
331             self.report_warning(
332                 'Assuming --restrict-filenames since file system encoding '
333                 'cannot encode all characters. '
334                 'Set the LC_ALL environment variable to fix this.')
335             self.params['restrictfilenames'] = True
336
337         if isinstance(params.get('outtmpl'), bytes):
338             self.report_warning(
339                 'Parameter outtmpl is bytes, but should be a unicode string. '
340                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
341
342         self._setup_opener()
343
344         if auto_init:
345             self.print_debug_header()
346             self.add_default_info_extractors()
347
348         for pp_def_raw in self.params.get('postprocessors', []):
349             pp_class = get_postprocessor(pp_def_raw['key'])
350             pp_def = dict(pp_def_raw)
351             del pp_def['key']
352             pp = pp_class(self, **compat_kwargs(pp_def))
353             self.add_post_processor(pp)
354
355         for ph in self.params.get('progress_hooks', []):
356             self.add_progress_hook(ph)
357
358     def warn_if_short_id(self, argv):
359         # short YouTube ID starting with dash?
360         idxs = [
361             i for i, a in enumerate(argv)
362             if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
363         if idxs:
364             correct_argv = (
365                 ['youtube-dl'] +
366                 [a for i, a in enumerate(argv) if i not in idxs] +
367                 ['--'] + [argv[i] for i in idxs]
368             )
369             self.report_warning(
370                 'Long argument string detected. '
371                 'Use -- to separate parameters and URLs, like this:\n%s\n' %
372                 args_to_str(correct_argv))
373
374     def add_info_extractor(self, ie):
375         """Add an InfoExtractor object to the end of the list."""
376         self._ies.append(ie)
377         self._ies_instances[ie.ie_key()] = ie
378         ie.set_downloader(self)
379
380     def get_info_extractor(self, ie_key):
381         """
382         Get an instance of an IE with name ie_key, it will try to get one from
383         the _ies list, if there's no instance it will create a new one and add
384         it to the extractor list.
385         """
386         ie = self._ies_instances.get(ie_key)
387         if ie is None:
388             ie = get_info_extractor(ie_key)()
389             self.add_info_extractor(ie)
390         return ie
391
392     def add_default_info_extractors(self):
393         """
394         Add the InfoExtractors returned by gen_extractors to the end of the list
395         """
396         for ie in gen_extractors():
397             self.add_info_extractor(ie)
398
399     def add_post_processor(self, pp):
400         """Add a PostProcessor object to the end of the chain."""
401         self._pps.append(pp)
402         pp.set_downloader(self)
403
404     def add_progress_hook(self, ph):
405         """Add the progress hook (currently only for the file downloader)"""
406         self._progress_hooks.append(ph)
407
408     def _bidi_workaround(self, message):
409         if not hasattr(self, '_output_channel'):
410             return message
411
412         assert hasattr(self, '_output_process')
413         assert isinstance(message, compat_str)
414         line_count = message.count('\n') + 1
415         self._output_process.stdin.write((message + '\n').encode('utf-8'))
416         self._output_process.stdin.flush()
417         res = ''.join(self._output_channel.readline().decode('utf-8')
418                       for _ in range(line_count))
419         return res[:-len('\n')]
420
421     def to_screen(self, message, skip_eol=False):
422         """Print message to stdout if not in quiet mode."""
423         return self.to_stdout(message, skip_eol, check_quiet=True)
424
425     def _write_string(self, s, out=None):
426         write_string(s, out=out, encoding=self.params.get('encoding'))
427
428     def to_stdout(self, message, skip_eol=False, check_quiet=False):
429         """Print message to stdout if not in quiet mode."""
430         if self.params.get('logger'):
431             self.params['logger'].debug(message)
432         elif not check_quiet or not self.params.get('quiet', False):
433             message = self._bidi_workaround(message)
434             terminator = ['\n', ''][skip_eol]
435             output = message + terminator
436
437             self._write_string(output, self._screen_file)
438
439     def to_stderr(self, message):
440         """Print message to stderr."""
441         assert isinstance(message, compat_str)
442         if self.params.get('logger'):
443             self.params['logger'].error(message)
444         else:
445             message = self._bidi_workaround(message)
446             output = message + '\n'
447             self._write_string(output, self._err_file)
448
449     def to_console_title(self, message):
450         if not self.params.get('consoletitle', False):
451             return
452         if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
453             # c_wchar_p() might not be necessary if `message` is
454             # already of type unicode()
455             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
456         elif 'TERM' in os.environ:
457             self._write_string('\033]0;%s\007' % message, self._screen_file)
458
459     def save_console_title(self):
460         if not self.params.get('consoletitle', False):
461             return
462         if 'TERM' in os.environ:
463             # Save the title on stack
464             self._write_string('\033[22;0t', self._screen_file)
465
466     def restore_console_title(self):
467         if not self.params.get('consoletitle', False):
468             return
469         if 'TERM' in os.environ:
470             # Restore the title from stack
471             self._write_string('\033[23;0t', self._screen_file)
472
473     def __enter__(self):
474         self.save_console_title()
475         return self
476
477     def __exit__(self, *args):
478         self.restore_console_title()
479
480         if self.params.get('cookiefile') is not None:
481             self.cookiejar.save()
482
483     def trouble(self, message=None, tb=None):
484         """Determine action to take when a download problem appears.
485
486         Depending on if the downloader has been configured to ignore
487         download errors or not, this method may throw an exception or
488         not when errors are found, after printing the message.
489
490         tb, if given, is additional traceback information.
491         """
492         if message is not None:
493             self.to_stderr(message)
494         if self.params.get('verbose'):
495             if tb is None:
496                 if sys.exc_info()[0]:  # if .trouble has been called from an except block
497                     tb = ''
498                     if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
499                         tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
500                     tb += encode_compat_str(traceback.format_exc())
501                 else:
502                     tb_data = traceback.format_list(traceback.extract_stack())
503                     tb = ''.join(tb_data)
504             self.to_stderr(tb)
505         if not self.params.get('ignoreerrors', False):
506             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
507                 exc_info = sys.exc_info()[1].exc_info
508             else:
509                 exc_info = sys.exc_info()
510             raise DownloadError(message, exc_info)
511         self._download_retcode = 1
512
513     def report_warning(self, message):
514         '''
515         Print the message to stderr, it will be prefixed with 'WARNING:'
516         If stderr is a tty file the 'WARNING:' will be colored
517         '''
518         if self.params.get('logger') is not None:
519             self.params['logger'].warning(message)
520         else:
521             if self.params.get('no_warnings'):
522                 return
523             if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
524                 _msg_header = '\033[0;33mWARNING:\033[0m'
525             else:
526                 _msg_header = 'WARNING:'
527             warning_message = '%s %s' % (_msg_header, message)
528             self.to_stderr(warning_message)
529
530     def report_error(self, message, tb=None):
531         '''
532         Do the same as trouble, but prefixes the message with 'ERROR:', colored
533         in red if stderr is a tty file.
534         '''
535         if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
536             _msg_header = '\033[0;31mERROR:\033[0m'
537         else:
538             _msg_header = 'ERROR:'
539         error_message = '%s %s' % (_msg_header, message)
540         self.trouble(error_message, tb)
541
542     def report_file_already_downloaded(self, file_name):
543         """Report file has already been fully downloaded."""
544         try:
545             self.to_screen('[download] %s has already been downloaded' % file_name)
546         except UnicodeEncodeError:
547             self.to_screen('[download] The file has already been downloaded')
548
549     def prepare_filename(self, info_dict):
550         """Generate the output filename."""
551         try:
552             template_dict = dict(info_dict)
553
554             template_dict['epoch'] = int(time.time())
555             autonumber_size = self.params.get('autonumber_size')
556             if autonumber_size is None:
557                 autonumber_size = 5
558             autonumber_templ = '%0' + str(autonumber_size) + 'd'
559             template_dict['autonumber'] = autonumber_templ % self._num_downloads
560             if template_dict.get('playlist_index') is not None:
561                 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
562             if template_dict.get('resolution') is None:
563                 if template_dict.get('width') and template_dict.get('height'):
564                     template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
565                 elif template_dict.get('height'):
566                     template_dict['resolution'] = '%sp' % template_dict['height']
567                 elif template_dict.get('width'):
568                     template_dict['resolution'] = '?x%d' % template_dict['width']
569
570             sanitize = lambda k, v: sanitize_filename(
571                 compat_str(v),
572                 restricted=self.params.get('restrictfilenames'),
573                 is_id=(k == 'id'))
574             template_dict = dict((k, sanitize(k, v))
575                                  for k, v in template_dict.items()
576                                  if v is not None)
577             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
578
579             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
580             tmpl = compat_expanduser(outtmpl)
581             filename = tmpl % template_dict
582             # Temporary fix for #4787
583             # 'Treat' all problem characters by passing filename through preferredencoding
584             # to workaround encoding issues with subprocess on python2 @ Windows
585             if sys.version_info < (3, 0) and sys.platform == 'win32':
586                 filename = encodeFilename(filename, True).decode(preferredencoding())
587             return sanitize_path(filename)
588         except ValueError as err:
589             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
590             return None
591
592     def _match_entry(self, info_dict, incomplete):
593         """ Returns None iff the file should be downloaded """
594
595         video_title = info_dict.get('title', info_dict.get('id', 'video'))
596         if 'title' in info_dict:
597             # This can happen when we're just evaluating the playlist
598             title = info_dict['title']
599             matchtitle = self.params.get('matchtitle', False)
600             if matchtitle:
601                 if not re.search(matchtitle, title, re.IGNORECASE):
602                     return '"' + title + '" title did not match pattern "' + matchtitle + '"'
603             rejecttitle = self.params.get('rejecttitle', False)
604             if rejecttitle:
605                 if re.search(rejecttitle, title, re.IGNORECASE):
606                     return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
607         date = info_dict.get('upload_date', None)
608         if date is not None:
609             dateRange = self.params.get('daterange', DateRange())
610             if date not in dateRange:
611                 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
612         view_count = info_dict.get('view_count', None)
613         if view_count is not None:
614             min_views = self.params.get('min_views')
615             if min_views is not None and view_count < min_views:
616                 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
617             max_views = self.params.get('max_views')
618             if max_views is not None and view_count > max_views:
619                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
620         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
621             return 'Skipping "%s" because it is age restricted' % video_title
622         if self.in_download_archive(info_dict):
623             return '%s has already been recorded in archive' % video_title
624
625         if not incomplete:
626             match_filter = self.params.get('match_filter')
627             if match_filter is not None:
628                 ret = match_filter(info_dict)
629                 if ret is not None:
630                     return ret
631
632         return None
633
634     @staticmethod
635     def add_extra_info(info_dict, extra_info):
636         '''Set the keys from extra_info in info dict if they are missing'''
637         for key, value in extra_info.items():
638             info_dict.setdefault(key, value)
639
640     def extract_info(self, url, download=True, ie_key=None, extra_info={},
641                      process=True, force_generic_extractor=False):
642         '''
643         Returns a list with a dictionary for each video we find.
644         If 'download', also downloads the videos.
645         extra_info is a dict containing the extra values to add to each result
646         '''
647
648         if not ie_key and force_generic_extractor:
649             ie_key = 'Generic'
650
651         if ie_key:
652             ies = [self.get_info_extractor(ie_key)]
653         else:
654             ies = self._ies
655
656         for ie in ies:
657             if not ie.suitable(url):
658                 continue
659
660             if not ie.working():
661                 self.report_warning('The program functionality for this site has been marked as broken, '
662                                     'and will probably not work.')
663
664             try:
665                 ie_result = ie.extract(url)
666                 if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
667                     break
668                 if isinstance(ie_result, list):
669                     # Backwards compatibility: old IE result format
670                     ie_result = {
671                         '_type': 'compat_list',
672                         'entries': ie_result,
673                     }
674                 self.add_default_extra_info(ie_result, ie, url)
675                 if process:
676                     return self.process_ie_result(ie_result, download, extra_info)
677                 else:
678                     return ie_result
679             except ExtractorError as e:  # An error we somewhat expected
680                 self.report_error(compat_str(e), e.format_traceback())
681                 break
682             except MaxDownloadsReached:
683                 raise
684             except Exception as e:
685                 if self.params.get('ignoreerrors', False):
686                     self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
687                     break
688                 else:
689                     raise
690         else:
691             self.report_error('no suitable InfoExtractor for URL %s' % url)
692
693     def add_default_extra_info(self, ie_result, ie, url):
694         self.add_extra_info(ie_result, {
695             'extractor': ie.IE_NAME,
696             'webpage_url': url,
697             'webpage_url_basename': url_basename(url),
698             'extractor_key': ie.ie_key(),
699         })
700
701     def process_ie_result(self, ie_result, download=True, extra_info={}):
702         """
703         Take the result of the ie(may be modified) and resolve all unresolved
704         references (URLs, playlist items).
705
706         It will also download the videos if 'download'.
707         Returns the resolved ie_result.
708         """
709
710         result_type = ie_result.get('_type', 'video')
711
712         if result_type in ('url', 'url_transparent'):
713             extract_flat = self.params.get('extract_flat', False)
714             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
715                     extract_flat is True):
716                 if self.params.get('forcejson', False):
717                     self.to_stdout(json.dumps(ie_result))
718                 return ie_result
719
720         if result_type == 'video':
721             self.add_extra_info(ie_result, extra_info)
722             return self.process_video_result(ie_result, download=download)
723         elif result_type == 'url':
724             # We have to add extra_info to the results because it may be
725             # contained in a playlist
726             return self.extract_info(ie_result['url'],
727                                      download,
728                                      ie_key=ie_result.get('ie_key'),
729                                      extra_info=extra_info)
730         elif result_type == 'url_transparent':
731             # Use the information from the embedding page
732             info = self.extract_info(
733                 ie_result['url'], ie_key=ie_result.get('ie_key'),
734                 extra_info=extra_info, download=False, process=False)
735
736             force_properties = dict(
737                 (k, v) for k, v in ie_result.items() if v is not None)
738             for f in ('_type', 'url'):
739                 if f in force_properties:
740                     del force_properties[f]
741             new_result = info.copy()
742             new_result.update(force_properties)
743
744             assert new_result.get('_type') != 'url_transparent'
745
746             return self.process_ie_result(
747                 new_result, download=download, extra_info=extra_info)
748         elif result_type == 'playlist' or result_type == 'multi_video':
749             # We process each entry in the playlist
750             playlist = ie_result.get('title', None) or ie_result.get('id', None)
751             self.to_screen('[download] Downloading playlist: %s' % playlist)
752
753             playlist_results = []
754
755             playliststart = self.params.get('playliststart', 1) - 1
756             playlistend = self.params.get('playlistend', None)
757             # For backwards compatibility, interpret -1 as whole list
758             if playlistend == -1:
759                 playlistend = None
760
761             playlistitems_str = self.params.get('playlist_items', None)
762             playlistitems = None
763             if playlistitems_str is not None:
764                 def iter_playlistitems(format):
765                     for string_segment in format.split(','):
766                         if '-' in string_segment:
767                             start, end = string_segment.split('-')
768                             for item in range(int(start), int(end) + 1):
769                                 yield int(item)
770                         else:
771                             yield int(string_segment)
772                 playlistitems = iter_playlistitems(playlistitems_str)
773
774             ie_entries = ie_result['entries']
775             if isinstance(ie_entries, list):
776                 n_all_entries = len(ie_entries)
777                 if playlistitems:
778                     entries = [
779                         ie_entries[i - 1] for i in playlistitems
780                         if -n_all_entries <= i - 1 < n_all_entries]
781                 else:
782                     entries = ie_entries[playliststart:playlistend]
783                 n_entries = len(entries)
784                 self.to_screen(
785                     "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
786                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
787             elif isinstance(ie_entries, PagedList):
788                 if playlistitems:
789                     entries = []
790                     for item in playlistitems:
791                         entries.extend(ie_entries.getslice(
792                             item - 1, item
793                         ))
794                 else:
795                     entries = ie_entries.getslice(
796                         playliststart, playlistend)
797                 n_entries = len(entries)
798                 self.to_screen(
799                     "[%s] playlist %s: Downloading %d videos" %
800                     (ie_result['extractor'], playlist, n_entries))
801             else:  # iterable
802                 if playlistitems:
803                     entry_list = list(ie_entries)
804                     entries = [entry_list[i - 1] for i in playlistitems]
805                 else:
806                     entries = list(itertools.islice(
807                         ie_entries, playliststart, playlistend))
808                 n_entries = len(entries)
809                 self.to_screen(
810                     "[%s] playlist %s: Downloading %d videos" %
811                     (ie_result['extractor'], playlist, n_entries))
812
813             if self.params.get('playlistreverse', False):
814                 entries = entries[::-1]
815
816             for i, entry in enumerate(entries, 1):
817                 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
818                 extra = {
819                     'n_entries': n_entries,
820                     'playlist': playlist,
821                     'playlist_id': ie_result.get('id'),
822                     'playlist_title': ie_result.get('title'),
823                     'playlist_index': i + playliststart,
824                     'extractor': ie_result['extractor'],
825                     'webpage_url': ie_result['webpage_url'],
826                     'webpage_url_basename': url_basename(ie_result['webpage_url']),
827                     'extractor_key': ie_result['extractor_key'],
828                 }
829
830                 reason = self._match_entry(entry, incomplete=True)
831                 if reason is not None:
832                     self.to_screen('[download] ' + reason)
833                     continue
834
835                 entry_result = self.process_ie_result(entry,
836                                                       download=download,
837                                                       extra_info=extra)
838                 playlist_results.append(entry_result)
839             ie_result['entries'] = playlist_results
840             self.to_screen('[download] Finished downloading playlist: %s' % playlist)
841             return ie_result
842         elif result_type == 'compat_list':
843             self.report_warning(
844                 'Extractor %s returned a compat_list result. '
845                 'It needs to be updated.' % ie_result.get('extractor'))
846
847             def _fixup(r):
848                 self.add_extra_info(
849                     r,
850                     {
851                         'extractor': ie_result['extractor'],
852                         'webpage_url': ie_result['webpage_url'],
853                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
854                         'extractor_key': ie_result['extractor_key'],
855                     }
856                 )
857                 return r
858             ie_result['entries'] = [
859                 self.process_ie_result(_fixup(r), download, extra_info)
860                 for r in ie_result['entries']
861             ]
862             return ie_result
863         else:
864             raise Exception('Invalid result type: %s' % result_type)
865
866     def _build_format_filter(self, filter_spec):
867         " Returns a function to filter the formats according to the filter_spec "
868
869         OPERATORS = {
870             '<': operator.lt,
871             '<=': operator.le,
872             '>': operator.gt,
873             '>=': operator.ge,
874             '=': operator.eq,
875             '!=': operator.ne,
876         }
877         operator_rex = re.compile(r'''(?x)\s*
878             (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
879             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
880             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
881             $
882             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
883         m = operator_rex.search(filter_spec)
884         if m:
885             try:
886                 comparison_value = int(m.group('value'))
887             except ValueError:
888                 comparison_value = parse_filesize(m.group('value'))
889                 if comparison_value is None:
890                     comparison_value = parse_filesize(m.group('value') + 'B')
891                 if comparison_value is None:
892                     raise ValueError(
893                         'Invalid value %r in format specification %r' % (
894                             m.group('value'), filter_spec))
895             op = OPERATORS[m.group('op')]
896
897         if not m:
898             STR_OPERATORS = {
899                 '=': operator.eq,
900                 '!=': operator.ne,
901             }
902             str_operator_rex = re.compile(r'''(?x)
903                 \s*(?P<key>ext|acodec|vcodec|container|protocol)
904                 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
905                 \s*(?P<value>[a-zA-Z0-9_-]+)
906                 \s*$
907                 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
908             m = str_operator_rex.search(filter_spec)
909             if m:
910                 comparison_value = m.group('value')
911                 op = STR_OPERATORS[m.group('op')]
912
913         if not m:
914             raise ValueError('Invalid filter specification %r' % filter_spec)
915
916         def _filter(f):
917             actual_value = f.get(m.group('key'))
918             if actual_value is None:
919                 return m.group('none_inclusive')
920             return op(actual_value, comparison_value)
921         return _filter
922
923     def build_format_selector(self, format_spec):
924         def syntax_error(note, start):
925             message = (
926                 'Invalid format specification: '
927                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
928             return SyntaxError(message)
929
930         PICKFIRST = 'PICKFIRST'
931         MERGE = 'MERGE'
932         SINGLE = 'SINGLE'
933         GROUP = 'GROUP'
934         FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
935
936         def _parse_filter(tokens):
937             filter_parts = []
938             for type, string, start, _, _ in tokens:
939                 if type == tokenize.OP and string == ']':
940                     return ''.join(filter_parts)
941                 else:
942                     filter_parts.append(string)
943
944         def _remove_unused_ops(tokens):
945             # Remove operators that we don't use and join them with the surrounding strings
946             # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
947             ALLOWED_OPS = ('/', '+', ',', '(', ')')
948             last_string, last_start, last_end, last_line = None, None, None, None
949             for type, string, start, end, line in tokens:
950                 if type == tokenize.OP and string == '[':
951                     if last_string:
952                         yield tokenize.NAME, last_string, last_start, last_end, last_line
953                         last_string = None
954                     yield type, string, start, end, line
955                     # everything inside brackets will be handled by _parse_filter
956                     for type, string, start, end, line in tokens:
957                         yield type, string, start, end, line
958                         if type == tokenize.OP and string == ']':
959                             break
960                 elif type == tokenize.OP and string in ALLOWED_OPS:
961                     if last_string:
962                         yield tokenize.NAME, last_string, last_start, last_end, last_line
963                         last_string = None
964                     yield type, string, start, end, line
965                 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
966                     if not last_string:
967                         last_string = string
968                         last_start = start
969                         last_end = end
970                     else:
971                         last_string += string
972             if last_string:
973                 yield tokenize.NAME, last_string, last_start, last_end, last_line
974
975         def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
976             selectors = []
977             current_selector = None
978             for type, string, start, _, _ in tokens:
979                 # ENCODING is only defined in python 3.x
980                 if type == getattr(tokenize, 'ENCODING', None):
981                     continue
982                 elif type in [tokenize.NAME, tokenize.NUMBER]:
983                     current_selector = FormatSelector(SINGLE, string, [])
984                 elif type == tokenize.OP:
985                     if string == ')':
986                         if not inside_group:
987                             # ')' will be handled by the parentheses group
988                             tokens.restore_last_token()
989                         break
990                     elif inside_merge and string in ['/', ',']:
991                         tokens.restore_last_token()
992                         break
993                     elif inside_choice and string == ',':
994                         tokens.restore_last_token()
995                         break
996                     elif string == ',':
997                         if not current_selector:
998                             raise syntax_error('"," must follow a format selector', start)
999                         selectors.append(current_selector)
1000                         current_selector = None
1001                     elif string == '/':
1002                         if not current_selector:
1003                             raise syntax_error('"/" must follow a format selector', start)
1004                         first_choice = current_selector
1005                         second_choice = _parse_format_selection(tokens, inside_choice=True)
1006                         current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1007                     elif string == '[':
1008                         if not current_selector:
1009                             current_selector = FormatSelector(SINGLE, 'best', [])
1010                         format_filter = _parse_filter(tokens)
1011                         current_selector.filters.append(format_filter)
1012                     elif string == '(':
1013                         if current_selector:
1014                             raise syntax_error('Unexpected "("', start)
1015                         group = _parse_format_selection(tokens, inside_group=True)
1016                         current_selector = FormatSelector(GROUP, group, [])
1017                     elif string == '+':
1018                         video_selector = current_selector
1019                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
1020                         if not video_selector or not audio_selector:
1021                             raise syntax_error('"+" must be between two format selectors', start)
1022                         current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1023                     else:
1024                         raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1025                 elif type == tokenize.ENDMARKER:
1026                     break
1027             if current_selector:
1028                 selectors.append(current_selector)
1029             return selectors
1030
1031         def _build_selector_function(selector):
1032             if isinstance(selector, list):
1033                 fs = [_build_selector_function(s) for s in selector]
1034
1035                 def selector_function(formats):
1036                     for f in fs:
1037                         for format in f(formats):
1038                             yield format
1039                 return selector_function
1040             elif selector.type == GROUP:
1041                 selector_function = _build_selector_function(selector.selector)
1042             elif selector.type == PICKFIRST:
1043                 fs = [_build_selector_function(s) for s in selector.selector]
1044
1045                 def selector_function(formats):
1046                     for f in fs:
1047                         picked_formats = list(f(formats))
1048                         if picked_formats:
1049                             return picked_formats
1050                     return []
1051             elif selector.type == SINGLE:
1052                 format_spec = selector.selector
1053
1054                 def selector_function(formats):
1055                     formats = list(formats)
1056                     if not formats:
1057                         return
1058                     if format_spec == 'all':
1059                         for f in formats:
1060                             yield f
1061                     elif format_spec in ['best', 'worst', None]:
1062                         format_idx = 0 if format_spec == 'worst' else -1
1063                         audiovideo_formats = [
1064                             f for f in formats
1065                             if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1066                         if audiovideo_formats:
1067                             yield audiovideo_formats[format_idx]
1068                         # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
1069                         elif (all(f.get('acodec') != 'none' for f in formats) or
1070                               all(f.get('vcodec') != 'none' for f in formats)):
1071                             yield formats[format_idx]
1072                     elif format_spec == 'bestaudio':
1073                         audio_formats = [
1074                             f for f in formats
1075                             if f.get('vcodec') == 'none']
1076                         if audio_formats:
1077                             yield audio_formats[-1]
1078                     elif format_spec == 'worstaudio':
1079                         audio_formats = [
1080                             f for f in formats
1081                             if f.get('vcodec') == 'none']
1082                         if audio_formats:
1083                             yield audio_formats[0]
1084                     elif format_spec == 'bestvideo':
1085                         video_formats = [
1086                             f for f in formats
1087                             if f.get('acodec') == 'none']
1088                         if video_formats:
1089                             yield video_formats[-1]
1090                     elif format_spec == 'worstvideo':
1091                         video_formats = [
1092                             f for f in formats
1093                             if f.get('acodec') == 'none']
1094                         if video_formats:
1095                             yield video_formats[0]
1096                     else:
1097                         extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1098                         if format_spec in extensions:
1099                             filter_f = lambda f: f['ext'] == format_spec
1100                         else:
1101                             filter_f = lambda f: f['format_id'] == format_spec
1102                         matches = list(filter(filter_f, formats))
1103                         if matches:
1104                             yield matches[-1]
1105             elif selector.type == MERGE:
1106                 def _merge(formats_info):
1107                     format_1, format_2 = [f['format_id'] for f in formats_info]
1108                     # The first format must contain the video and the
1109                     # second the audio
1110                     if formats_info[0].get('vcodec') == 'none':
1111                         self.report_error('The first format must '
1112                                           'contain the video, try using '
1113                                           '"-f %s+%s"' % (format_2, format_1))
1114                         return
1115                     # Formats must be opposite (video+audio)
1116                     if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1117                         self.report_error(
1118                             'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1119                             % (format_1, format_2))
1120                         return
1121                     output_ext = (
1122                         formats_info[0]['ext']
1123                         if self.params.get('merge_output_format') is None
1124                         else self.params['merge_output_format'])
1125                     return {
1126                         'requested_formats': formats_info,
1127                         'format': '%s+%s' % (formats_info[0].get('format'),
1128                                              formats_info[1].get('format')),
1129                         'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1130                                                 formats_info[1].get('format_id')),
1131                         'width': formats_info[0].get('width'),
1132                         'height': formats_info[0].get('height'),
1133                         'resolution': formats_info[0].get('resolution'),
1134                         'fps': formats_info[0].get('fps'),
1135                         'vcodec': formats_info[0].get('vcodec'),
1136                         'vbr': formats_info[0].get('vbr'),
1137                         'stretched_ratio': formats_info[0].get('stretched_ratio'),
1138                         'acodec': formats_info[1].get('acodec'),
1139                         'abr': formats_info[1].get('abr'),
1140                         'ext': output_ext,
1141                     }
1142                 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1143
1144                 def selector_function(formats):
1145                     formats = list(formats)
1146                     for pair in itertools.product(video_selector(formats), audio_selector(formats)):
1147                         yield _merge(pair)
1148
1149             filters = [self._build_format_filter(f) for f in selector.filters]
1150
1151             def final_selector(formats):
1152                 for _filter in filters:
1153                     formats = list(filter(_filter, formats))
1154                 return selector_function(formats)
1155             return final_selector
1156
1157         stream = io.BytesIO(format_spec.encode('utf-8'))
1158         try:
1159             tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1160         except tokenize.TokenError:
1161             raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1162
1163         class TokenIterator(object):
1164             def __init__(self, tokens):
1165                 self.tokens = tokens
1166                 self.counter = 0
1167
1168             def __iter__(self):
1169                 return self
1170
1171             def __next__(self):
1172                 if self.counter >= len(self.tokens):
1173                     raise StopIteration()
1174                 value = self.tokens[self.counter]
1175                 self.counter += 1
1176                 return value
1177
1178             next = __next__
1179
1180             def restore_last_token(self):
1181                 self.counter -= 1
1182
1183         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1184         return _build_selector_function(parsed_selector)
1185
1186     def _calc_headers(self, info_dict):
1187         res = std_headers.copy()
1188
1189         add_headers = info_dict.get('http_headers')
1190         if add_headers:
1191             res.update(add_headers)
1192
1193         cookies = self._calc_cookies(info_dict)
1194         if cookies:
1195             res['Cookie'] = cookies
1196
1197         return res
1198
1199     def _calc_cookies(self, info_dict):
1200         pr = sanitized_Request(info_dict['url'])
1201         self.cookiejar.add_cookie_header(pr)
1202         return pr.get_header('Cookie')
1203
1204     def process_video_result(self, info_dict, download=True):
1205         assert info_dict.get('_type', 'video') == 'video'
1206
1207         if 'id' not in info_dict:
1208             raise ExtractorError('Missing "id" field in extractor result')
1209         if 'title' not in info_dict:
1210             raise ExtractorError('Missing "title" field in extractor result')
1211
1212         if 'playlist' not in info_dict:
1213             # It isn't part of a playlist
1214             info_dict['playlist'] = None
1215             info_dict['playlist_index'] = None
1216
1217         thumbnails = info_dict.get('thumbnails')
1218         if thumbnails is None:
1219             thumbnail = info_dict.get('thumbnail')
1220             if thumbnail:
1221                 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1222         if thumbnails:
1223             thumbnails.sort(key=lambda t: (
1224                 t.get('preference'), t.get('width'), t.get('height'),
1225                 t.get('id'), t.get('url')))
1226             for i, t in enumerate(thumbnails):
1227                 if t.get('width') and t.get('height'):
1228                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
1229                 if t.get('id') is None:
1230                     t['id'] = '%d' % i
1231
1232         if thumbnails and 'thumbnail' not in info_dict:
1233             info_dict['thumbnail'] = thumbnails[-1]['url']
1234
1235         if 'display_id' not in info_dict and 'id' in info_dict:
1236             info_dict['display_id'] = info_dict['id']
1237
1238         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1239             # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1240             # see http://bugs.python.org/issue1646728)
1241             try:
1242                 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1243                 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1244             except (ValueError, OverflowError, OSError):
1245                 pass
1246
1247         subtitles = info_dict.get('subtitles')
1248         if subtitles:
1249             for _, subtitle in subtitles.items():
1250                 for subtitle_format in subtitle:
1251                     if 'ext' not in subtitle_format:
1252                         subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1253
1254         if self.params.get('listsubtitles', False):
1255             if 'automatic_captions' in info_dict:
1256                 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1257             self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1258             return
1259         info_dict['requested_subtitles'] = self.process_subtitles(
1260             info_dict['id'], subtitles,
1261             info_dict.get('automatic_captions'))
1262
1263         # We now pick which formats have to be downloaded
1264         if info_dict.get('formats') is None:
1265             # There's only one format available
1266             formats = [info_dict]
1267         else:
1268             formats = info_dict['formats']
1269
1270         if not formats:
1271             raise ExtractorError('No video formats found!')
1272
1273         formats_dict = {}
1274
1275         # We check that all the formats have the format and format_id fields
1276         for i, format in enumerate(formats):
1277             if 'url' not in format:
1278                 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1279
1280             if format.get('format_id') is None:
1281                 format['format_id'] = compat_str(i)
1282             format_id = format['format_id']
1283             if format_id not in formats_dict:
1284                 formats_dict[format_id] = []
1285             formats_dict[format_id].append(format)
1286
1287         # Make sure all formats have unique format_id
1288         for format_id, ambiguous_formats in formats_dict.items():
1289             if len(ambiguous_formats) > 1:
1290                 for i, format in enumerate(ambiguous_formats):
1291                     format['format_id'] = '%s-%d' % (format_id, i)
1292
1293         for i, format in enumerate(formats):
1294             if format.get('format') is None:
1295                 format['format'] = '{id} - {res}{note}'.format(
1296                     id=format['format_id'],
1297                     res=self.format_resolution(format),
1298                     note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1299                 )
1300             # Automatically determine file extension if missing
1301             if 'ext' not in format:
1302                 format['ext'] = determine_ext(format['url']).lower()
1303             # Add HTTP headers, so that external programs can use them from the
1304             # json output
1305             full_format_info = info_dict.copy()
1306             full_format_info.update(format)
1307             format['http_headers'] = self._calc_headers(full_format_info)
1308
1309         # TODO Central sorting goes here
1310
1311         if formats[0] is not info_dict:
1312             # only set the 'formats' fields if the original info_dict list them
1313             # otherwise we end up with a circular reference, the first (and unique)
1314             # element in the 'formats' field in info_dict is info_dict itself,
1315             # wich can't be exported to json
1316             info_dict['formats'] = formats
1317         if self.params.get('listformats'):
1318             self.list_formats(info_dict)
1319             return
1320         if self.params.get('list_thumbnails'):
1321             self.list_thumbnails(info_dict)
1322             return
1323
1324         req_format = self.params.get('format')
1325         if req_format is None:
1326             req_format_list = []
1327             if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1328                     info_dict['extractor'] in ['youtube', 'ted'] and
1329                     not info_dict.get('is_live')):
1330                 merger = FFmpegMergerPP(self)
1331                 if merger.available and merger.can_merge():
1332                     req_format_list.append('bestvideo+bestaudio')
1333             req_format_list.append('best')
1334             req_format = '/'.join(req_format_list)
1335         format_selector = self.build_format_selector(req_format)
1336         formats_to_download = list(format_selector(formats))
1337         if not formats_to_download:
1338             raise ExtractorError('requested format not available',
1339                                  expected=True)
1340
1341         if download:
1342             if len(formats_to_download) > 1:
1343                 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1344             for format in formats_to_download:
1345                 new_info = dict(info_dict)
1346                 new_info.update(format)
1347                 self.process_info(new_info)
1348         # We update the info dict with the best quality format (backwards compatibility)
1349         info_dict.update(formats_to_download[-1])
1350         return info_dict
1351
1352     def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1353         """Select the requested subtitles and their format"""
1354         available_subs = {}
1355         if normal_subtitles and self.params.get('writesubtitles'):
1356             available_subs.update(normal_subtitles)
1357         if automatic_captions and self.params.get('writeautomaticsub'):
1358             for lang, cap_info in automatic_captions.items():
1359                 if lang not in available_subs:
1360                     available_subs[lang] = cap_info
1361
1362         if (not self.params.get('writesubtitles') and not
1363                 self.params.get('writeautomaticsub') or not
1364                 available_subs):
1365             return None
1366
1367         if self.params.get('allsubtitles', False):
1368             requested_langs = available_subs.keys()
1369         else:
1370             if self.params.get('subtitleslangs', False):
1371                 requested_langs = self.params.get('subtitleslangs')
1372             elif 'en' in available_subs:
1373                 requested_langs = ['en']
1374             else:
1375                 requested_langs = [list(available_subs.keys())[0]]
1376
1377         formats_query = self.params.get('subtitlesformat', 'best')
1378         formats_preference = formats_query.split('/') if formats_query else []
1379         subs = {}
1380         for lang in requested_langs:
1381             formats = available_subs.get(lang)
1382             if formats is None:
1383                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1384                 continue
1385             for ext in formats_preference:
1386                 if ext == 'best':
1387                     f = formats[-1]
1388                     break
1389                 matches = list(filter(lambda f: f['ext'] == ext, formats))
1390                 if matches:
1391                     f = matches[-1]
1392                     break
1393             else:
1394                 f = formats[-1]
1395                 self.report_warning(
1396                     'No subtitle format found matching "%s" for language %s, '
1397                     'using %s' % (formats_query, lang, f['ext']))
1398             subs[lang] = f
1399         return subs
1400
1401     def process_info(self, info_dict):
1402         """Process a single resolved IE result."""
1403
1404         assert info_dict.get('_type', 'video') == 'video'
1405
1406         max_downloads = self.params.get('max_downloads')
1407         if max_downloads is not None:
1408             if self._num_downloads >= int(max_downloads):
1409                 raise MaxDownloadsReached()
1410
1411         info_dict['fulltitle'] = info_dict['title']
1412         if len(info_dict['title']) > 200:
1413             info_dict['title'] = info_dict['title'][:197] + '...'
1414
1415         if 'format' not in info_dict:
1416             info_dict['format'] = info_dict['ext']
1417
1418         reason = self._match_entry(info_dict, incomplete=False)
1419         if reason is not None:
1420             self.to_screen('[download] ' + reason)
1421             return
1422
1423         self._num_downloads += 1
1424
1425         info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1426
1427         # Forced printings
1428         if self.params.get('forcetitle', False):
1429             self.to_stdout(info_dict['fulltitle'])
1430         if self.params.get('forceid', False):
1431             self.to_stdout(info_dict['id'])
1432         if self.params.get('forceurl', False):
1433             if info_dict.get('requested_formats') is not None:
1434                 for f in info_dict['requested_formats']:
1435                     self.to_stdout(f['url'] + f.get('play_path', ''))
1436             else:
1437                 # For RTMP URLs, also include the playpath
1438                 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1439         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1440             self.to_stdout(info_dict['thumbnail'])
1441         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1442             self.to_stdout(info_dict['description'])
1443         if self.params.get('forcefilename', False) and filename is not None:
1444             self.to_stdout(filename)
1445         if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1446             self.to_stdout(formatSeconds(info_dict['duration']))
1447         if self.params.get('forceformat', False):
1448             self.to_stdout(info_dict['format'])
1449         if self.params.get('forcejson', False):
1450             self.to_stdout(json.dumps(info_dict))
1451
1452         # Do nothing else if in simulate mode
1453         if self.params.get('simulate', False):
1454             return
1455
1456         if filename is None:
1457             return
1458
1459         try:
1460             dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1461             if dn and not os.path.exists(dn):
1462                 os.makedirs(dn)
1463         except (OSError, IOError) as err:
1464             self.report_error('unable to create directory ' + error_to_compat_str(err))
1465             return
1466
1467         if self.params.get('writedescription', False):
1468             descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1469             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1470                 self.to_screen('[info] Video description is already present')
1471             elif info_dict.get('description') is None:
1472                 self.report_warning('There\'s no description to write.')
1473             else:
1474                 try:
1475                     self.to_screen('[info] Writing video description to: ' + descfn)
1476                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1477                         descfile.write(info_dict['description'])
1478                 except (OSError, IOError):
1479                     self.report_error('Cannot write description file ' + descfn)
1480                     return
1481
1482         if self.params.get('writeannotations', False):
1483             annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1484             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1485                 self.to_screen('[info] Video annotations are already present')
1486             else:
1487                 try:
1488                     self.to_screen('[info] Writing video annotations to: ' + annofn)
1489                     with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1490                         annofile.write(info_dict['annotations'])
1491                 except (KeyError, TypeError):
1492                     self.report_warning('There are no annotations to write.')
1493                 except (OSError, IOError):
1494                     self.report_error('Cannot write annotations file: ' + annofn)
1495                     return
1496
1497         subtitles_are_requested = any([self.params.get('writesubtitles', False),
1498                                        self.params.get('writeautomaticsub')])
1499
1500         if subtitles_are_requested and info_dict.get('requested_subtitles'):
1501             # subtitles download errors are already managed as troubles in relevant IE
1502             # that way it will silently go on when used with unsupporting IE
1503             subtitles = info_dict['requested_subtitles']
1504             ie = self.get_info_extractor(info_dict['extractor_key'])
1505             for sub_lang, sub_info in subtitles.items():
1506                 sub_format = sub_info['ext']
1507                 if sub_info.get('data') is not None:
1508                     sub_data = sub_info['data']
1509                 else:
1510                     try:
1511                         sub_data = ie._download_webpage(
1512                             sub_info['url'], info_dict['id'], note=False)
1513                     except ExtractorError as err:
1514                         self.report_warning('Unable to download subtitle for "%s": %s' %
1515                                             (sub_lang, error_to_compat_str(err.cause)))
1516                         continue
1517                 try:
1518                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1519                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1520                         self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1521                     else:
1522                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1523                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1524                             subfile.write(sub_data)
1525                 except (OSError, IOError):
1526                     self.report_error('Cannot write subtitles file ' + sub_filename)
1527                     return
1528
1529         if self.params.get('writeinfojson', False):
1530             infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1531             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1532                 self.to_screen('[info] Video description metadata is already present')
1533             else:
1534                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1535                 try:
1536                     write_json_file(self.filter_requested_info(info_dict), infofn)
1537                 except (OSError, IOError):
1538                     self.report_error('Cannot write metadata to JSON file ' + infofn)
1539                     return
1540
1541         self._write_thumbnails(info_dict, filename)
1542
1543         if not self.params.get('skip_download', False):
1544             try:
1545                 def dl(name, info):
1546                     fd = get_suitable_downloader(info, self.params)(self, self.params)
1547                     for ph in self._progress_hooks:
1548                         fd.add_progress_hook(ph)
1549                     if self.params.get('verbose'):
1550                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1551                     return fd.download(name, info)
1552
1553                 if info_dict.get('requested_formats') is not None:
1554                     downloaded = []
1555                     success = True
1556                     merger = FFmpegMergerPP(self)
1557                     if not merger.available:
1558                         postprocessors = []
1559                         self.report_warning('You have requested multiple '
1560                                             'formats but ffmpeg or avconv are not installed.'
1561                                             ' The formats won\'t be merged.')
1562                     else:
1563                         postprocessors = [merger]
1564
1565                     def compatible_formats(formats):
1566                         video, audio = formats
1567                         # Check extension
1568                         video_ext, audio_ext = audio.get('ext'), video.get('ext')
1569                         if video_ext and audio_ext:
1570                             COMPATIBLE_EXTS = (
1571                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1572                                 ('webm')
1573                             )
1574                             for exts in COMPATIBLE_EXTS:
1575                                 if video_ext in exts and audio_ext in exts:
1576                                     return True
1577                         # TODO: Check acodec/vcodec
1578                         return False
1579
1580                     filename_real_ext = os.path.splitext(filename)[1][1:]
1581                     filename_wo_ext = (
1582                         os.path.splitext(filename)[0]
1583                         if filename_real_ext == info_dict['ext']
1584                         else filename)
1585                     requested_formats = info_dict['requested_formats']
1586                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1587                         info_dict['ext'] = 'mkv'
1588                         self.report_warning(
1589                             'Requested formats are incompatible for merge and will be merged into mkv.')
1590                     # Ensure filename always has a correct extension for successful merge
1591                     filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1592                     if os.path.exists(encodeFilename(filename)):
1593                         self.to_screen(
1594                             '[download] %s has already been downloaded and '
1595                             'merged' % filename)
1596                     else:
1597                         for f in requested_formats:
1598                             new_info = dict(info_dict)
1599                             new_info.update(f)
1600                             fname = self.prepare_filename(new_info)
1601                             fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1602                             downloaded.append(fname)
1603                             partial_success = dl(fname, new_info)
1604                             success = success and partial_success
1605                         info_dict['__postprocessors'] = postprocessors
1606                         info_dict['__files_to_merge'] = downloaded
1607                 else:
1608                     # Just a single file
1609                     success = dl(filename, info_dict)
1610             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1611                 self.report_error('unable to download video data: %s' % str(err))
1612                 return
1613             except (OSError, IOError) as err:
1614                 raise UnavailableVideoError(err)
1615             except (ContentTooShortError, ) as err:
1616                 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1617                 return
1618
1619             if success:
1620                 # Fixup content
1621                 fixup_policy = self.params.get('fixup')
1622                 if fixup_policy is None:
1623                     fixup_policy = 'detect_or_warn'
1624
1625                 stretched_ratio = info_dict.get('stretched_ratio')
1626                 if stretched_ratio is not None and stretched_ratio != 1:
1627                     if fixup_policy == 'warn':
1628                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1629                             info_dict['id'], stretched_ratio))
1630                     elif fixup_policy == 'detect_or_warn':
1631                         stretched_pp = FFmpegFixupStretchedPP(self)
1632                         if stretched_pp.available:
1633                             info_dict.setdefault('__postprocessors', [])
1634                             info_dict['__postprocessors'].append(stretched_pp)
1635                         else:
1636                             self.report_warning(
1637                                 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1638                                     info_dict['id'], stretched_ratio))
1639                     else:
1640                         assert fixup_policy in ('ignore', 'never')
1641
1642                 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1643                     if fixup_policy == 'warn':
1644                         self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1645                             info_dict['id']))
1646                     elif fixup_policy == 'detect_or_warn':
1647                         fixup_pp = FFmpegFixupM4aPP(self)
1648                         if fixup_pp.available:
1649                             info_dict.setdefault('__postprocessors', [])
1650                             info_dict['__postprocessors'].append(fixup_pp)
1651                         else:
1652                             self.report_warning(
1653                                 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1654                                     info_dict['id']))
1655                     else:
1656                         assert fixup_policy in ('ignore', 'never')
1657
1658                 try:
1659                     self.post_process(filename, info_dict)
1660                 except (PostProcessingError) as err:
1661                     self.report_error('postprocessing: %s' % str(err))
1662                     return
1663                 self.record_download_archive(info_dict)
1664
1665     def download(self, url_list):
1666         """Download a given list of URLs."""
1667         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1668         if (len(url_list) > 1 and
1669                 '%' not in outtmpl and
1670                 self.params.get('max_downloads') != 1):
1671             raise SameFileError(outtmpl)
1672
1673         for url in url_list:
1674             try:
1675                 # It also downloads the videos
1676                 res = self.extract_info(
1677                     url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1678             except UnavailableVideoError:
1679                 self.report_error('unable to download video')
1680             except MaxDownloadsReached:
1681                 self.to_screen('[info] Maximum number of downloaded files reached.')
1682                 raise
1683             else:
1684                 if self.params.get('dump_single_json', False):
1685                     self.to_stdout(json.dumps(res))
1686
1687         return self._download_retcode
1688
1689     def download_with_info_file(self, info_filename):
1690         with contextlib.closing(fileinput.FileInput(
1691                 [info_filename], mode='r',
1692                 openhook=fileinput.hook_encoded('utf-8'))) as f:
1693             # FileInput doesn't have a read method, we can't call json.load
1694             info = self.filter_requested_info(json.loads('\n'.join(f)))
1695         try:
1696             self.process_ie_result(info, download=True)
1697         except DownloadError:
1698             webpage_url = info.get('webpage_url')
1699             if webpage_url is not None:
1700                 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1701                 return self.download([webpage_url])
1702             else:
1703                 raise
1704         return self._download_retcode
1705
1706     @staticmethod
1707     def filter_requested_info(info_dict):
1708         return dict(
1709             (k, v) for k, v in info_dict.items()
1710             if k not in ['requested_formats', 'requested_subtitles'])
1711
1712     def post_process(self, filename, ie_info):
1713         """Run all the postprocessors on the given file."""
1714         info = dict(ie_info)
1715         info['filepath'] = filename
1716         pps_chain = []
1717         if ie_info.get('__postprocessors') is not None:
1718             pps_chain.extend(ie_info['__postprocessors'])
1719         pps_chain.extend(self._pps)
1720         for pp in pps_chain:
1721             files_to_delete = []
1722             try:
1723                 files_to_delete, info = pp.run(info)
1724             except PostProcessingError as e:
1725                 self.report_error(e.msg)
1726             if files_to_delete and not self.params.get('keepvideo', False):
1727                 for old_filename in files_to_delete:
1728                     self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1729                     try:
1730                         os.remove(encodeFilename(old_filename))
1731                     except (IOError, OSError):
1732                         self.report_warning('Unable to remove downloaded original file')
1733
1734     def _make_archive_id(self, info_dict):
1735         # Future-proof against any change in case
1736         # and backwards compatibility with prior versions
1737         extractor = info_dict.get('extractor_key')
1738         if extractor is None:
1739             if 'id' in info_dict:
1740                 extractor = info_dict.get('ie_key')  # key in a playlist
1741         if extractor is None:
1742             return None  # Incomplete video information
1743         return extractor.lower() + ' ' + info_dict['id']
1744
1745     def in_download_archive(self, info_dict):
1746         fn = self.params.get('download_archive')
1747         if fn is None:
1748             return False
1749
1750         vid_id = self._make_archive_id(info_dict)
1751         if vid_id is None:
1752             return False  # Incomplete video information
1753
1754         try:
1755             with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1756                 for line in archive_file:
1757                     if line.strip() == vid_id:
1758                         return True
1759         except IOError as ioe:
1760             if ioe.errno != errno.ENOENT:
1761                 raise
1762         return False
1763
1764     def record_download_archive(self, info_dict):
1765         fn = self.params.get('download_archive')
1766         if fn is None:
1767             return
1768         vid_id = self._make_archive_id(info_dict)
1769         assert vid_id
1770         with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1771             archive_file.write(vid_id + '\n')
1772
1773     @staticmethod
1774     def format_resolution(format, default='unknown'):
1775         if format.get('vcodec') == 'none':
1776             return 'audio only'
1777         if format.get('resolution') is not None:
1778             return format['resolution']
1779         if format.get('height') is not None:
1780             if format.get('width') is not None:
1781                 res = '%sx%s' % (format['width'], format['height'])
1782             else:
1783                 res = '%sp' % format['height']
1784         elif format.get('width') is not None:
1785             res = '?x%d' % format['width']
1786         else:
1787             res = default
1788         return res
1789
1790     def _format_note(self, fdict):
1791         res = ''
1792         if fdict.get('ext') in ['f4f', 'f4m']:
1793             res += '(unsupported) '
1794         if fdict.get('format_note') is not None:
1795             res += fdict['format_note'] + ' '
1796         if fdict.get('tbr') is not None:
1797             res += '%4dk ' % fdict['tbr']
1798         if fdict.get('container') is not None:
1799             if res:
1800                 res += ', '
1801             res += '%s container' % fdict['container']
1802         if (fdict.get('vcodec') is not None and
1803                 fdict.get('vcodec') != 'none'):
1804             if res:
1805                 res += ', '
1806             res += fdict['vcodec']
1807             if fdict.get('vbr') is not None:
1808                 res += '@'
1809         elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1810             res += 'video@'
1811         if fdict.get('vbr') is not None:
1812             res += '%4dk' % fdict['vbr']
1813         if fdict.get('fps') is not None:
1814             res += ', %sfps' % fdict['fps']
1815         if fdict.get('acodec') is not None:
1816             if res:
1817                 res += ', '
1818             if fdict['acodec'] == 'none':
1819                 res += 'video only'
1820             else:
1821                 res += '%-5s' % fdict['acodec']
1822         elif fdict.get('abr') is not None:
1823             if res:
1824                 res += ', '
1825             res += 'audio'
1826         if fdict.get('abr') is not None:
1827             res += '@%3dk' % fdict['abr']
1828         if fdict.get('asr') is not None:
1829             res += ' (%5dHz)' % fdict['asr']
1830         if fdict.get('filesize') is not None:
1831             if res:
1832                 res += ', '
1833             res += format_bytes(fdict['filesize'])
1834         elif fdict.get('filesize_approx') is not None:
1835             if res:
1836                 res += ', '
1837             res += '~' + format_bytes(fdict['filesize_approx'])
1838         return res
1839
1840     def list_formats(self, info_dict):
1841         formats = info_dict.get('formats', [info_dict])
1842         table = [
1843             [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1844             for f in formats
1845             if f.get('preference') is None or f['preference'] >= -1000]
1846         if len(formats) > 1:
1847             table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1848
1849         header_line = ['format code', 'extension', 'resolution', 'note']
1850         self.to_screen(
1851             '[info] Available formats for %s:\n%s' %
1852             (info_dict['id'], render_table(header_line, table)))
1853
1854     def list_thumbnails(self, info_dict):
1855         thumbnails = info_dict.get('thumbnails')
1856         if not thumbnails:
1857             tn_url = info_dict.get('thumbnail')
1858             if tn_url:
1859                 thumbnails = [{'id': '0', 'url': tn_url}]
1860             else:
1861                 self.to_screen(
1862                     '[info] No thumbnails present for %s' % info_dict['id'])
1863                 return
1864
1865         self.to_screen(
1866             '[info] Thumbnails for %s:' % info_dict['id'])
1867         self.to_screen(render_table(
1868             ['ID', 'width', 'height', 'URL'],
1869             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1870
1871     def list_subtitles(self, video_id, subtitles, name='subtitles'):
1872         if not subtitles:
1873             self.to_screen('%s has no %s' % (video_id, name))
1874             return
1875         self.to_screen(
1876             'Available %s for %s:' % (name, video_id))
1877         self.to_screen(render_table(
1878             ['Language', 'formats'],
1879             [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1880                 for lang, formats in subtitles.items()]))
1881
1882     def urlopen(self, req):
1883         """ Start an HTTP download """
1884         if isinstance(req, compat_basestring):
1885             req = sanitized_Request(req)
1886         return self._opener.open(req, timeout=self._socket_timeout)
1887
1888     def print_debug_header(self):
1889         if not self.params.get('verbose'):
1890             return
1891
1892         if type('') is not compat_str:
1893             # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1894             self.report_warning(
1895                 'Your Python is broken! Update to a newer and supported version')
1896
1897         stdout_encoding = getattr(
1898             sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1899         encoding_str = (
1900             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1901                 locale.getpreferredencoding(),
1902                 sys.getfilesystemencoding(),
1903                 stdout_encoding,
1904                 self.get_encoding()))
1905         write_string(encoding_str, encoding=None)
1906
1907         self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1908         try:
1909             sp = subprocess.Popen(
1910                 ['git', 'rev-parse', '--short', 'HEAD'],
1911                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1912                 cwd=os.path.dirname(os.path.abspath(__file__)))
1913             out, err = sp.communicate()
1914             out = out.decode().strip()
1915             if re.match('[0-9a-f]+', out):
1916                 self._write_string('[debug] Git HEAD: ' + out + '\n')
1917         except Exception:
1918             try:
1919                 sys.exc_clear()
1920             except Exception:
1921                 pass
1922         self._write_string('[debug] Python version %s - %s\n' % (
1923             platform.python_version(), platform_name()))
1924
1925         exe_versions = FFmpegPostProcessor.get_versions(self)
1926         exe_versions['rtmpdump'] = rtmpdump_version()
1927         exe_str = ', '.join(
1928             '%s %s' % (exe, v)
1929             for exe, v in sorted(exe_versions.items())
1930             if v
1931         )
1932         if not exe_str:
1933             exe_str = 'none'
1934         self._write_string('[debug] exe versions: %s\n' % exe_str)
1935
1936         proxy_map = {}
1937         for handler in self._opener.handlers:
1938             if hasattr(handler, 'proxies'):
1939                 proxy_map.update(handler.proxies)
1940         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1941
1942         if self.params.get('call_home', False):
1943             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1944             self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1945             latest_version = self.urlopen(
1946                 'https://yt-dl.org/latest/version').read().decode('utf-8')
1947             if version_tuple(latest_version) > version_tuple(__version__):
1948                 self.report_warning(
1949                     'You are using an outdated version (newest version: %s)! '
1950                     'See https://yt-dl.org/update if you need help updating.' %
1951                     latest_version)
1952
1953     def _setup_opener(self):
1954         timeout_val = self.params.get('socket_timeout')
1955         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1956
1957         opts_cookiefile = self.params.get('cookiefile')
1958         opts_proxy = self.params.get('proxy')
1959
1960         if opts_cookiefile is None:
1961             self.cookiejar = compat_cookiejar.CookieJar()
1962         else:
1963             self.cookiejar = compat_cookiejar.MozillaCookieJar(
1964                 opts_cookiefile)
1965             if os.access(opts_cookiefile, os.R_OK):
1966                 self.cookiejar.load()
1967
1968         cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
1969         if opts_proxy is not None:
1970             if opts_proxy == '':
1971                 proxies = {}
1972             else:
1973                 proxies = {'http': opts_proxy, 'https': opts_proxy}
1974         else:
1975             proxies = compat_urllib_request.getproxies()
1976             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1977             if 'http' in proxies and 'https' not in proxies:
1978                 proxies['https'] = proxies['http']
1979         proxy_handler = PerRequestProxyHandler(proxies)
1980
1981         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1982         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1983         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1984         data_handler = compat_urllib_request_DataHandler()
1985         opener = compat_urllib_request.build_opener(
1986             proxy_handler, https_handler, cookie_processor, ydlh, data_handler)
1987
1988         # Delete the default user-agent header, which would otherwise apply in
1989         # cases where our custom HTTP handler doesn't come into play
1990         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1991         opener.addheaders = []
1992         self._opener = opener
1993
1994     def encode(self, s):
1995         if isinstance(s, bytes):
1996             return s  # Already encoded
1997
1998         try:
1999             return s.encode(self.get_encoding())
2000         except UnicodeEncodeError as err:
2001             err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2002             raise
2003
2004     def get_encoding(self):
2005         encoding = self.params.get('encoding')
2006         if encoding is None:
2007             encoding = preferredencoding()
2008         return encoding
2009
2010     def _write_thumbnails(self, info_dict, filename):
2011         if self.params.get('writethumbnail', False):
2012             thumbnails = info_dict.get('thumbnails')
2013             if thumbnails:
2014                 thumbnails = [thumbnails[-1]]
2015         elif self.params.get('write_all_thumbnails', False):
2016             thumbnails = info_dict.get('thumbnails')
2017         else:
2018             return
2019
2020         if not thumbnails:
2021             # No thumbnails present, so return immediately
2022             return
2023
2024         for t in thumbnails:
2025             thumb_ext = determine_ext(t['url'], 'jpg')
2026             suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2027             thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2028             t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2029
2030             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2031                 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2032                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2033             else:
2034                 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2035                                (info_dict['extractor'], info_dict['id'], thumb_display_id))
2036                 try:
2037                     uf = self.urlopen(t['url'])
2038                     with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2039                         shutil.copyfileobj(uf, thumbf)
2040                     self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2041                                    (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2042                 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2043                     self.report_warning('Unable to download thumbnail "%s": %s' %
2044                                         (t['url'], error_to_compat_str(err)))