4 from __future__ import absolute_import, unicode_literals
29 from string import ascii_letters
34 compat_get_terminal_size,
40 compat_tokenize_tokenize,
42 compat_urllib_request,
43 compat_urllib_request_DataHandler,
71 PerRequestProxyHandler,
76 register_socks_protocols,
86 UnavailableVideoError,
91 YoutubeDLCookieProcessor,
94 from .cache import Cache
95 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
96 from .extractor.openload import PhantomJSwrapper
97 from .downloader import get_suitable_downloader
98 from .downloader.rtmp import rtmpdump_version
99 from .postprocessor import (
102 FFmpegFixupStretchedPP,
107 from .version import __version__
109 if compat_os_name == 'nt':
113 class YoutubeDL(object):
116 YoutubeDL objects are the ones responsible of downloading the
117 actual video file and writing it to disk if the user has requested
118 it, among some other tasks. In most cases there should be one per
119 program. As, given a video URL, the downloader doesn't know how to
120 extract all the needed information, task that InfoExtractors do, it
121 has to pass the URL to one of them.
123 For this, YoutubeDL objects have a method that allows
124 InfoExtractors to be registered in a given order. When it is passed
125 a URL, the YoutubeDL object handles it to the first InfoExtractor it
126 finds that reports being able to handle it. The InfoExtractor extracts
127 all the information about the video or videos the URL refers to, and
128 YoutubeDL process the extracted information, possibly using a File
129 Downloader to download the video.
131 YoutubeDL objects accept a lot of parameters. In order not to saturate
132 the object constructor with arguments, it receives a dictionary of
133 options instead. These options are available through the params
134 attribute for the InfoExtractors to use. The YoutubeDL also
135 registers itself as the downloader in charge for the InfoExtractors
136 that are added to it, so this is a "mutual registration".
140 username: Username for authentication purposes.
141 password: Password for authentication purposes.
142 videopassword: Password for accessing a video.
143 ap_mso: Adobe Pass multiple-system operator identifier.
144 ap_username: Multiple-system operator account username.
145 ap_password: Multiple-system operator account password.
146 usenetrc: Use netrc for authentication instead.
147 verbose: Print additional info to stdout.
148 quiet: Do not print messages to stdout.
149 no_warnings: Do not print out anything for warnings.
150 forceurl: Force printing final URL.
151 forcetitle: Force printing title.
152 forceid: Force printing ID.
153 forcethumbnail: Force printing thumbnail URL.
154 forcedescription: Force printing description.
155 forcefilename: Force printing final filename.
156 forceduration: Force printing duration.
157 forcejson: Force printing info_dict as JSON.
158 dump_single_json: Force printing the info_dict of the whole playlist
159 (or video) as a single JSON line.
160 simulate: Do not download the video files.
161 format: Video format code. See options.py for more information.
162 outtmpl: Template for output names.
163 restrictfilenames: Do not allow "&" and spaces in file names
164 ignoreerrors: Do not stop on download errors.
165 force_generic_extractor: Force downloader to use the generic extractor
166 nooverwrites: Prevent overwriting files.
167 playliststart: Playlist item to start at.
168 playlistend: Playlist item to end at.
169 playlist_items: Specific indices of playlist to download.
170 playlistreverse: Download playlist items in reverse order.
171 playlistrandom: Download playlist items in random order.
172 matchtitle: Download only matching titles.
173 rejecttitle: Reject downloads for matching titles.
174 logger: Log messages to a logging.Logger instance.
175 logtostderr: Log messages to stderr instead of stdout.
176 writedescription: Write the video description to a .description file
177 writeinfojson: Write the video description to a .info.json file
178 writeannotations: Write the video annotations to a .annotations.xml file
179 writethumbnail: Write the thumbnail image to a file
180 write_all_thumbnails: Write all thumbnail formats to files
181 writesubtitles: Write the video subtitles to a file
182 writeautomaticsub: Write the automatically generated subtitles to a file
183 allsubtitles: Downloads all the subtitles of the video
184 (requires writesubtitles or writeautomaticsub)
185 listsubtitles: Lists all available subtitles for the video
186 subtitlesformat: The format code for subtitles
187 subtitleslangs: List of languages of the subtitles to download
188 keepvideo: Keep the video file after post-processing
189 daterange: A DateRange object, download only if the upload_date is in the range.
190 skip_download: Skip the actual download of the video file
191 cachedir: Location of the cache files in the filesystem.
192 False to disable filesystem cache.
193 noplaylist: Download single video instead of a playlist if in doubt.
194 age_limit: An integer representing the user's age in years.
195 Unsuitable videos for the given age are skipped.
196 min_views: An integer representing the minimum view count the video
197 must have in order to not be skipped.
198 Videos without view count information are always
199 downloaded. None for no limit.
200 max_views: An integer representing the maximum view count.
201 Videos that are more popular than that are not
203 Videos without view count information are always
204 downloaded. None for no limit.
205 download_archive: File name of a file where all downloads are recorded.
206 Videos already present in the file are not downloaded
208 cookiefile: File name where cookies should be read from and dumped to.
209 nocheckcertificate:Do not verify SSL certificates
210 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
211 At the moment, this is only supported by YouTube.
212 proxy: URL of the proxy server to use
213 geo_verification_proxy: URL of the proxy to use for IP address verification
214 on geo-restricted sites. (Experimental)
215 socket_timeout: Time to wait for unresponsive hosts, in seconds
216 bidi_workaround: Work around buggy terminals without bidirectional text
217 support, using fridibi
218 debug_printtraffic:Print out sent and received HTTP traffic
219 include_ads: Download ads as well
220 default_search: Prepend this string if an input url is not valid.
221 'auto' for elaborate guessing
222 encoding: Use this encoding instead of the system-specified.
223 extract_flat: Do not resolve URLs, return the immediate result.
224 Pass in 'in_playlist' to only show this behavior for
226 postprocessors: A list of dictionaries, each with an entry
227 * key: The name of the postprocessor. See
228 youtube_dl/postprocessor/__init__.py for a list.
229 as well as any further keyword arguments for the
231 progress_hooks: A list of functions that get called on download
232 progress, with a dictionary with the entries
233 * status: One of "downloading", "error", or "finished".
234 Check this first and ignore unknown values.
236 If status is one of "downloading", or "finished", the
237 following properties may also be present:
238 * filename: The final filename (always present)
239 * tmpfilename: The filename we're currently writing to
240 * downloaded_bytes: Bytes on disk
241 * total_bytes: Size of the whole file, None if unknown
242 * total_bytes_estimate: Guess of the eventual file size,
244 * elapsed: The number of seconds since download started.
245 * eta: The estimated time in seconds, None if unknown
246 * speed: The download speed in bytes/second, None if
248 * fragment_index: The counter of the currently
249 downloaded video fragment.
250 * fragment_count: The number of fragments (= individual
251 files that will be merged)
253 Progress hooks are guaranteed to be called at least once
254 (with status "finished") if the download is successful.
255 merge_output_format: Extension to use when merging formats.
256 fixup: Automatically correct known faults of the file.
258 - "never": do nothing
259 - "warn": only emit a warning
260 - "detect_or_warn": check whether we can do anything
261 about it, warn otherwise (default)
262 source_address: (Experimental) Client-side IP address to bind to.
263 call_home: Boolean, true iff we are allowed to contact the
264 youtube-dl servers for debugging.
265 sleep_interval: Number of seconds to sleep before each download when
266 used alone or a lower bound of a range for randomized
267 sleep before each download (minimum possible number
268 of seconds to sleep) when used along with
270 max_sleep_interval:Upper bound of a range for randomized sleep before each
271 download (maximum possible number of seconds to sleep).
272 Must only be used along with sleep_interval.
273 Actual sleep time will be a random float from range
274 [sleep_interval; max_sleep_interval].
275 listformats: Print an overview of available video formats and exit.
276 list_thumbnails: Print a table of all thumbnails and exit.
277 match_filter: A function that gets called with the info_dict of
279 If it returns a message, the video is ignored.
280 If it returns None, the video is downloaded.
281 match_filter_func in utils.py is one example for this.
282 no_color: Do not emit color codes in output.
283 geo_bypass: Bypass geographic restriction via faking X-Forwarded-For
284 HTTP header (experimental)
286 Two-letter ISO 3166-2 country code that will be used for
287 explicit geographic restriction bypassing via faking
288 X-Forwarded-For HTTP header (experimental)
290 The following options determine which downloader is picked:
291 external_downloader: Executable of the external downloader to call.
292 None or unset for standard (built-in) downloader.
293 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
294 if True, otherwise use ffmpeg/avconv if False, otherwise
295 use downloader suggested by extractor if None.
297 The following parameters are not used by YoutubeDL itself, they are used by
298 the downloader (see youtube_dl/downloader/common.py):
299 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
300 noresizebuffer, retries, continuedl, noprogress, consoletitle,
301 xattr_set_filesize, external_downloader_args, hls_use_mpegts,
304 The following options are used by the post processors:
305 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
306 otherwise prefer avconv.
307 postprocessor_args: A list of additional command-line arguments for the
310 The following options are used by the Youtube extractor:
311 youtube_include_dash_manifest: If True (default), DASH manifests and related
312 data will be downloaded and processed by extractor.
313 You can reduce network I/O by disabling it if you don't
317 _NUMERIC_FIELDS = set((
318 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx',
319 'timestamp', 'upload_year', 'upload_month', 'upload_day',
320 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
321 'average_rating', 'comment_count', 'age_limit',
322 'start_time', 'end_time',
323 'chapter_number', 'season_number', 'episode_number',
324 'track_number', 'disc_number', 'release_year',
331 _download_retcode = None
332 _num_downloads = None
335 def __init__(self, params=None, auto_init=True):
336 """Create a FileDownloader object with the given options."""
340 self._ies_instances = {}
342 self._progress_hooks = []
343 self._download_retcode = 0
344 self._num_downloads = 0
345 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
346 self._err_file = sys.stderr
349 'nocheckcertificate': False,
351 self.params.update(params)
352 self.cache = Cache(self)
354 def check_deprecated(param, option, suggestion):
355 if self.params.get(param) is not None:
357 '%s is deprecated. Use %s instead.' % (option, suggestion))
361 if check_deprecated('cn_verification_proxy', '--cn-verification-proxy', '--geo-verification-proxy'):
362 if self.params.get('geo_verification_proxy') is None:
363 self.params['geo_verification_proxy'] = self.params['cn_verification_proxy']
365 check_deprecated('autonumber_size', '--autonumber-size', 'output template with %(autonumber)0Nd, where N in the number of digits')
366 check_deprecated('autonumber', '--auto-number', '-o "%(autonumber)s-%(title)s.%(ext)s"')
367 check_deprecated('usetitle', '--title', '-o "%(title)s-%(id)s.%(ext)s"')
369 if params.get('bidi_workaround', False):
372 master, slave = pty.openpty()
373 width = compat_get_terminal_size().columns
377 width_args = ['-w', str(width)]
379 stdin=subprocess.PIPE,
381 stderr=self._err_file)
383 self._output_process = subprocess.Popen(
384 ['bidiv'] + width_args, **sp_kwargs
387 self._output_process = subprocess.Popen(
388 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
389 self._output_channel = os.fdopen(master, 'rb')
390 except OSError as ose:
391 if ose.errno == errno.ENOENT:
392 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
396 if (sys.platform != 'win32' and
397 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
398 not params.get('restrictfilenames', False)):
399 # Unicode filesystem API will throw errors (#1474, #13027)
401 'Assuming --restrict-filenames since file system encoding '
402 'cannot encode all characters. '
403 'Set the LC_ALL environment variable to fix this.')
404 self.params['restrictfilenames'] = True
406 if isinstance(params.get('outtmpl'), bytes):
408 'Parameter outtmpl is bytes, but should be a unicode string. '
409 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
414 self.print_debug_header()
415 self.add_default_info_extractors()
417 for pp_def_raw in self.params.get('postprocessors', []):
418 pp_class = get_postprocessor(pp_def_raw['key'])
419 pp_def = dict(pp_def_raw)
421 pp = pp_class(self, **compat_kwargs(pp_def))
422 self.add_post_processor(pp)
424 for ph in self.params.get('progress_hooks', []):
425 self.add_progress_hook(ph)
427 register_socks_protocols()
429 def warn_if_short_id(self, argv):
430 # short YouTube ID starting with dash?
432 i for i, a in enumerate(argv)
433 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
437 [a for i, a in enumerate(argv) if i not in idxs] +
438 ['--'] + [argv[i] for i in idxs]
441 'Long argument string detected. '
442 'Use -- to separate parameters and URLs, like this:\n%s\n' %
443 args_to_str(correct_argv))
445 def add_info_extractor(self, ie):
446 """Add an InfoExtractor object to the end of the list."""
448 if not isinstance(ie, type):
449 self._ies_instances[ie.ie_key()] = ie
450 ie.set_downloader(self)
452 def get_info_extractor(self, ie_key):
454 Get an instance of an IE with name ie_key, it will try to get one from
455 the _ies list, if there's no instance it will create a new one and add
456 it to the extractor list.
458 ie = self._ies_instances.get(ie_key)
460 ie = get_info_extractor(ie_key)()
461 self.add_info_extractor(ie)
464 def add_default_info_extractors(self):
466 Add the InfoExtractors returned by gen_extractors to the end of the list
468 for ie in gen_extractor_classes():
469 self.add_info_extractor(ie)
471 def add_post_processor(self, pp):
472 """Add a PostProcessor object to the end of the chain."""
474 pp.set_downloader(self)
476 def add_progress_hook(self, ph):
477 """Add the progress hook (currently only for the file downloader)"""
478 self._progress_hooks.append(ph)
480 def _bidi_workaround(self, message):
481 if not hasattr(self, '_output_channel'):
484 assert hasattr(self, '_output_process')
485 assert isinstance(message, compat_str)
486 line_count = message.count('\n') + 1
487 self._output_process.stdin.write((message + '\n').encode('utf-8'))
488 self._output_process.stdin.flush()
489 res = ''.join(self._output_channel.readline().decode('utf-8')
490 for _ in range(line_count))
491 return res[:-len('\n')]
493 def to_screen(self, message, skip_eol=False):
494 """Print message to stdout if not in quiet mode."""
495 return self.to_stdout(message, skip_eol, check_quiet=True)
497 def _write_string(self, s, out=None):
498 write_string(s, out=out, encoding=self.params.get('encoding'))
500 def to_stdout(self, message, skip_eol=False, check_quiet=False):
501 """Print message to stdout if not in quiet mode."""
502 if self.params.get('logger'):
503 self.params['logger'].debug(message)
504 elif not check_quiet or not self.params.get('quiet', False):
505 message = self._bidi_workaround(message)
506 terminator = ['\n', ''][skip_eol]
507 output = message + terminator
509 self._write_string(output, self._screen_file)
511 def to_stderr(self, message):
512 """Print message to stderr."""
513 assert isinstance(message, compat_str)
514 if self.params.get('logger'):
515 self.params['logger'].error(message)
517 message = self._bidi_workaround(message)
518 output = message + '\n'
519 self._write_string(output, self._err_file)
521 def to_console_title(self, message):
522 if not self.params.get('consoletitle', False):
524 if compat_os_name == 'nt':
525 if ctypes.windll.kernel32.GetConsoleWindow():
526 # c_wchar_p() might not be necessary if `message` is
527 # already of type unicode()
528 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
529 elif 'TERM' in os.environ:
530 self._write_string('\033]0;%s\007' % message, self._screen_file)
532 def save_console_title(self):
533 if not self.params.get('consoletitle', False):
535 if compat_os_name != 'nt' and 'TERM' in os.environ:
536 # Save the title on stack
537 self._write_string('\033[22;0t', self._screen_file)
539 def restore_console_title(self):
540 if not self.params.get('consoletitle', False):
542 if compat_os_name != 'nt' and 'TERM' in os.environ:
543 # Restore the title from stack
544 self._write_string('\033[23;0t', self._screen_file)
547 self.save_console_title()
550 def __exit__(self, *args):
551 self.restore_console_title()
553 if self.params.get('cookiefile') is not None:
554 self.cookiejar.save()
556 def trouble(self, message=None, tb=None):
557 """Determine action to take when a download problem appears.
559 Depending on if the downloader has been configured to ignore
560 download errors or not, this method may throw an exception or
561 not when errors are found, after printing the message.
563 tb, if given, is additional traceback information.
565 if message is not None:
566 self.to_stderr(message)
567 if self.params.get('verbose'):
569 if sys.exc_info()[0]: # if .trouble has been called from an except block
571 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
572 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
573 tb += encode_compat_str(traceback.format_exc())
575 tb_data = traceback.format_list(traceback.extract_stack())
576 tb = ''.join(tb_data)
578 if not self.params.get('ignoreerrors', False):
579 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
580 exc_info = sys.exc_info()[1].exc_info
582 exc_info = sys.exc_info()
583 raise DownloadError(message, exc_info)
584 self._download_retcode = 1
586 def report_warning(self, message):
588 Print the message to stderr, it will be prefixed with 'WARNING:'
589 If stderr is a tty file the 'WARNING:' will be colored
591 if self.params.get('logger') is not None:
592 self.params['logger'].warning(message)
594 if self.params.get('no_warnings'):
596 if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
597 _msg_header = '\033[0;33mWARNING:\033[0m'
599 _msg_header = 'WARNING:'
600 warning_message = '%s %s' % (_msg_header, message)
601 self.to_stderr(warning_message)
603 def report_error(self, message, tb=None):
605 Do the same as trouble, but prefixes the message with 'ERROR:', colored
606 in red if stderr is a tty file.
608 if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
609 _msg_header = '\033[0;31mERROR:\033[0m'
611 _msg_header = 'ERROR:'
612 error_message = '%s %s' % (_msg_header, message)
613 self.trouble(error_message, tb)
615 def report_file_already_downloaded(self, file_name):
616 """Report file has already been fully downloaded."""
618 self.to_screen('[download] %s has already been downloaded' % file_name)
619 except UnicodeEncodeError:
620 self.to_screen('[download] The file has already been downloaded')
622 def prepare_filename(self, info_dict):
623 """Generate the output filename."""
625 template_dict = dict(info_dict)
627 template_dict['epoch'] = int(time.time())
628 autonumber_size = self.params.get('autonumber_size')
629 if autonumber_size is None:
631 template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
632 if template_dict.get('resolution') is None:
633 if template_dict.get('width') and template_dict.get('height'):
634 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
635 elif template_dict.get('height'):
636 template_dict['resolution'] = '%sp' % template_dict['height']
637 elif template_dict.get('width'):
638 template_dict['resolution'] = '%dx?' % template_dict['width']
640 sanitize = lambda k, v: sanitize_filename(
642 restricted=self.params.get('restrictfilenames'),
643 is_id=(k == 'id' or k.endswith('_id')))
644 template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
645 for k, v in template_dict.items()
646 if v is not None and not isinstance(v, (list, tuple, dict)))
647 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
649 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
651 # For fields playlist_index and autonumber convert all occurrences
652 # of %(field)s to %(field)0Nd for backward compatibility
653 field_size_compat_map = {
654 'playlist_index': len(str(template_dict['n_entries'])),
655 'autonumber': autonumber_size,
657 FIELD_SIZE_COMPAT_RE = r'(?<!%)%\((?P<field>autonumber|playlist_index)\)s'
658 mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl)
661 FIELD_SIZE_COMPAT_RE,
662 r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
665 # Missing numeric fields used together with integer presentation types
666 # in format specification will break the argument substitution since
667 # string 'NA' is returned for missing fields. We will patch output
668 # template for missing fields to meet string presentation type.
669 for numeric_field in self._NUMERIC_FIELDS:
670 if numeric_field not in template_dict:
671 # As of [1] format syntax is:
672 # %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
673 # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
677 \({0}\) # mapping key
678 (?:[#0\-+ ]+)? # conversion flags (optional)
679 (?:\d+)? # minimum field width (optional)
680 (?:\.\d+)? # precision (optional)
681 [hlL]? # length modifier (optional)
682 [diouxXeEfFgGcrs%] # conversion type
685 FORMAT_RE.format(numeric_field),
686 r'%({0})s'.format(numeric_field), outtmpl)
688 # expand_path translates '%%' into '%' and '$$' into '$'
689 # correspondingly that is not what we want since we need to keep
690 # '%%' intact for template dict substitution step. Working around
691 # with boundary-alike separator hack.
692 sep = ''.join([random.choice(ascii_letters) for _ in range(32)])
693 outtmpl = outtmpl.replace('%%', '%{0}%'.format(sep)).replace('$$', '${0}$'.format(sep))
695 # outtmpl should be expand_path'ed before template dict substitution
696 # because meta fields may contain env variables we don't want to
697 # be expanded. For example, for outtmpl "%(title)s.%(ext)s" and
698 # title "Hello $PATH", we don't want `$PATH` to be expanded.
699 filename = expand_path(outtmpl).replace(sep, '') % template_dict
701 # Temporary fix for #4787
702 # 'Treat' all problem characters by passing filename through preferredencoding
703 # to workaround encoding issues with subprocess on python2 @ Windows
704 if sys.version_info < (3, 0) and sys.platform == 'win32':
705 filename = encodeFilename(filename, True).decode(preferredencoding())
706 return sanitize_path(filename)
707 except ValueError as err:
708 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
711 def _match_entry(self, info_dict, incomplete):
712 """ Returns None iff the file should be downloaded """
714 video_title = info_dict.get('title', info_dict.get('id', 'video'))
715 if 'title' in info_dict:
716 # This can happen when we're just evaluating the playlist
717 title = info_dict['title']
718 matchtitle = self.params.get('matchtitle', False)
720 if not re.search(matchtitle, title, re.IGNORECASE):
721 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
722 rejecttitle = self.params.get('rejecttitle', False)
724 if re.search(rejecttitle, title, re.IGNORECASE):
725 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
726 date = info_dict.get('upload_date')
728 dateRange = self.params.get('daterange', DateRange())
729 if date not in dateRange:
730 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
731 view_count = info_dict.get('view_count')
732 if view_count is not None:
733 min_views = self.params.get('min_views')
734 if min_views is not None and view_count < min_views:
735 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
736 max_views = self.params.get('max_views')
737 if max_views is not None and view_count > max_views:
738 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
739 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
740 return 'Skipping "%s" because it is age restricted' % video_title
741 if self.in_download_archive(info_dict):
742 return '%s has already been recorded in archive' % video_title
745 match_filter = self.params.get('match_filter')
746 if match_filter is not None:
747 ret = match_filter(info_dict)
754 def add_extra_info(info_dict, extra_info):
755 '''Set the keys from extra_info in info dict if they are missing'''
756 for key, value in extra_info.items():
757 info_dict.setdefault(key, value)
759 def extract_info(self, url, download=True, ie_key=None, extra_info={},
760 process=True, force_generic_extractor=False):
762 Returns a list with a dictionary for each video we find.
763 If 'download', also downloads the videos.
764 extra_info is a dict containing the extra values to add to each result
767 if not ie_key and force_generic_extractor:
771 ies = [self.get_info_extractor(ie_key)]
776 if not ie.suitable(url):
779 ie = self.get_info_extractor(ie.ie_key())
781 self.report_warning('The program functionality for this site has been marked as broken, '
782 'and will probably not work.')
785 ie_result = ie.extract(url)
786 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
788 if isinstance(ie_result, list):
789 # Backwards compatibility: old IE result format
791 '_type': 'compat_list',
792 'entries': ie_result,
794 self.add_default_extra_info(ie_result, ie, url)
796 return self.process_ie_result(ie_result, download, extra_info)
799 except GeoRestrictedError as e:
802 msg += '\nThis video is available in %s.' % ', '.join(
803 map(ISO3166Utils.short2full, e.countries))
804 msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
805 self.report_error(msg)
807 except ExtractorError as e: # An error we somewhat expected
808 self.report_error(compat_str(e), e.format_traceback())
810 except MaxDownloadsReached:
812 except Exception as e:
813 if self.params.get('ignoreerrors', False):
814 self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc()))
819 self.report_error('no suitable InfoExtractor for URL %s' % url)
821 def add_default_extra_info(self, ie_result, ie, url):
822 self.add_extra_info(ie_result, {
823 'extractor': ie.IE_NAME,
825 'webpage_url_basename': url_basename(url),
826 'extractor_key': ie.ie_key(),
829 def process_ie_result(self, ie_result, download=True, extra_info={}):
831 Take the result of the ie(may be modified) and resolve all unresolved
832 references (URLs, playlist items).
834 It will also download the videos if 'download'.
835 Returns the resolved ie_result.
837 result_type = ie_result.get('_type', 'video')
839 if result_type in ('url', 'url_transparent'):
840 ie_result['url'] = sanitize_url(ie_result['url'])
841 extract_flat = self.params.get('extract_flat', False)
842 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
843 extract_flat is True):
844 if self.params.get('forcejson', False):
845 self.to_stdout(json.dumps(ie_result))
848 if result_type == 'video':
849 self.add_extra_info(ie_result, extra_info)
850 return self.process_video_result(ie_result, download=download)
851 elif result_type == 'url':
852 # We have to add extra_info to the results because it may be
853 # contained in a playlist
854 return self.extract_info(ie_result['url'],
856 ie_key=ie_result.get('ie_key'),
857 extra_info=extra_info)
858 elif result_type == 'url_transparent':
859 # Use the information from the embedding page
860 info = self.extract_info(
861 ie_result['url'], ie_key=ie_result.get('ie_key'),
862 extra_info=extra_info, download=False, process=False)
864 # extract_info may return None when ignoreerrors is enabled and
865 # extraction failed with an error, don't crash and return early
870 force_properties = dict(
871 (k, v) for k, v in ie_result.items() if v is not None)
872 for f in ('_type', 'url', 'id', 'extractor', 'extractor_key', 'ie_key'):
873 if f in force_properties:
874 del force_properties[f]
875 new_result = info.copy()
876 new_result.update(force_properties)
878 # Extracted info may not be a video result (i.e.
879 # info.get('_type', 'video') != video) but rather an url or
880 # url_transparent. In such cases outer metadata (from ie_result)
881 # should be propagated to inner one (info). For this to happen
882 # _type of info should be overridden with url_transparent. This
883 # fixes issue from https://github.com/rg3/youtube-dl/pull/11163.
884 if new_result.get('_type') == 'url':
885 new_result['_type'] = 'url_transparent'
887 return self.process_ie_result(
888 new_result, download=download, extra_info=extra_info)
889 elif result_type in ('playlist', 'multi_video'):
890 # We process each entry in the playlist
891 playlist = ie_result.get('title') or ie_result.get('id')
892 self.to_screen('[download] Downloading playlist: %s' % playlist)
894 playlist_results = []
896 playliststart = self.params.get('playliststart', 1) - 1
897 playlistend = self.params.get('playlistend')
898 # For backwards compatibility, interpret -1 as whole list
899 if playlistend == -1:
902 playlistitems_str = self.params.get('playlist_items')
904 if playlistitems_str is not None:
905 def iter_playlistitems(format):
906 for string_segment in format.split(','):
907 if '-' in string_segment:
908 start, end = string_segment.split('-')
909 for item in range(int(start), int(end) + 1):
912 yield int(string_segment)
913 playlistitems = orderedSet(iter_playlistitems(playlistitems_str))
915 ie_entries = ie_result['entries']
917 def make_playlistitems_entries(list_ie_entries):
918 num_entries = len(list_ie_entries)
920 list_ie_entries[i - 1] for i in playlistitems
921 if -num_entries <= i - 1 < num_entries]
923 def report_download(num_entries):
925 '[%s] playlist %s: Downloading %d videos' %
926 (ie_result['extractor'], playlist, num_entries))
928 if isinstance(ie_entries, list):
929 n_all_entries = len(ie_entries)
931 entries = make_playlistitems_entries(ie_entries)
933 entries = ie_entries[playliststart:playlistend]
934 n_entries = len(entries)
936 '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
937 (ie_result['extractor'], playlist, n_all_entries, n_entries))
938 elif isinstance(ie_entries, PagedList):
941 for item in playlistitems:
942 entries.extend(ie_entries.getslice(
946 entries = ie_entries.getslice(
947 playliststart, playlistend)
948 n_entries = len(entries)
949 report_download(n_entries)
952 entries = make_playlistitems_entries(list(itertools.islice(
953 ie_entries, 0, max(playlistitems))))
955 entries = list(itertools.islice(
956 ie_entries, playliststart, playlistend))
957 n_entries = len(entries)
958 report_download(n_entries)
960 if self.params.get('playlistreverse', False):
961 entries = entries[::-1]
963 if self.params.get('playlistrandom', False):
964 random.shuffle(entries)
966 x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
968 for i, entry in enumerate(entries, 1):
969 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
970 # This __x_forwarded_for_ip thing is a bit ugly but requires
973 entry['__x_forwarded_for_ip'] = x_forwarded_for
975 'n_entries': n_entries,
976 'playlist': playlist,
977 'playlist_id': ie_result.get('id'),
978 'playlist_title': ie_result.get('title'),
979 'playlist_uploader': ie_result.get('uploader'),
980 'playlist_uploader_id': ie_result.get('uploader_id'),
981 'playlist_index': i + playliststart,
982 'extractor': ie_result['extractor'],
983 'webpage_url': ie_result['webpage_url'],
984 'webpage_url_basename': url_basename(ie_result['webpage_url']),
985 'extractor_key': ie_result['extractor_key'],
988 reason = self._match_entry(entry, incomplete=True)
989 if reason is not None:
990 self.to_screen('[download] ' + reason)
993 entry_result = self.process_ie_result(entry,
996 playlist_results.append(entry_result)
997 ie_result['entries'] = playlist_results
998 self.to_screen('[download] Finished downloading playlist: %s' % playlist)
1000 elif result_type == 'compat_list':
1001 self.report_warning(
1002 'Extractor %s returned a compat_list result. '
1003 'It needs to be updated.' % ie_result.get('extractor'))
1006 self.add_extra_info(
1009 'extractor': ie_result['extractor'],
1010 'webpage_url': ie_result['webpage_url'],
1011 'webpage_url_basename': url_basename(ie_result['webpage_url']),
1012 'extractor_key': ie_result['extractor_key'],
1016 ie_result['entries'] = [
1017 self.process_ie_result(_fixup(r), download, extra_info)
1018 for r in ie_result['entries']
1022 raise Exception('Invalid result type: %s' % result_type)
1024 def _build_format_filter(self, filter_spec):
1025 " Returns a function to filter the formats according to the filter_spec "
1035 operator_rex = re.compile(r'''(?x)\s*
1036 (?P<key>width|height|tbr|abr|vbr|asr|filesize|filesize_approx|fps)
1037 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1038 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
1040 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
1041 m = operator_rex.search(filter_spec)
1044 comparison_value = int(m.group('value'))
1046 comparison_value = parse_filesize(m.group('value'))
1047 if comparison_value is None:
1048 comparison_value = parse_filesize(m.group('value') + 'B')
1049 if comparison_value is None:
1051 'Invalid value %r in format specification %r' % (
1052 m.group('value'), filter_spec))
1053 op = OPERATORS[m.group('op')]
1059 '^=': lambda attr, value: attr.startswith(value),
1060 '$=': lambda attr, value: attr.endswith(value),
1061 '*=': lambda attr, value: value in attr,
1063 str_operator_rex = re.compile(r'''(?x)
1064 \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
1065 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
1066 \s*(?P<value>[a-zA-Z0-9._-]+)
1068 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
1069 m = str_operator_rex.search(filter_spec)
1071 comparison_value = m.group('value')
1072 op = STR_OPERATORS[m.group('op')]
1075 raise ValueError('Invalid filter specification %r' % filter_spec)
1078 actual_value = f.get(m.group('key'))
1079 if actual_value is None:
1080 return m.group('none_inclusive')
1081 return op(actual_value, comparison_value)
1084 def _default_format_spec(self, info_dict, download=True):
1087 merger = FFmpegMergerPP(self)
1088 return merger.available and merger.can_merge()
1091 if self.params.get('simulate', False):
1095 if self.params.get('outtmpl', DEFAULT_OUTTMPL) == '-':
1097 if info_dict.get('is_live'):
1103 req_format_list = ['bestvideo+bestaudio', 'best']
1105 req_format_list.reverse()
1106 return '/'.join(req_format_list)
1108 def build_format_selector(self, format_spec):
1109 def syntax_error(note, start):
1111 'Invalid format specification: '
1112 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
1113 return SyntaxError(message)
1115 PICKFIRST = 'PICKFIRST'
1119 FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
1121 def _parse_filter(tokens):
1123 for type, string, start, _, _ in tokens:
1124 if type == tokenize.OP and string == ']':
1125 return ''.join(filter_parts)
1127 filter_parts.append(string)
1129 def _remove_unused_ops(tokens):
1130 # Remove operators that we don't use and join them with the surrounding strings
1131 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
1132 ALLOWED_OPS = ('/', '+', ',', '(', ')')
1133 last_string, last_start, last_end, last_line = None, None, None, None
1134 for type, string, start, end, line in tokens:
1135 if type == tokenize.OP and string == '[':
1137 yield tokenize.NAME, last_string, last_start, last_end, last_line
1139 yield type, string, start, end, line
1140 # everything inside brackets will be handled by _parse_filter
1141 for type, string, start, end, line in tokens:
1142 yield type, string, start, end, line
1143 if type == tokenize.OP and string == ']':
1145 elif type == tokenize.OP and string in ALLOWED_OPS:
1147 yield tokenize.NAME, last_string, last_start, last_end, last_line
1149 yield type, string, start, end, line
1150 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
1152 last_string = string
1156 last_string += string
1158 yield tokenize.NAME, last_string, last_start, last_end, last_line
1160 def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
1162 current_selector = None
1163 for type, string, start, _, _ in tokens:
1164 # ENCODING is only defined in python 3.x
1165 if type == getattr(tokenize, 'ENCODING', None):
1167 elif type in [tokenize.NAME, tokenize.NUMBER]:
1168 current_selector = FormatSelector(SINGLE, string, [])
1169 elif type == tokenize.OP:
1171 if not inside_group:
1172 # ')' will be handled by the parentheses group
1173 tokens.restore_last_token()
1175 elif inside_merge and string in ['/', ',']:
1176 tokens.restore_last_token()
1178 elif inside_choice and string == ',':
1179 tokens.restore_last_token()
1182 if not current_selector:
1183 raise syntax_error('"," must follow a format selector', start)
1184 selectors.append(current_selector)
1185 current_selector = None
1187 if not current_selector:
1188 raise syntax_error('"/" must follow a format selector', start)
1189 first_choice = current_selector
1190 second_choice = _parse_format_selection(tokens, inside_choice=True)
1191 current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1193 if not current_selector:
1194 current_selector = FormatSelector(SINGLE, 'best', [])
1195 format_filter = _parse_filter(tokens)
1196 current_selector.filters.append(format_filter)
1198 if current_selector:
1199 raise syntax_error('Unexpected "("', start)
1200 group = _parse_format_selection(tokens, inside_group=True)
1201 current_selector = FormatSelector(GROUP, group, [])
1203 video_selector = current_selector
1204 audio_selector = _parse_format_selection(tokens, inside_merge=True)
1205 if not video_selector or not audio_selector:
1206 raise syntax_error('"+" must be between two format selectors', start)
1207 current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1209 raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1210 elif type == tokenize.ENDMARKER:
1212 if current_selector:
1213 selectors.append(current_selector)
1216 def _build_selector_function(selector):
1217 if isinstance(selector, list):
1218 fs = [_build_selector_function(s) for s in selector]
1220 def selector_function(ctx):
1222 for format in f(ctx):
1224 return selector_function
1225 elif selector.type == GROUP:
1226 selector_function = _build_selector_function(selector.selector)
1227 elif selector.type == PICKFIRST:
1228 fs = [_build_selector_function(s) for s in selector.selector]
1230 def selector_function(ctx):
1232 picked_formats = list(f(ctx))
1234 return picked_formats
1236 elif selector.type == SINGLE:
1237 format_spec = selector.selector
1239 def selector_function(ctx):
1240 formats = list(ctx['formats'])
1243 if format_spec == 'all':
1246 elif format_spec in ['best', 'worst', None]:
1247 format_idx = 0 if format_spec == 'worst' else -1
1248 audiovideo_formats = [
1250 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1251 if audiovideo_formats:
1252 yield audiovideo_formats[format_idx]
1253 # for extractors with incomplete formats (audio only (soundcloud)
1254 # or video only (imgur)) we will fallback to best/worst
1255 # {video,audio}-only format
1256 elif ctx['incomplete_formats']:
1257 yield formats[format_idx]
1258 elif format_spec == 'bestaudio':
1261 if f.get('vcodec') == 'none']
1263 yield audio_formats[-1]
1264 elif format_spec == 'worstaudio':
1267 if f.get('vcodec') == 'none']
1269 yield audio_formats[0]
1270 elif format_spec == 'bestvideo':
1273 if f.get('acodec') == 'none']
1275 yield video_formats[-1]
1276 elif format_spec == 'worstvideo':
1279 if f.get('acodec') == 'none']
1281 yield video_formats[0]
1283 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1284 if format_spec in extensions:
1285 filter_f = lambda f: f['ext'] == format_spec
1287 filter_f = lambda f: f['format_id'] == format_spec
1288 matches = list(filter(filter_f, formats))
1291 elif selector.type == MERGE:
1292 def _merge(formats_info):
1293 format_1, format_2 = [f['format_id'] for f in formats_info]
1294 # The first format must contain the video and the
1296 if formats_info[0].get('vcodec') == 'none':
1297 self.report_error('The first format must '
1298 'contain the video, try using '
1299 '"-f %s+%s"' % (format_2, format_1))
1301 # Formats must be opposite (video+audio)
1302 if formats_info[0].get('acodec') == 'none' and formats_info[1].get('acodec') == 'none':
1304 'Both formats %s and %s are video-only, you must specify "-f video+audio"'
1305 % (format_1, format_2))
1308 formats_info[0]['ext']
1309 if self.params.get('merge_output_format') is None
1310 else self.params['merge_output_format'])
1312 'requested_formats': formats_info,
1313 'format': '%s+%s' % (formats_info[0].get('format'),
1314 formats_info[1].get('format')),
1315 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1316 formats_info[1].get('format_id')),
1317 'width': formats_info[0].get('width'),
1318 'height': formats_info[0].get('height'),
1319 'resolution': formats_info[0].get('resolution'),
1320 'fps': formats_info[0].get('fps'),
1321 'vcodec': formats_info[0].get('vcodec'),
1322 'vbr': formats_info[0].get('vbr'),
1323 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1324 'acodec': formats_info[1].get('acodec'),
1325 'abr': formats_info[1].get('abr'),
1328 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1330 def selector_function(ctx):
1331 for pair in itertools.product(
1332 video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
1335 filters = [self._build_format_filter(f) for f in selector.filters]
1337 def final_selector(ctx):
1338 ctx_copy = copy.deepcopy(ctx)
1339 for _filter in filters:
1340 ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
1341 return selector_function(ctx_copy)
1342 return final_selector
1344 stream = io.BytesIO(format_spec.encode('utf-8'))
1346 tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1347 except tokenize.TokenError:
1348 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1350 class TokenIterator(object):
1351 def __init__(self, tokens):
1352 self.tokens = tokens
1359 if self.counter >= len(self.tokens):
1360 raise StopIteration()
1361 value = self.tokens[self.counter]
1367 def restore_last_token(self):
1370 parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1371 return _build_selector_function(parsed_selector)
1373 def _calc_headers(self, info_dict):
1374 res = std_headers.copy()
1376 add_headers = info_dict.get('http_headers')
1378 res.update(add_headers)
1380 cookies = self._calc_cookies(info_dict)
1382 res['Cookie'] = cookies
1384 if 'X-Forwarded-For' not in res:
1385 x_forwarded_for_ip = info_dict.get('__x_forwarded_for_ip')
1386 if x_forwarded_for_ip:
1387 res['X-Forwarded-For'] = x_forwarded_for_ip
1391 def _calc_cookies(self, info_dict):
1392 pr = sanitized_Request(info_dict['url'])
1393 self.cookiejar.add_cookie_header(pr)
1394 return pr.get_header('Cookie')
1396 def process_video_result(self, info_dict, download=True):
1397 assert info_dict.get('_type', 'video') == 'video'
1399 if 'id' not in info_dict:
1400 raise ExtractorError('Missing "id" field in extractor result')
1401 if 'title' not in info_dict:
1402 raise ExtractorError('Missing "title" field in extractor result')
1404 def report_force_conversion(field, field_not, conversion):
1405 self.report_warning(
1406 '"%s" field is not %s - forcing %s conversion, there is an error in extractor'
1407 % (field, field_not, conversion))
1409 def sanitize_string_field(info, string_field):
1410 field = info.get(string_field)
1411 if field is None or isinstance(field, compat_str):
1413 report_force_conversion(string_field, 'a string', 'string')
1414 info[string_field] = compat_str(field)
1416 def sanitize_numeric_fields(info):
1417 for numeric_field in self._NUMERIC_FIELDS:
1418 field = info.get(numeric_field)
1419 if field is None or isinstance(field, compat_numeric_types):
1421 report_force_conversion(numeric_field, 'numeric', 'int')
1422 info[numeric_field] = int_or_none(field)
1424 sanitize_string_field(info_dict, 'id')
1425 sanitize_numeric_fields(info_dict)
1427 if 'playlist' not in info_dict:
1428 # It isn't part of a playlist
1429 info_dict['playlist'] = None
1430 info_dict['playlist_index'] = None
1432 thumbnails = info_dict.get('thumbnails')
1433 if thumbnails is None:
1434 thumbnail = info_dict.get('thumbnail')
1436 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1438 thumbnails.sort(key=lambda t: (
1439 t.get('preference') if t.get('preference') is not None else -1,
1440 t.get('width') if t.get('width') is not None else -1,
1441 t.get('height') if t.get('height') is not None else -1,
1442 t.get('id') if t.get('id') is not None else '', t.get('url')))
1443 for i, t in enumerate(thumbnails):
1444 t['url'] = sanitize_url(t['url'])
1445 if t.get('width') and t.get('height'):
1446 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1447 if t.get('id') is None:
1450 if self.params.get('list_thumbnails'):
1451 self.list_thumbnails(info_dict)
1454 thumbnail = info_dict.get('thumbnail')
1456 info_dict['thumbnail'] = sanitize_url(thumbnail)
1458 info_dict['thumbnail'] = thumbnails[-1]['url']
1460 if 'display_id' not in info_dict and 'id' in info_dict:
1461 info_dict['display_id'] = info_dict['id']
1463 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1464 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1465 # see http://bugs.python.org/issue1646728)
1467 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1468 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1469 except (ValueError, OverflowError, OSError):
1472 # Auto generate title fields corresponding to the *_number fields when missing
1473 # in order to always have clean titles. This is very common for TV series.
1474 for field in ('chapter', 'season', 'episode'):
1475 if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
1476 info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
1478 subtitles = info_dict.get('subtitles')
1480 for _, subtitle in subtitles.items():
1481 for subtitle_format in subtitle:
1482 if subtitle_format.get('url'):
1483 subtitle_format['url'] = sanitize_url(subtitle_format['url'])
1484 if subtitle_format.get('ext') is None:
1485 subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
1487 if self.params.get('listsubtitles', False):
1488 if 'automatic_captions' in info_dict:
1489 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1490 self.list_subtitles(info_dict['id'], subtitles, 'subtitles')
1492 info_dict['requested_subtitles'] = self.process_subtitles(
1493 info_dict['id'], subtitles,
1494 info_dict.get('automatic_captions'))
1496 # We now pick which formats have to be downloaded
1497 if info_dict.get('formats') is None:
1498 # There's only one format available
1499 formats = [info_dict]
1501 formats = info_dict['formats']
1504 raise ExtractorError('No video formats found!')
1506 def is_wellformed(f):
1509 self.report_warning(
1510 '"url" field is missing or empty - skipping format, '
1511 'there is an error in extractor')
1513 if isinstance(url, bytes):
1514 sanitize_string_field(f, 'url')
1517 # Filter out malformed formats for better extraction robustness
1518 formats = list(filter(is_wellformed, formats))
1522 # We check that all the formats have the format and format_id fields
1523 for i, format in enumerate(formats):
1524 sanitize_string_field(format, 'format_id')
1525 sanitize_numeric_fields(format)
1526 format['url'] = sanitize_url(format['url'])
1527 if not format.get('format_id'):
1528 format['format_id'] = compat_str(i)
1530 # Sanitize format_id from characters used in format selector expression
1531 format['format_id'] = re.sub(r'[\s,/+\[\]()]', '_', format['format_id'])
1532 format_id = format['format_id']
1533 if format_id not in formats_dict:
1534 formats_dict[format_id] = []
1535 formats_dict[format_id].append(format)
1537 # Make sure all formats have unique format_id
1538 for format_id, ambiguous_formats in formats_dict.items():
1539 if len(ambiguous_formats) > 1:
1540 for i, format in enumerate(ambiguous_formats):
1541 format['format_id'] = '%s-%d' % (format_id, i)
1543 for i, format in enumerate(formats):
1544 if format.get('format') is None:
1545 format['format'] = '{id} - {res}{note}'.format(
1546 id=format['format_id'],
1547 res=self.format_resolution(format),
1548 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1550 # Automatically determine file extension if missing
1551 if format.get('ext') is None:
1552 format['ext'] = determine_ext(format['url']).lower()
1553 # Automatically determine protocol if missing (useful for format
1554 # selection purposes)
1555 if format.get('protocol') is None:
1556 format['protocol'] = determine_protocol(format)
1557 # Add HTTP headers, so that external programs can use them from the
1559 full_format_info = info_dict.copy()
1560 full_format_info.update(format)
1561 format['http_headers'] = self._calc_headers(full_format_info)
1562 # Remove private housekeeping stuff
1563 if '__x_forwarded_for_ip' in info_dict:
1564 del info_dict['__x_forwarded_for_ip']
1566 # TODO Central sorting goes here
1568 if formats[0] is not info_dict:
1569 # only set the 'formats' fields if the original info_dict list them
1570 # otherwise we end up with a circular reference, the first (and unique)
1571 # element in the 'formats' field in info_dict is info_dict itself,
1572 # which can't be exported to json
1573 info_dict['formats'] = formats
1574 if self.params.get('listformats'):
1575 self.list_formats(info_dict)
1578 req_format = self.params.get('format')
1579 if req_format is None:
1580 req_format = self._default_format_spec(info_dict, download=download)
1581 if self.params.get('verbose'):
1582 self.to_stdout('[debug] Default format spec: %s' % req_format)
1584 format_selector = self.build_format_selector(req_format)
1586 # While in format selection we may need to have an access to the original
1587 # format set in order to calculate some metrics or do some processing.
1588 # For now we need to be able to guess whether original formats provided
1589 # by extractor are incomplete or not (i.e. whether extractor provides only
1590 # video-only or audio-only formats) for proper formats selection for
1591 # extractors with such incomplete formats (see
1592 # https://github.com/rg3/youtube-dl/pull/5556).
1593 # Since formats may be filtered during format selection and may not match
1594 # the original formats the results may be incorrect. Thus original formats
1595 # or pre-calculated metrics should be passed to format selection routines
1597 # We will pass a context object containing all necessary additional data
1598 # instead of just formats.
1599 # This fixes incorrect format selection issue (see
1600 # https://github.com/rg3/youtube-dl/issues/10083).
1601 incomplete_formats = (
1602 # All formats are video-only or
1603 all(f.get('vcodec') != 'none' and f.get('acodec') == 'none' for f in formats) or
1604 # all formats are audio-only
1605 all(f.get('vcodec') == 'none' and f.get('acodec') != 'none' for f in formats))
1609 'incomplete_formats': incomplete_formats,
1612 formats_to_download = list(format_selector(ctx))
1613 if not formats_to_download:
1614 raise ExtractorError('requested format not available',
1618 if len(formats_to_download) > 1:
1619 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1620 for format in formats_to_download:
1621 new_info = dict(info_dict)
1622 new_info.update(format)
1623 self.process_info(new_info)
1624 # We update the info dict with the best quality format (backwards compatibility)
1625 info_dict.update(formats_to_download[-1])
1628 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1629 """Select the requested subtitles and their format"""
1631 if normal_subtitles and self.params.get('writesubtitles'):
1632 available_subs.update(normal_subtitles)
1633 if automatic_captions and self.params.get('writeautomaticsub'):
1634 for lang, cap_info in automatic_captions.items():
1635 if lang not in available_subs:
1636 available_subs[lang] = cap_info
1638 if (not self.params.get('writesubtitles') and not
1639 self.params.get('writeautomaticsub') or not
1643 if self.params.get('allsubtitles', False):
1644 requested_langs = available_subs.keys()
1646 if self.params.get('subtitleslangs', False):
1647 requested_langs = self.params.get('subtitleslangs')
1648 elif 'en' in available_subs:
1649 requested_langs = ['en']
1651 requested_langs = [list(available_subs.keys())[0]]
1653 formats_query = self.params.get('subtitlesformat', 'best')
1654 formats_preference = formats_query.split('/') if formats_query else []
1656 for lang in requested_langs:
1657 formats = available_subs.get(lang)
1659 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1661 for ext in formats_preference:
1665 matches = list(filter(lambda f: f['ext'] == ext, formats))
1671 self.report_warning(
1672 'No subtitle format found matching "%s" for language %s, '
1673 'using %s' % (formats_query, lang, f['ext']))
1677 def process_info(self, info_dict):
1678 """Process a single resolved IE result."""
1680 assert info_dict.get('_type', 'video') == 'video'
1682 max_downloads = self.params.get('max_downloads')
1683 if max_downloads is not None:
1684 if self._num_downloads >= int(max_downloads):
1685 raise MaxDownloadsReached()
1687 info_dict['fulltitle'] = info_dict['title']
1688 if len(info_dict['title']) > 200:
1689 info_dict['title'] = info_dict['title'][:197] + '...'
1691 if 'format' not in info_dict:
1692 info_dict['format'] = info_dict['ext']
1694 reason = self._match_entry(info_dict, incomplete=False)
1695 if reason is not None:
1696 self.to_screen('[download] ' + reason)
1699 self._num_downloads += 1
1701 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1704 if self.params.get('forcetitle', False):
1705 self.to_stdout(info_dict['fulltitle'])
1706 if self.params.get('forceid', False):
1707 self.to_stdout(info_dict['id'])
1708 if self.params.get('forceurl', False):
1709 if info_dict.get('requested_formats') is not None:
1710 for f in info_dict['requested_formats']:
1711 self.to_stdout(f['url'] + f.get('play_path', ''))
1713 # For RTMP URLs, also include the playpath
1714 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1715 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1716 self.to_stdout(info_dict['thumbnail'])
1717 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1718 self.to_stdout(info_dict['description'])
1719 if self.params.get('forcefilename', False) and filename is not None:
1720 self.to_stdout(filename)
1721 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1722 self.to_stdout(formatSeconds(info_dict['duration']))
1723 if self.params.get('forceformat', False):
1724 self.to_stdout(info_dict['format'])
1725 if self.params.get('forcejson', False):
1726 self.to_stdout(json.dumps(info_dict))
1728 # Do nothing else if in simulate mode
1729 if self.params.get('simulate', False):
1732 if filename is None:
1735 def ensure_dir_exists(path):
1737 dn = os.path.dirname(path)
1738 if dn and not os.path.exists(dn):
1741 except (OSError, IOError) as err:
1742 self.report_error('unable to create directory ' + error_to_compat_str(err))
1745 if not ensure_dir_exists(sanitize_path(encodeFilename(filename))):
1748 if self.params.get('writedescription', False):
1749 descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1750 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1751 self.to_screen('[info] Video description is already present')
1752 elif info_dict.get('description') is None:
1753 self.report_warning('There\'s no description to write.')
1756 self.to_screen('[info] Writing video description to: ' + descfn)
1757 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1758 descfile.write(info_dict['description'])
1759 except (OSError, IOError):
1760 self.report_error('Cannot write description file ' + descfn)
1763 if self.params.get('writeannotations', False):
1764 annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1765 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1766 self.to_screen('[info] Video annotations are already present')
1769 self.to_screen('[info] Writing video annotations to: ' + annofn)
1770 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1771 annofile.write(info_dict['annotations'])
1772 except (KeyError, TypeError):
1773 self.report_warning('There are no annotations to write.')
1774 except (OSError, IOError):
1775 self.report_error('Cannot write annotations file: ' + annofn)
1778 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1779 self.params.get('writeautomaticsub')])
1781 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1782 # subtitles download errors are already managed as troubles in relevant IE
1783 # that way it will silently go on when used with unsupporting IE
1784 subtitles = info_dict['requested_subtitles']
1785 ie = self.get_info_extractor(info_dict['extractor_key'])
1786 for sub_lang, sub_info in subtitles.items():
1787 sub_format = sub_info['ext']
1788 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1789 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1790 self.to_screen('[info] Video subtitle %s.%s is already present' % (sub_lang, sub_format))
1792 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1793 if sub_info.get('data') is not None:
1795 # Use newline='' to prevent conversion of newline characters
1796 # See https://github.com/rg3/youtube-dl/issues/10268
1797 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
1798 subfile.write(sub_info['data'])
1799 except (OSError, IOError):
1800 self.report_error('Cannot write subtitles file ' + sub_filename)
1804 sub_data = ie._request_webpage(
1805 sub_info['url'], info_dict['id'], note=False).read()
1806 with io.open(encodeFilename(sub_filename), 'wb') as subfile:
1807 subfile.write(sub_data)
1808 except (ExtractorError, IOError, OSError, ValueError) as err:
1809 self.report_warning('Unable to download subtitle for "%s": %s' %
1810 (sub_lang, error_to_compat_str(err)))
1813 if self.params.get('writeinfojson', False):
1814 infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1815 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1816 self.to_screen('[info] Video description metadata is already present')
1818 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1820 write_json_file(self.filter_requested_info(info_dict), infofn)
1821 except (OSError, IOError):
1822 self.report_error('Cannot write metadata to JSON file ' + infofn)
1825 self._write_thumbnails(info_dict, filename)
1827 if not self.params.get('skip_download', False):
1830 fd = get_suitable_downloader(info, self.params)(self, self.params)
1831 for ph in self._progress_hooks:
1832 fd.add_progress_hook(ph)
1833 if self.params.get('verbose'):
1834 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1835 return fd.download(name, info)
1837 if info_dict.get('requested_formats') is not None:
1840 merger = FFmpegMergerPP(self)
1841 if not merger.available:
1843 self.report_warning('You have requested multiple '
1844 'formats but ffmpeg or avconv are not installed.'
1845 ' The formats won\'t be merged.')
1847 postprocessors = [merger]
1849 def compatible_formats(formats):
1850 video, audio = formats
1852 video_ext, audio_ext = audio.get('ext'), video.get('ext')
1853 if video_ext and audio_ext:
1855 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'),
1858 for exts in COMPATIBLE_EXTS:
1859 if video_ext in exts and audio_ext in exts:
1861 # TODO: Check acodec/vcodec
1864 filename_real_ext = os.path.splitext(filename)[1][1:]
1866 os.path.splitext(filename)[0]
1867 if filename_real_ext == info_dict['ext']
1869 requested_formats = info_dict['requested_formats']
1870 if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1871 info_dict['ext'] = 'mkv'
1872 self.report_warning(
1873 'Requested formats are incompatible for merge and will be merged into mkv.')
1874 # Ensure filename always has a correct extension for successful merge
1875 filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1876 if os.path.exists(encodeFilename(filename)):
1878 '[download] %s has already been downloaded and '
1879 'merged' % filename)
1881 for f in requested_formats:
1882 new_info = dict(info_dict)
1884 fname = prepend_extension(
1885 self.prepare_filename(new_info),
1886 'f%s' % f['format_id'], new_info['ext'])
1887 if not ensure_dir_exists(fname):
1889 downloaded.append(fname)
1890 partial_success = dl(fname, new_info)
1891 success = success and partial_success
1892 info_dict['__postprocessors'] = postprocessors
1893 info_dict['__files_to_merge'] = downloaded
1895 # Just a single file
1896 success = dl(filename, info_dict)
1897 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1898 self.report_error('unable to download video data: %s' % error_to_compat_str(err))
1900 except (OSError, IOError) as err:
1901 raise UnavailableVideoError(err)
1902 except (ContentTooShortError, ) as err:
1903 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1906 if success and filename != '-':
1908 fixup_policy = self.params.get('fixup')
1909 if fixup_policy is None:
1910 fixup_policy = 'detect_or_warn'
1912 INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
1914 stretched_ratio = info_dict.get('stretched_ratio')
1915 if stretched_ratio is not None and stretched_ratio != 1:
1916 if fixup_policy == 'warn':
1917 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1918 info_dict['id'], stretched_ratio))
1919 elif fixup_policy == 'detect_or_warn':
1920 stretched_pp = FFmpegFixupStretchedPP(self)
1921 if stretched_pp.available:
1922 info_dict.setdefault('__postprocessors', [])
1923 info_dict['__postprocessors'].append(stretched_pp)
1925 self.report_warning(
1926 '%s: Non-uniform pixel ratio (%s). %s'
1927 % (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE))
1929 assert fixup_policy in ('ignore', 'never')
1931 if (info_dict.get('requested_formats') is None and
1932 info_dict.get('container') == 'm4a_dash'):
1933 if fixup_policy == 'warn':
1934 self.report_warning(
1935 '%s: writing DASH m4a. '
1936 'Only some players support this container.'
1938 elif fixup_policy == 'detect_or_warn':
1939 fixup_pp = FFmpegFixupM4aPP(self)
1940 if fixup_pp.available:
1941 info_dict.setdefault('__postprocessors', [])
1942 info_dict['__postprocessors'].append(fixup_pp)
1944 self.report_warning(
1945 '%s: writing DASH m4a. '
1946 'Only some players support this container. %s'
1947 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1949 assert fixup_policy in ('ignore', 'never')
1951 if (info_dict.get('protocol') == 'm3u8_native' or
1952 info_dict.get('protocol') == 'm3u8' and
1953 self.params.get('hls_prefer_native')):
1954 if fixup_policy == 'warn':
1955 self.report_warning('%s: malformed AAC bitstream detected.' % (
1957 elif fixup_policy == 'detect_or_warn':
1958 fixup_pp = FFmpegFixupM3u8PP(self)
1959 if fixup_pp.available:
1960 info_dict.setdefault('__postprocessors', [])
1961 info_dict['__postprocessors'].append(fixup_pp)
1963 self.report_warning(
1964 '%s: malformed AAC bitstream detected. %s'
1965 % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
1967 assert fixup_policy in ('ignore', 'never')
1970 self.post_process(filename, info_dict)
1971 except (PostProcessingError) as err:
1972 self.report_error('postprocessing: %s' % str(err))
1974 self.record_download_archive(info_dict)
1976 def download(self, url_list):
1977 """Download a given list of URLs."""
1978 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1979 if (len(url_list) > 1 and
1981 '%' not in outtmpl and
1982 self.params.get('max_downloads') != 1):
1983 raise SameFileError(outtmpl)
1985 for url in url_list:
1987 # It also downloads the videos
1988 res = self.extract_info(
1989 url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1990 except UnavailableVideoError:
1991 self.report_error('unable to download video')
1992 except MaxDownloadsReached:
1993 self.to_screen('[info] Maximum number of downloaded files reached.')
1996 if self.params.get('dump_single_json', False):
1997 self.to_stdout(json.dumps(res))
1999 return self._download_retcode
2001 def download_with_info_file(self, info_filename):
2002 with contextlib.closing(fileinput.FileInput(
2003 [info_filename], mode='r',
2004 openhook=fileinput.hook_encoded('utf-8'))) as f:
2005 # FileInput doesn't have a read method, we can't call json.load
2006 info = self.filter_requested_info(json.loads('\n'.join(f)))
2008 self.process_ie_result(info, download=True)
2009 except DownloadError:
2010 webpage_url = info.get('webpage_url')
2011 if webpage_url is not None:
2012 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
2013 return self.download([webpage_url])
2016 return self._download_retcode
2019 def filter_requested_info(info_dict):
2021 (k, v) for k, v in info_dict.items()
2022 if k not in ['requested_formats', 'requested_subtitles'])
2024 def post_process(self, filename, ie_info):
2025 """Run all the postprocessors on the given file."""
2026 info = dict(ie_info)
2027 info['filepath'] = filename
2029 if ie_info.get('__postprocessors') is not None:
2030 pps_chain.extend(ie_info['__postprocessors'])
2031 pps_chain.extend(self._pps)
2032 for pp in pps_chain:
2033 files_to_delete = []
2035 files_to_delete, info = pp.run(info)
2036 except PostProcessingError as e:
2037 self.report_error(e.msg)
2038 if files_to_delete and not self.params.get('keepvideo', False):
2039 for old_filename in files_to_delete:
2040 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
2042 os.remove(encodeFilename(old_filename))
2043 except (IOError, OSError):
2044 self.report_warning('Unable to remove downloaded original file')
2046 def _make_archive_id(self, info_dict):
2047 # Future-proof against any change in case
2048 # and backwards compatibility with prior versions
2049 extractor = info_dict.get('extractor_key')
2050 if extractor is None:
2051 if 'id' in info_dict:
2052 extractor = info_dict.get('ie_key') # key in a playlist
2053 if extractor is None:
2054 return None # Incomplete video information
2055 return extractor.lower() + ' ' + info_dict['id']
2057 def in_download_archive(self, info_dict):
2058 fn = self.params.get('download_archive')
2062 vid_id = self._make_archive_id(info_dict)
2064 return False # Incomplete video information
2067 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
2068 for line in archive_file:
2069 if line.strip() == vid_id:
2071 except IOError as ioe:
2072 if ioe.errno != errno.ENOENT:
2076 def record_download_archive(self, info_dict):
2077 fn = self.params.get('download_archive')
2080 vid_id = self._make_archive_id(info_dict)
2082 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
2083 archive_file.write(vid_id + '\n')
2086 def format_resolution(format, default='unknown'):
2087 if format.get('vcodec') == 'none':
2089 if format.get('resolution') is not None:
2090 return format['resolution']
2091 if format.get('height') is not None:
2092 if format.get('width') is not None:
2093 res = '%sx%s' % (format['width'], format['height'])
2095 res = '%sp' % format['height']
2096 elif format.get('width') is not None:
2097 res = '%dx?' % format['width']
2102 def _format_note(self, fdict):
2104 if fdict.get('ext') in ['f4f', 'f4m']:
2105 res += '(unsupported) '
2106 if fdict.get('language'):
2109 res += '[%s] ' % fdict['language']
2110 if fdict.get('format_note') is not None:
2111 res += fdict['format_note'] + ' '
2112 if fdict.get('tbr') is not None:
2113 res += '%4dk ' % fdict['tbr']
2114 if fdict.get('container') is not None:
2117 res += '%s container' % fdict['container']
2118 if (fdict.get('vcodec') is not None and
2119 fdict.get('vcodec') != 'none'):
2122 res += fdict['vcodec']
2123 if fdict.get('vbr') is not None:
2125 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
2127 if fdict.get('vbr') is not None:
2128 res += '%4dk' % fdict['vbr']
2129 if fdict.get('fps') is not None:
2132 res += '%sfps' % fdict['fps']
2133 if fdict.get('acodec') is not None:
2136 if fdict['acodec'] == 'none':
2139 res += '%-5s' % fdict['acodec']
2140 elif fdict.get('abr') is not None:
2144 if fdict.get('abr') is not None:
2145 res += '@%3dk' % fdict['abr']
2146 if fdict.get('asr') is not None:
2147 res += ' (%5dHz)' % fdict['asr']
2148 if fdict.get('filesize') is not None:
2151 res += format_bytes(fdict['filesize'])
2152 elif fdict.get('filesize_approx') is not None:
2155 res += '~' + format_bytes(fdict['filesize_approx'])
2158 def list_formats(self, info_dict):
2159 formats = info_dict.get('formats', [info_dict])
2161 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
2163 if f.get('preference') is None or f['preference'] >= -1000]
2164 if len(formats) > 1:
2165 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
2167 header_line = ['format code', 'extension', 'resolution', 'note']
2169 '[info] Available formats for %s:\n%s' %
2170 (info_dict['id'], render_table(header_line, table)))
2172 def list_thumbnails(self, info_dict):
2173 thumbnails = info_dict.get('thumbnails')
2175 self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
2179 '[info] Thumbnails for %s:' % info_dict['id'])
2180 self.to_screen(render_table(
2181 ['ID', 'width', 'height', 'URL'],
2182 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
2184 def list_subtitles(self, video_id, subtitles, name='subtitles'):
2186 self.to_screen('%s has no %s' % (video_id, name))
2189 'Available %s for %s:' % (name, video_id))
2190 self.to_screen(render_table(
2191 ['Language', 'formats'],
2192 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
2193 for lang, formats in subtitles.items()]))
2195 def urlopen(self, req):
2196 """ Start an HTTP download """
2197 if isinstance(req, compat_basestring):
2198 req = sanitized_Request(req)
2199 return self._opener.open(req, timeout=self._socket_timeout)
2201 def print_debug_header(self):
2202 if not self.params.get('verbose'):
2205 if type('') is not compat_str:
2206 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
2207 self.report_warning(
2208 'Your Python is broken! Update to a newer and supported version')
2210 stdout_encoding = getattr(
2211 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
2213 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
2214 locale.getpreferredencoding(),
2215 sys.getfilesystemencoding(),
2217 self.get_encoding()))
2218 write_string(encoding_str, encoding=None)
2220 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
2222 self._write_string('[debug] Lazy loading extractors enabled' + '\n')
2224 sp = subprocess.Popen(
2225 ['git', 'rev-parse', '--short', 'HEAD'],
2226 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2227 cwd=os.path.dirname(os.path.abspath(__file__)))
2228 out, err = sp.communicate()
2229 out = out.decode().strip()
2230 if re.match('[0-9a-f]+', out):
2231 self._write_string('[debug] Git HEAD: ' + out + '\n')
2238 def python_implementation():
2239 impl_name = platform.python_implementation()
2240 if impl_name == 'PyPy' and hasattr(sys, 'pypy_version_info'):
2241 return impl_name + ' version %d.%d.%d' % sys.pypy_version_info[:3]
2244 self._write_string('[debug] Python version %s (%s) - %s\n' % (
2245 platform.python_version(), python_implementation(),
2248 exe_versions = FFmpegPostProcessor.get_versions(self)
2249 exe_versions['rtmpdump'] = rtmpdump_version()
2250 exe_versions['phantomjs'] = PhantomJSwrapper._version()
2251 exe_str = ', '.join(
2253 for exe, v in sorted(exe_versions.items())
2258 self._write_string('[debug] exe versions: %s\n' % exe_str)
2261 for handler in self._opener.handlers:
2262 if hasattr(handler, 'proxies'):
2263 proxy_map.update(handler.proxies)
2264 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
2266 if self.params.get('call_home', False):
2267 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
2268 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
2269 latest_version = self.urlopen(
2270 'https://yt-dl.org/latest/version').read().decode('utf-8')
2271 if version_tuple(latest_version) > version_tuple(__version__):
2272 self.report_warning(
2273 'You are using an outdated version (newest version: %s)! '
2274 'See https://yt-dl.org/update if you need help updating.' %
2277 def _setup_opener(self):
2278 timeout_val = self.params.get('socket_timeout')
2279 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
2281 opts_cookiefile = self.params.get('cookiefile')
2282 opts_proxy = self.params.get('proxy')
2284 if opts_cookiefile is None:
2285 self.cookiejar = compat_cookiejar.CookieJar()
2287 opts_cookiefile = expand_path(opts_cookiefile)
2288 self.cookiejar = compat_cookiejar.MozillaCookieJar(
2290 if os.access(opts_cookiefile, os.R_OK):
2291 self.cookiejar.load()
2293 cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
2294 if opts_proxy is not None:
2295 if opts_proxy == '':
2298 proxies = {'http': opts_proxy, 'https': opts_proxy}
2300 proxies = compat_urllib_request.getproxies()
2301 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
2302 if 'http' in proxies and 'https' not in proxies:
2303 proxies['https'] = proxies['http']
2304 proxy_handler = PerRequestProxyHandler(proxies)
2306 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
2307 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
2308 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
2309 data_handler = compat_urllib_request_DataHandler()
2311 # When passing our own FileHandler instance, build_opener won't add the
2312 # default FileHandler and allows us to disable the file protocol, which
2313 # can be used for malicious purposes (see
2314 # https://github.com/rg3/youtube-dl/issues/8227)
2315 file_handler = compat_urllib_request.FileHandler()
2317 def file_open(*args, **kwargs):
2318 raise compat_urllib_error.URLError('file:// scheme is explicitly disabled in youtube-dl for security reasons')
2319 file_handler.file_open = file_open
2321 opener = compat_urllib_request.build_opener(
2322 proxy_handler, https_handler, cookie_processor, ydlh, data_handler, file_handler)
2324 # Delete the default user-agent header, which would otherwise apply in
2325 # cases where our custom HTTP handler doesn't come into play
2326 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
2327 opener.addheaders = []
2328 self._opener = opener
2330 def encode(self, s):
2331 if isinstance(s, bytes):
2332 return s # Already encoded
2335 return s.encode(self.get_encoding())
2336 except UnicodeEncodeError as err:
2337 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
2340 def get_encoding(self):
2341 encoding = self.params.get('encoding')
2342 if encoding is None:
2343 encoding = preferredencoding()
2346 def _write_thumbnails(self, info_dict, filename):
2347 if self.params.get('writethumbnail', False):
2348 thumbnails = info_dict.get('thumbnails')
2350 thumbnails = [thumbnails[-1]]
2351 elif self.params.get('write_all_thumbnails', False):
2352 thumbnails = info_dict.get('thumbnails')
2357 # No thumbnails present, so return immediately
2360 for t in thumbnails:
2361 thumb_ext = determine_ext(t['url'], 'jpg')
2362 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2363 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2364 t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2366 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2367 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2368 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2370 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2371 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2373 uf = self.urlopen(t['url'])
2374 with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2375 shutil.copyfileobj(uf, thumbf)
2376 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2377 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2378 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2379 self.report_warning('Unable to download thumbnail "%s": %s' %
2380 (t['url'], error_to_compat_str(err)))