2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import, unicode_literals
33 compat_get_terminal_size,
38 compat_urllib_request,
58 PerRequestProxyHandler,
68 UnavailableVideoError,
79 from .cache import Cache
80 from .extractor import get_info_extractor, gen_extractors
81 from .downloader import get_suitable_downloader
82 from .downloader.rtmp import rtmpdump_version
83 from .postprocessor import (
85 FFmpegFixupStretchedPP,
90 from .version import __version__
93 class YoutubeDL(object):
96 YoutubeDL objects are the ones responsible of downloading the
97 actual video file and writing it to disk if the user has requested
98 it, among some other tasks. In most cases there should be one per
99 program. As, given a video URL, the downloader doesn't know how to
100 extract all the needed information, task that InfoExtractors do, it
101 has to pass the URL to one of them.
103 For this, YoutubeDL objects have a method that allows
104 InfoExtractors to be registered in a given order. When it is passed
105 a URL, the YoutubeDL object handles it to the first InfoExtractor it
106 finds that reports being able to handle it. The InfoExtractor extracts
107 all the information about the video or videos the URL refers to, and
108 YoutubeDL process the extracted information, possibly using a File
109 Downloader to download the video.
111 YoutubeDL objects accept a lot of parameters. In order not to saturate
112 the object constructor with arguments, it receives a dictionary of
113 options instead. These options are available through the params
114 attribute for the InfoExtractors to use. The YoutubeDL also
115 registers itself as the downloader in charge for the InfoExtractors
116 that are added to it, so this is a "mutual registration".
120 username: Username for authentication purposes.
121 password: Password for authentication purposes.
122 videopassword: Password for acces a video.
123 usenetrc: Use netrc for authentication instead.
124 verbose: Print additional info to stdout.
125 quiet: Do not print messages to stdout.
126 no_warnings: Do not print out anything for warnings.
127 forceurl: Force printing final URL.
128 forcetitle: Force printing title.
129 forceid: Force printing ID.
130 forcethumbnail: Force printing thumbnail URL.
131 forcedescription: Force printing description.
132 forcefilename: Force printing final filename.
133 forceduration: Force printing duration.
134 forcejson: Force printing info_dict as JSON.
135 dump_single_json: Force printing the info_dict of the whole playlist
136 (or video) as a single JSON line.
137 simulate: Do not download the video files.
138 format: Video format code. See options.py for more information.
139 outtmpl: Template for output names.
140 restrictfilenames: Do not allow "&" and spaces in file names
141 ignoreerrors: Do not stop on download errors.
142 force_generic_extractor: Force downloader to use the generic extractor
143 nooverwrites: Prevent overwriting files.
144 playliststart: Playlist item to start at.
145 playlistend: Playlist item to end at.
146 playlist_items: Specific indices of playlist to download.
147 playlistreverse: Download playlist items in reverse order.
148 matchtitle: Download only matching titles.
149 rejecttitle: Reject downloads for matching titles.
150 logger: Log messages to a logging.Logger instance.
151 logtostderr: Log messages to stderr instead of stdout.
152 writedescription: Write the video description to a .description file
153 writeinfojson: Write the video description to a .info.json file
154 writeannotations: Write the video annotations to a .annotations.xml file
155 writethumbnail: Write the thumbnail image to a file
156 write_all_thumbnails: Write all thumbnail formats to files
157 writesubtitles: Write the video subtitles to a file
158 writeautomaticsub: Write the automatic subtitles to a file
159 allsubtitles: Downloads all the subtitles of the video
160 (requires writesubtitles or writeautomaticsub)
161 listsubtitles: Lists all available subtitles for the video
162 subtitlesformat: The format code for subtitles
163 subtitleslangs: List of languages of the subtitles to download
164 keepvideo: Keep the video file after post-processing
165 daterange: A DateRange object, download only if the upload_date is in the range.
166 skip_download: Skip the actual download of the video file
167 cachedir: Location of the cache files in the filesystem.
168 False to disable filesystem cache.
169 noplaylist: Download single video instead of a playlist if in doubt.
170 age_limit: An integer representing the user's age in years.
171 Unsuitable videos for the given age are skipped.
172 min_views: An integer representing the minimum view count the video
173 must have in order to not be skipped.
174 Videos without view count information are always
175 downloaded. None for no limit.
176 max_views: An integer representing the maximum view count.
177 Videos that are more popular than that are not
179 Videos without view count information are always
180 downloaded. None for no limit.
181 download_archive: File name of a file where all downloads are recorded.
182 Videos already present in the file are not downloaded
184 cookiefile: File name where cookies should be read from and dumped to.
185 nocheckcertificate:Do not verify SSL certificates
186 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
187 At the moment, this is only supported by YouTube.
188 proxy: URL of the proxy server to use
189 cn_verification_proxy: URL of the proxy to use for IP address verification
190 on Chinese sites. (Experimental)
191 socket_timeout: Time to wait for unresponsive hosts, in seconds
192 bidi_workaround: Work around buggy terminals without bidirectional text
193 support, using fridibi
194 debug_printtraffic:Print out sent and received HTTP traffic
195 include_ads: Download ads as well
196 default_search: Prepend this string if an input url is not valid.
197 'auto' for elaborate guessing
198 encoding: Use this encoding instead of the system-specified.
199 extract_flat: Do not resolve URLs, return the immediate result.
200 Pass in 'in_playlist' to only show this behavior for
202 postprocessors: A list of dictionaries, each with an entry
203 * key: The name of the postprocessor. See
204 youtube_dl/postprocessor/__init__.py for a list.
205 as well as any further keyword arguments for the
207 progress_hooks: A list of functions that get called on download
208 progress, with a dictionary with the entries
209 * status: One of "downloading", "error", or "finished".
210 Check this first and ignore unknown values.
212 If status is one of "downloading", or "finished", the
213 following properties may also be present:
214 * filename: The final filename (always present)
215 * tmpfilename: The filename we're currently writing to
216 * downloaded_bytes: Bytes on disk
217 * total_bytes: Size of the whole file, None if unknown
218 * total_bytes_estimate: Guess of the eventual file size,
220 * elapsed: The number of seconds since download started.
221 * eta: The estimated time in seconds, None if unknown
222 * speed: The download speed in bytes/second, None if
224 * fragment_index: The counter of the currently
225 downloaded video fragment.
226 * fragment_count: The number of fragments (= individual
227 files that will be merged)
229 Progress hooks are guaranteed to be called at least once
230 (with status "finished") if the download is successful.
231 merge_output_format: Extension to use when merging formats.
232 fixup: Automatically correct known faults of the file.
234 - "never": do nothing
235 - "warn": only emit a warning
236 - "detect_or_warn": check whether we can do anything
237 about it, warn otherwise (default)
238 source_address: (Experimental) Client-side IP address to bind to.
239 call_home: Boolean, true iff we are allowed to contact the
240 youtube-dl servers for debugging.
241 sleep_interval: Number of seconds to sleep before each download.
242 listformats: Print an overview of available video formats and exit.
243 list_thumbnails: Print a table of all thumbnails and exit.
244 match_filter: A function that gets called with the info_dict of
246 If it returns a message, the video is ignored.
247 If it returns None, the video is downloaded.
248 match_filter_func in utils.py is one example for this.
249 no_color: Do not emit color codes in output.
251 The following options determine which downloader is picked:
252 external_downloader: Executable of the external downloader to call.
253 None or unset for standard (built-in) downloader.
254 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
256 The following parameters are not used by YoutubeDL itself, they are used by
257 the downloader (see youtube_dl/downloader/common.py):
258 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
259 noresizebuffer, retries, continuedl, noprogress, consoletitle,
260 xattr_set_filesize, external_downloader_args.
262 The following options are used by the post processors:
263 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
264 otherwise prefer avconv.
270 _download_retcode = None
271 _num_downloads = None
274 def __init__(self, params=None, auto_init=True):
275 """Create a FileDownloader object with the given options."""
279 self._ies_instances = {}
281 self._progress_hooks = []
282 self._download_retcode = 0
283 self._num_downloads = 0
284 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
285 self._err_file = sys.stderr
287 self.cache = Cache(self)
289 if params.get('bidi_workaround', False):
292 master, slave = pty.openpty()
293 width = compat_get_terminal_size().columns
297 width_args = ['-w', str(width)]
299 stdin=subprocess.PIPE,
301 stderr=self._err_file)
303 self._output_process = subprocess.Popen(
304 ['bidiv'] + width_args, **sp_kwargs
307 self._output_process = subprocess.Popen(
308 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
309 self._output_channel = os.fdopen(master, 'rb')
310 except OSError as ose:
312 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
316 if (sys.version_info >= (3,) and sys.platform != 'win32' and
317 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
318 not params.get('restrictfilenames', False)):
319 # On Python 3, the Unicode filesystem API will throw errors (#1474)
321 'Assuming --restrict-filenames since file system encoding '
322 'cannot encode all characters. '
323 'Set the LC_ALL environment variable to fix this.')
324 self.params['restrictfilenames'] = True
326 if isinstance(params.get('outtmpl'), bytes):
328 'Parameter outtmpl is bytes, but should be a unicode string. '
329 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
334 self.print_debug_header()
335 self.add_default_info_extractors()
337 for pp_def_raw in self.params.get('postprocessors', []):
338 pp_class = get_postprocessor(pp_def_raw['key'])
339 pp_def = dict(pp_def_raw)
341 pp = pp_class(self, **compat_kwargs(pp_def))
342 self.add_post_processor(pp)
344 for ph in self.params.get('progress_hooks', []):
345 self.add_progress_hook(ph)
347 def warn_if_short_id(self, argv):
348 # short YouTube ID starting with dash?
350 i for i, a in enumerate(argv)
351 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
355 [a for i, a in enumerate(argv) if i not in idxs] +
356 ['--'] + [argv[i] for i in idxs]
359 'Long argument string detected. '
360 'Use -- to separate parameters and URLs, like this:\n%s\n' %
361 args_to_str(correct_argv))
363 def add_info_extractor(self, ie):
364 """Add an InfoExtractor object to the end of the list."""
366 self._ies_instances[ie.ie_key()] = ie
367 ie.set_downloader(self)
369 def get_info_extractor(self, ie_key):
371 Get an instance of an IE with name ie_key, it will try to get one from
372 the _ies list, if there's no instance it will create a new one and add
373 it to the extractor list.
375 ie = self._ies_instances.get(ie_key)
377 ie = get_info_extractor(ie_key)()
378 self.add_info_extractor(ie)
381 def add_default_info_extractors(self):
383 Add the InfoExtractors returned by gen_extractors to the end of the list
385 for ie in gen_extractors():
386 self.add_info_extractor(ie)
388 def add_post_processor(self, pp):
389 """Add a PostProcessor object to the end of the chain."""
391 pp.set_downloader(self)
393 def add_progress_hook(self, ph):
394 """Add the progress hook (currently only for the file downloader)"""
395 self._progress_hooks.append(ph)
397 def _bidi_workaround(self, message):
398 if not hasattr(self, '_output_channel'):
401 assert hasattr(self, '_output_process')
402 assert isinstance(message, compat_str)
403 line_count = message.count('\n') + 1
404 self._output_process.stdin.write((message + '\n').encode('utf-8'))
405 self._output_process.stdin.flush()
406 res = ''.join(self._output_channel.readline().decode('utf-8')
407 for _ in range(line_count))
408 return res[:-len('\n')]
410 def to_screen(self, message, skip_eol=False):
411 """Print message to stdout if not in quiet mode."""
412 return self.to_stdout(message, skip_eol, check_quiet=True)
414 def _write_string(self, s, out=None):
415 write_string(s, out=out, encoding=self.params.get('encoding'))
417 def to_stdout(self, message, skip_eol=False, check_quiet=False):
418 """Print message to stdout if not in quiet mode."""
419 if self.params.get('logger'):
420 self.params['logger'].debug(message)
421 elif not check_quiet or not self.params.get('quiet', False):
422 message = self._bidi_workaround(message)
423 terminator = ['\n', ''][skip_eol]
424 output = message + terminator
426 self._write_string(output, self._screen_file)
428 def to_stderr(self, message):
429 """Print message to stderr."""
430 assert isinstance(message, compat_str)
431 if self.params.get('logger'):
432 self.params['logger'].error(message)
434 message = self._bidi_workaround(message)
435 output = message + '\n'
436 self._write_string(output, self._err_file)
438 def to_console_title(self, message):
439 if not self.params.get('consoletitle', False):
441 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
442 # c_wchar_p() might not be necessary if `message` is
443 # already of type unicode()
444 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
445 elif 'TERM' in os.environ:
446 self._write_string('\033]0;%s\007' % message, self._screen_file)
448 def save_console_title(self):
449 if not self.params.get('consoletitle', False):
451 if 'TERM' in os.environ:
452 # Save the title on stack
453 self._write_string('\033[22;0t', self._screen_file)
455 def restore_console_title(self):
456 if not self.params.get('consoletitle', False):
458 if 'TERM' in os.environ:
459 # Restore the title from stack
460 self._write_string('\033[23;0t', self._screen_file)
463 self.save_console_title()
466 def __exit__(self, *args):
467 self.restore_console_title()
469 if self.params.get('cookiefile') is not None:
470 self.cookiejar.save()
472 def trouble(self, message=None, tb=None):
473 """Determine action to take when a download problem appears.
475 Depending on if the downloader has been configured to ignore
476 download errors or not, this method may throw an exception or
477 not when errors are found, after printing the message.
479 tb, if given, is additional traceback information.
481 if message is not None:
482 self.to_stderr(message)
483 if self.params.get('verbose'):
485 if sys.exc_info()[0]: # if .trouble has been called from an except block
487 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
488 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
489 tb += compat_str(traceback.format_exc())
491 tb_data = traceback.format_list(traceback.extract_stack())
492 tb = ''.join(tb_data)
494 if not self.params.get('ignoreerrors', False):
495 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
496 exc_info = sys.exc_info()[1].exc_info
498 exc_info = sys.exc_info()
499 raise DownloadError(message, exc_info)
500 self._download_retcode = 1
502 def report_warning(self, message):
504 Print the message to stderr, it will be prefixed with 'WARNING:'
505 If stderr is a tty file the 'WARNING:' will be colored
507 if self.params.get('logger') is not None:
508 self.params['logger'].warning(message)
510 if self.params.get('no_warnings'):
512 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
513 _msg_header = '\033[0;33mWARNING:\033[0m'
515 _msg_header = 'WARNING:'
516 warning_message = '%s %s' % (_msg_header, message)
517 self.to_stderr(warning_message)
519 def report_error(self, message, tb=None):
521 Do the same as trouble, but prefixes the message with 'ERROR:', colored
522 in red if stderr is a tty file.
524 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
525 _msg_header = '\033[0;31mERROR:\033[0m'
527 _msg_header = 'ERROR:'
528 error_message = '%s %s' % (_msg_header, message)
529 self.trouble(error_message, tb)
531 def report_file_already_downloaded(self, file_name):
532 """Report file has already been fully downloaded."""
534 self.to_screen('[download] %s has already been downloaded' % file_name)
535 except UnicodeEncodeError:
536 self.to_screen('[download] The file has already been downloaded')
538 def prepare_filename(self, info_dict):
539 """Generate the output filename."""
541 template_dict = dict(info_dict)
543 template_dict['epoch'] = int(time.time())
544 autonumber_size = self.params.get('autonumber_size')
545 if autonumber_size is None:
547 autonumber_templ = '%0' + str(autonumber_size) + 'd'
548 template_dict['autonumber'] = autonumber_templ % self._num_downloads
549 if template_dict.get('playlist_index') is not None:
550 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
551 if template_dict.get('resolution') is None:
552 if template_dict.get('width') and template_dict.get('height'):
553 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
554 elif template_dict.get('height'):
555 template_dict['resolution'] = '%sp' % template_dict['height']
556 elif template_dict.get('width'):
557 template_dict['resolution'] = '?x%d' % template_dict['width']
559 sanitize = lambda k, v: sanitize_filename(
561 restricted=self.params.get('restrictfilenames'),
563 template_dict = dict((k, sanitize(k, v))
564 for k, v in template_dict.items()
566 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
568 outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
569 tmpl = compat_expanduser(outtmpl)
570 filename = tmpl % template_dict
571 # Temporary fix for #4787
572 # 'Treat' all problem characters by passing filename through preferredencoding
573 # to workaround encoding issues with subprocess on python2 @ Windows
574 if sys.version_info < (3, 0) and sys.platform == 'win32':
575 filename = encodeFilename(filename, True).decode(preferredencoding())
577 except ValueError as err:
578 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
581 def _match_entry(self, info_dict, incomplete):
582 """ Returns None iff the file should be downloaded """
584 video_title = info_dict.get('title', info_dict.get('id', 'video'))
585 if 'title' in info_dict:
586 # This can happen when we're just evaluating the playlist
587 title = info_dict['title']
588 matchtitle = self.params.get('matchtitle', False)
590 if not re.search(matchtitle, title, re.IGNORECASE):
591 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
592 rejecttitle = self.params.get('rejecttitle', False)
594 if re.search(rejecttitle, title, re.IGNORECASE):
595 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
596 date = info_dict.get('upload_date', None)
598 dateRange = self.params.get('daterange', DateRange())
599 if date not in dateRange:
600 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
601 view_count = info_dict.get('view_count', None)
602 if view_count is not None:
603 min_views = self.params.get('min_views')
604 if min_views is not None and view_count < min_views:
605 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
606 max_views = self.params.get('max_views')
607 if max_views is not None and view_count > max_views:
608 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
609 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
610 return 'Skipping "%s" because it is age restricted' % video_title
611 if self.in_download_archive(info_dict):
612 return '%s has already been recorded in archive' % video_title
615 match_filter = self.params.get('match_filter')
616 if match_filter is not None:
617 ret = match_filter(info_dict)
624 def add_extra_info(info_dict, extra_info):
625 '''Set the keys from extra_info in info dict if they are missing'''
626 for key, value in extra_info.items():
627 info_dict.setdefault(key, value)
629 def extract_info(self, url, download=True, ie_key=None, extra_info={},
630 process=True, force_generic_extractor=False):
632 Returns a list with a dictionary for each video we find.
633 If 'download', also downloads the videos.
634 extra_info is a dict containing the extra values to add to each result
637 if not ie_key and force_generic_extractor:
641 ies = [self.get_info_extractor(ie_key)]
646 if not ie.suitable(url):
650 self.report_warning('The program functionality for this site has been marked as broken, '
651 'and will probably not work.')
654 ie_result = ie.extract(url)
655 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
657 if isinstance(ie_result, list):
658 # Backwards compatibility: old IE result format
660 '_type': 'compat_list',
661 'entries': ie_result,
663 self.add_default_extra_info(ie_result, ie, url)
665 return self.process_ie_result(ie_result, download, extra_info, force_generic_extractor=False)
668 except ExtractorError as de: # An error we somewhat expected
669 self.report_error(compat_str(de), de.format_traceback())
671 except MaxDownloadsReached:
673 except Exception as e:
674 if self.params.get('ignoreerrors', False):
675 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
680 self.report_error('no suitable InfoExtractor for URL %s' % url)
682 def add_default_extra_info(self, ie_result, ie, url):
683 self.add_extra_info(ie_result, {
684 'extractor': ie.IE_NAME,
686 'webpage_url_basename': url_basename(url),
687 'extractor_key': ie.ie_key(),
690 def process_ie_result(self, ie_result, download=True, extra_info={}, force_generic_extractor=False):
692 Take the result of the ie(may be modified) and resolve all unresolved
693 references (URLs, playlist items).
695 It will also download the videos if 'download'.
696 Returns the resolved ie_result.
699 result_type = ie_result.get('_type', 'video')
701 if result_type in ('url', 'url_transparent'):
702 extract_flat = self.params.get('extract_flat', False)
703 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
704 extract_flat is True):
705 if self.params.get('forcejson', False):
706 self.to_stdout(json.dumps(ie_result))
709 if result_type == 'video':
710 self.add_extra_info(ie_result, extra_info)
711 return self.process_video_result(ie_result, download=download)
712 elif result_type == 'url':
713 # We have to add extra_info to the results because it may be
714 # contained in a playlist
715 return self.extract_info(ie_result['url'],
717 ie_key=ie_result.get('ie_key'),
718 extra_info=extra_info,
719 force_generic_extractor=force_generic_extractor)
720 elif result_type == 'url_transparent':
721 # Use the information from the embedding page
722 info = self.extract_info(
723 ie_result['url'], ie_key=ie_result.get('ie_key'),
724 extra_info=extra_info, download=False, process=False)
726 force_properties = dict(
727 (k, v) for k, v in ie_result.items() if v is not None)
728 for f in ('_type', 'url'):
729 if f in force_properties:
730 del force_properties[f]
731 new_result = info.copy()
732 new_result.update(force_properties)
734 assert new_result.get('_type') != 'url_transparent'
736 return self.process_ie_result(
737 new_result, download=download, extra_info=extra_info)
738 elif result_type == 'playlist' or result_type == 'multi_video':
739 # We process each entry in the playlist
740 playlist = ie_result.get('title', None) or ie_result.get('id', None)
741 self.to_screen('[download] Downloading playlist: %s' % playlist)
743 playlist_results = []
745 playliststart = self.params.get('playliststart', 1) - 1
746 playlistend = self.params.get('playlistend', None)
747 # For backwards compatibility, interpret -1 as whole list
748 if playlistend == -1:
751 playlistitems_str = self.params.get('playlist_items', None)
753 if playlistitems_str is not None:
754 def iter_playlistitems(format):
755 for string_segment in format.split(','):
756 if '-' in string_segment:
757 start, end = string_segment.split('-')
758 for item in range(int(start), int(end) + 1):
761 yield int(string_segment)
762 playlistitems = iter_playlistitems(playlistitems_str)
764 ie_entries = ie_result['entries']
765 if isinstance(ie_entries, list):
766 n_all_entries = len(ie_entries)
769 ie_entries[i - 1] for i in playlistitems
770 if -n_all_entries <= i - 1 < n_all_entries]
772 entries = ie_entries[playliststart:playlistend]
773 n_entries = len(entries)
775 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
776 (ie_result['extractor'], playlist, n_all_entries, n_entries))
777 elif isinstance(ie_entries, PagedList):
780 for item in playlistitems:
781 entries.extend(ie_entries.getslice(
785 entries = ie_entries.getslice(
786 playliststart, playlistend)
787 n_entries = len(entries)
789 "[%s] playlist %s: Downloading %d videos" %
790 (ie_result['extractor'], playlist, n_entries))
793 entry_list = list(ie_entries)
794 entries = [entry_list[i - 1] for i in playlistitems]
796 entries = list(itertools.islice(
797 ie_entries, playliststart, playlistend))
798 n_entries = len(entries)
800 "[%s] playlist %s: Downloading %d videos" %
801 (ie_result['extractor'], playlist, n_entries))
803 if self.params.get('playlistreverse', False):
804 entries = entries[::-1]
806 for i, entry in enumerate(entries, 1):
807 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
809 'n_entries': n_entries,
810 'playlist': playlist,
811 'playlist_id': ie_result.get('id'),
812 'playlist_title': ie_result.get('title'),
813 'playlist_index': i + playliststart,
814 'extractor': ie_result['extractor'],
815 'webpage_url': ie_result['webpage_url'],
816 'webpage_url_basename': url_basename(ie_result['webpage_url']),
817 'extractor_key': ie_result['extractor_key'],
820 reason = self._match_entry(entry, incomplete=True)
821 if reason is not None:
822 self.to_screen('[download] ' + reason)
825 entry_result = self.process_ie_result(entry,
828 playlist_results.append(entry_result)
829 ie_result['entries'] = playlist_results
831 elif result_type == 'compat_list':
833 'Extractor %s returned a compat_list result. '
834 'It needs to be updated.' % ie_result.get('extractor'))
840 'extractor': ie_result['extractor'],
841 'webpage_url': ie_result['webpage_url'],
842 'webpage_url_basename': url_basename(ie_result['webpage_url']),
843 'extractor_key': ie_result['extractor_key'],
847 ie_result['entries'] = [
848 self.process_ie_result(_fixup(r), download, extra_info)
849 for r in ie_result['entries']
853 raise Exception('Invalid result type: %s' % result_type)
855 def _apply_format_filter(self, format_spec, available_formats):
856 " Returns a tuple of the remaining format_spec and filtered formats "
866 operator_rex = re.compile(r'''(?x)\s*\[
867 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
868 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
869 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
871 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
872 m = operator_rex.search(format_spec)
875 comparison_value = int(m.group('value'))
877 comparison_value = parse_filesize(m.group('value'))
878 if comparison_value is None:
879 comparison_value = parse_filesize(m.group('value') + 'B')
880 if comparison_value is None:
882 'Invalid value %r in format specification %r' % (
883 m.group('value'), format_spec))
884 op = OPERATORS[m.group('op')]
891 str_operator_rex = re.compile(r'''(?x)\s*\[
892 \s*(?P<key>ext|acodec|vcodec|container|protocol)
893 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
894 \s*(?P<value>[a-zA-Z0-9_-]+)
896 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
897 m = str_operator_rex.search(format_spec)
899 comparison_value = m.group('value')
900 op = STR_OPERATORS[m.group('op')]
903 raise ValueError('Invalid format specification %r' % format_spec)
906 actual_value = f.get(m.group('key'))
907 if actual_value is None:
908 return m.group('none_inclusive')
909 return op(actual_value, comparison_value)
910 new_formats = [f for f in available_formats if _filter(f)]
912 new_format_spec = format_spec[:-len(m.group(0))]
913 if not new_format_spec:
914 new_format_spec = 'best'
916 return (new_format_spec, new_formats)
918 def select_format(self, format_spec, available_formats):
919 while format_spec.endswith(']'):
920 format_spec, available_formats = self._apply_format_filter(
921 format_spec, available_formats)
922 if not available_formats:
925 if format_spec in ['best', 'worst', None]:
926 format_idx = 0 if format_spec == 'worst' else -1
927 audiovideo_formats = [
928 f for f in available_formats
929 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
930 if audiovideo_formats:
931 return audiovideo_formats[format_idx]
932 # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
933 elif (all(f.get('acodec') != 'none' for f in available_formats) or
934 all(f.get('vcodec') != 'none' for f in available_formats)):
935 return available_formats[format_idx]
936 elif format_spec == 'bestaudio':
938 f for f in available_formats
939 if f.get('vcodec') == 'none']
941 return audio_formats[-1]
942 elif format_spec == 'worstaudio':
944 f for f in available_formats
945 if f.get('vcodec') == 'none']
947 return audio_formats[0]
948 elif format_spec == 'bestvideo':
950 f for f in available_formats
951 if f.get('acodec') == 'none']
953 return video_formats[-1]
954 elif format_spec == 'worstvideo':
956 f for f in available_formats
957 if f.get('acodec') == 'none']
959 return video_formats[0]
961 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
962 if format_spec in extensions:
963 filter_f = lambda f: f['ext'] == format_spec
965 filter_f = lambda f: f['format_id'] == format_spec
966 matches = list(filter(filter_f, available_formats))
971 def _calc_headers(self, info_dict):
972 res = std_headers.copy()
974 add_headers = info_dict.get('http_headers')
976 res.update(add_headers)
978 cookies = self._calc_cookies(info_dict)
980 res['Cookie'] = cookies
984 def _calc_cookies(self, info_dict):
985 pr = compat_urllib_request.Request(info_dict['url'])
986 self.cookiejar.add_cookie_header(pr)
987 return pr.get_header('Cookie')
989 def process_video_result(self, info_dict, download=True):
990 assert info_dict.get('_type', 'video') == 'video'
992 if 'id' not in info_dict:
993 raise ExtractorError('Missing "id" field in extractor result')
994 if 'title' not in info_dict:
995 raise ExtractorError('Missing "title" field in extractor result')
997 if 'playlist' not in info_dict:
998 # It isn't part of a playlist
999 info_dict['playlist'] = None
1000 info_dict['playlist_index'] = None
1002 thumbnails = info_dict.get('thumbnails')
1003 if thumbnails is None:
1004 thumbnail = info_dict.get('thumbnail')
1006 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1008 thumbnails.sort(key=lambda t: (
1009 t.get('preference'), t.get('width'), t.get('height'),
1010 t.get('id'), t.get('url')))
1011 for i, t in enumerate(thumbnails):
1012 if 'width' in t and 'height' in t:
1013 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1014 if t.get('id') is None:
1017 if thumbnails and 'thumbnail' not in info_dict:
1018 info_dict['thumbnail'] = thumbnails[-1]['url']
1020 if 'display_id' not in info_dict and 'id' in info_dict:
1021 info_dict['display_id'] = info_dict['id']
1023 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1024 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1025 # see http://bugs.python.org/issue1646728)
1027 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1028 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1029 except (ValueError, OverflowError, OSError):
1032 if self.params.get('listsubtitles', False):
1033 if 'automatic_captions' in info_dict:
1034 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1035 self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
1037 info_dict['requested_subtitles'] = self.process_subtitles(
1038 info_dict['id'], info_dict.get('subtitles'),
1039 info_dict.get('automatic_captions'))
1041 # This extractors handle format selection themselves
1042 if info_dict['extractor'] in ['Youku']:
1044 self.process_info(info_dict)
1047 # We now pick which formats have to be downloaded
1048 if info_dict.get('formats') is None:
1049 # There's only one format available
1050 formats = [info_dict]
1052 formats = info_dict['formats']
1055 raise ExtractorError('No video formats found!')
1059 # We check that all the formats have the format and format_id fields
1060 for i, format in enumerate(formats):
1061 if 'url' not in format:
1062 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1064 if format.get('format_id') is None:
1065 format['format_id'] = compat_str(i)
1066 format_id = format['format_id']
1067 if format_id not in formats_dict:
1068 formats_dict[format_id] = []
1069 formats_dict[format_id].append(format)
1071 # Make sure all formats have unique format_id
1072 for format_id, ambiguous_formats in formats_dict.items():
1073 if len(ambiguous_formats) > 1:
1074 for i, format in enumerate(ambiguous_formats):
1075 format['format_id'] = '%s-%d' % (format_id, i)
1077 for i, format in enumerate(formats):
1078 if format.get('format') is None:
1079 format['format'] = '{id} - {res}{note}'.format(
1080 id=format['format_id'],
1081 res=self.format_resolution(format),
1082 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1084 # Automatically determine file extension if missing
1085 if 'ext' not in format:
1086 format['ext'] = determine_ext(format['url']).lower()
1087 # Add HTTP headers, so that external programs can use them from the
1089 full_format_info = info_dict.copy()
1090 full_format_info.update(format)
1091 format['http_headers'] = self._calc_headers(full_format_info)
1093 # TODO Central sorting goes here
1095 if formats[0] is not info_dict:
1096 # only set the 'formats' fields if the original info_dict list them
1097 # otherwise we end up with a circular reference, the first (and unique)
1098 # element in the 'formats' field in info_dict is info_dict itself,
1099 # wich can't be exported to json
1100 info_dict['formats'] = formats
1101 if self.params.get('listformats'):
1102 self.list_formats(info_dict)
1104 if self.params.get('list_thumbnails'):
1105 self.list_thumbnails(info_dict)
1108 req_format = self.params.get('format')
1109 if req_format is None:
1110 req_format_list = []
1111 if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1112 info_dict['extractor'] in ['youtube', 'ted']):
1113 merger = FFmpegMergerPP(self)
1114 if merger.available and merger.can_merge():
1115 req_format_list.append('bestvideo+bestaudio')
1116 req_format_list.append('best')
1117 req_format = '/'.join(req_format_list)
1118 formats_to_download = []
1119 if req_format == 'all':
1120 formats_to_download = formats
1122 for rfstr in req_format.split(','):
1123 # We can accept formats requested in the format: 34/5/best, we pick
1124 # the first that is available, starting from left
1125 req_formats = rfstr.split('/')
1126 for rf in req_formats:
1127 if re.match(r'.+?\+.+?', rf) is not None:
1128 # Two formats have been requested like '137+139'
1129 format_1, format_2 = rf.split('+')
1130 formats_info = (self.select_format(format_1, formats),
1131 self.select_format(format_2, formats))
1132 if all(formats_info):
1133 # The first format must contain the video and the
1135 if formats_info[0].get('vcodec') == 'none':
1136 self.report_error('The first format must '
1137 'contain the video, try using '
1138 '"-f %s+%s"' % (format_2, format_1))
1141 formats_info[0]['ext']
1142 if self.params.get('merge_output_format') is None
1143 else self.params['merge_output_format'])
1145 'requested_formats': formats_info,
1146 'format': '%s+%s' % (formats_info[0].get('format'),
1147 formats_info[1].get('format')),
1148 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1149 formats_info[1].get('format_id')),
1150 'width': formats_info[0].get('width'),
1151 'height': formats_info[0].get('height'),
1152 'resolution': formats_info[0].get('resolution'),
1153 'fps': formats_info[0].get('fps'),
1154 'vcodec': formats_info[0].get('vcodec'),
1155 'vbr': formats_info[0].get('vbr'),
1156 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1157 'acodec': formats_info[1].get('acodec'),
1158 'abr': formats_info[1].get('abr'),
1162 selected_format = None
1164 selected_format = self.select_format(rf, formats)
1165 if selected_format is not None:
1166 formats_to_download.append(selected_format)
1168 if not formats_to_download:
1169 raise ExtractorError('requested format not available',
1173 if len(formats_to_download) > 1:
1174 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1175 for format in formats_to_download:
1176 new_info = dict(info_dict)
1177 new_info.update(format)
1178 self.process_info(new_info)
1179 # We update the info dict with the best quality format (backwards compatibility)
1180 info_dict.update(formats_to_download[-1])
1183 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1184 """Select the requested subtitles and their format"""
1186 if normal_subtitles and self.params.get('writesubtitles'):
1187 available_subs.update(normal_subtitles)
1188 if automatic_captions and self.params.get('writeautomaticsub'):
1189 for lang, cap_info in automatic_captions.items():
1190 if lang not in available_subs:
1191 available_subs[lang] = cap_info
1193 if (not self.params.get('writesubtitles') and not
1194 self.params.get('writeautomaticsub') or not
1198 if self.params.get('allsubtitles', False):
1199 requested_langs = available_subs.keys()
1201 if self.params.get('subtitleslangs', False):
1202 requested_langs = self.params.get('subtitleslangs')
1203 elif 'en' in available_subs:
1204 requested_langs = ['en']
1206 requested_langs = [list(available_subs.keys())[0]]
1208 formats_query = self.params.get('subtitlesformat', 'best')
1209 formats_preference = formats_query.split('/') if formats_query else []
1211 for lang in requested_langs:
1212 formats = available_subs.get(lang)
1214 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1216 for ext in formats_preference:
1220 matches = list(filter(lambda f: f['ext'] == ext, formats))
1226 self.report_warning(
1227 'No subtitle format found matching "%s" for language %s, '
1228 'using %s' % (formats_query, lang, f['ext']))
1232 def process_info(self, info_dict):
1233 """Process a single resolved IE result."""
1235 assert info_dict.get('_type', 'video') == 'video'
1237 max_downloads = self.params.get('max_downloads')
1238 if max_downloads is not None:
1239 if self._num_downloads >= int(max_downloads):
1240 raise MaxDownloadsReached()
1242 info_dict['fulltitle'] = info_dict['title']
1243 if len(info_dict['title']) > 200:
1244 info_dict['title'] = info_dict['title'][:197] + '...'
1246 if 'format' not in info_dict:
1247 info_dict['format'] = info_dict['ext']
1249 reason = self._match_entry(info_dict, incomplete=False)
1250 if reason is not None:
1251 self.to_screen('[download] ' + reason)
1254 self._num_downloads += 1
1256 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1259 if self.params.get('forcetitle', False):
1260 self.to_stdout(info_dict['fulltitle'])
1261 if self.params.get('forceid', False):
1262 self.to_stdout(info_dict['id'])
1263 if self.params.get('forceurl', False):
1264 if info_dict.get('requested_formats') is not None:
1265 for f in info_dict['requested_formats']:
1266 self.to_stdout(f['url'] + f.get('play_path', ''))
1268 # For RTMP URLs, also include the playpath
1269 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1270 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1271 self.to_stdout(info_dict['thumbnail'])
1272 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1273 self.to_stdout(info_dict['description'])
1274 if self.params.get('forcefilename', False) and filename is not None:
1275 self.to_stdout(filename)
1276 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1277 self.to_stdout(formatSeconds(info_dict['duration']))
1278 if self.params.get('forceformat', False):
1279 self.to_stdout(info_dict['format'])
1280 if self.params.get('forcejson', False):
1281 self.to_stdout(json.dumps(info_dict))
1283 # Do nothing else if in simulate mode
1284 if self.params.get('simulate', False):
1287 if filename is None:
1291 dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1292 if dn and not os.path.exists(dn):
1294 except (OSError, IOError) as err:
1295 self.report_error('unable to create directory ' + compat_str(err))
1298 if self.params.get('writedescription', False):
1299 descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1300 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1301 self.to_screen('[info] Video description is already present')
1302 elif info_dict.get('description') is None:
1303 self.report_warning('There\'s no description to write.')
1306 self.to_screen('[info] Writing video description to: ' + descfn)
1307 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1308 descfile.write(info_dict['description'])
1309 except (OSError, IOError):
1310 self.report_error('Cannot write description file ' + descfn)
1313 if self.params.get('writeannotations', False):
1314 annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1315 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1316 self.to_screen('[info] Video annotations are already present')
1319 self.to_screen('[info] Writing video annotations to: ' + annofn)
1320 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1321 annofile.write(info_dict['annotations'])
1322 except (KeyError, TypeError):
1323 self.report_warning('There are no annotations to write.')
1324 except (OSError, IOError):
1325 self.report_error('Cannot write annotations file: ' + annofn)
1328 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1329 self.params.get('writeautomaticsub')])
1331 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1332 # subtitles download errors are already managed as troubles in relevant IE
1333 # that way it will silently go on when used with unsupporting IE
1334 subtitles = info_dict['requested_subtitles']
1335 ie = self.get_info_extractor(info_dict['extractor_key'])
1336 for sub_lang, sub_info in subtitles.items():
1337 sub_format = sub_info['ext']
1338 if sub_info.get('data') is not None:
1339 sub_data = sub_info['data']
1342 sub_data = ie._download_webpage(
1343 sub_info['url'], info_dict['id'], note=False)
1344 except ExtractorError as err:
1345 self.report_warning('Unable to download subtitle for "%s": %s' %
1346 (sub_lang, compat_str(err.cause)))
1349 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1350 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1351 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1353 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1354 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1355 subfile.write(sub_data)
1356 except (OSError, IOError):
1357 self.report_error('Cannot write subtitles file ' + sub_filename)
1360 if self.params.get('writeinfojson', False):
1361 infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1362 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1363 self.to_screen('[info] Video description metadata is already present')
1365 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1367 write_json_file(self.filter_requested_info(info_dict), infofn)
1368 except (OSError, IOError):
1369 self.report_error('Cannot write metadata to JSON file ' + infofn)
1372 self._write_thumbnails(info_dict, filename)
1374 if not self.params.get('skip_download', False):
1377 fd = get_suitable_downloader(info, self.params)(self, self.params)
1378 for ph in self._progress_hooks:
1379 fd.add_progress_hook(ph)
1380 if self.params.get('verbose'):
1381 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1382 return fd.download(name, info)
1384 if info_dict.get('requested_formats') is not None:
1387 merger = FFmpegMergerPP(self)
1388 if not merger.available:
1390 self.report_warning('You have requested multiple '
1391 'formats but ffmpeg or avconv are not installed.'
1392 ' The formats won\'t be merged.')
1394 postprocessors = [merger]
1396 def compatible_formats(formats):
1397 video, audio = formats
1399 video_ext, audio_ext = audio.get('ext'), video.get('ext')
1400 if video_ext and audio_ext:
1402 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1405 for exts in COMPATIBLE_EXTS:
1406 if video_ext in exts and audio_ext in exts:
1408 # TODO: Check acodec/vcodec
1411 filename_real_ext = os.path.splitext(filename)[1][1:]
1413 os.path.splitext(filename)[0]
1414 if filename_real_ext == info_dict['ext']
1416 requested_formats = info_dict['requested_formats']
1417 if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1418 info_dict['ext'] = 'mkv'
1419 self.report_warning(
1420 'Requested formats are incompatible for merge and will be merged into mkv.')
1421 # Ensure filename always has a correct extension for successful merge
1422 filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1423 if os.path.exists(encodeFilename(filename)):
1425 '[download] %s has already been downloaded and '
1426 'merged' % filename)
1428 for f in requested_formats:
1429 new_info = dict(info_dict)
1431 fname = self.prepare_filename(new_info)
1432 fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1433 downloaded.append(fname)
1434 partial_success = dl(fname, new_info)
1435 success = success and partial_success
1436 info_dict['__postprocessors'] = postprocessors
1437 info_dict['__files_to_merge'] = downloaded
1439 # Just a single file
1440 success = dl(filename, info_dict)
1441 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1442 self.report_error('unable to download video data: %s' % str(err))
1444 except (OSError, IOError) as err:
1445 raise UnavailableVideoError(err)
1446 except (ContentTooShortError, ) as err:
1447 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1452 fixup_policy = self.params.get('fixup')
1453 if fixup_policy is None:
1454 fixup_policy = 'detect_or_warn'
1456 stretched_ratio = info_dict.get('stretched_ratio')
1457 if stretched_ratio is not None and stretched_ratio != 1:
1458 if fixup_policy == 'warn':
1459 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1460 info_dict['id'], stretched_ratio))
1461 elif fixup_policy == 'detect_or_warn':
1462 stretched_pp = FFmpegFixupStretchedPP(self)
1463 if stretched_pp.available:
1464 info_dict.setdefault('__postprocessors', [])
1465 info_dict['__postprocessors'].append(stretched_pp)
1467 self.report_warning(
1468 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1469 info_dict['id'], stretched_ratio))
1471 assert fixup_policy in ('ignore', 'never')
1473 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1474 if fixup_policy == 'warn':
1475 self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1477 elif fixup_policy == 'detect_or_warn':
1478 fixup_pp = FFmpegFixupM4aPP(self)
1479 if fixup_pp.available:
1480 info_dict.setdefault('__postprocessors', [])
1481 info_dict['__postprocessors'].append(fixup_pp)
1483 self.report_warning(
1484 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1487 assert fixup_policy in ('ignore', 'never')
1490 self.post_process(filename, info_dict)
1491 except (PostProcessingError) as err:
1492 self.report_error('postprocessing: %s' % str(err))
1494 self.record_download_archive(info_dict)
1496 def download(self, url_list):
1497 """Download a given list of URLs."""
1498 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1499 if (len(url_list) > 1 and
1500 '%' not in outtmpl and
1501 self.params.get('max_downloads') != 1):
1502 raise SameFileError(outtmpl)
1504 for url in url_list:
1506 # It also downloads the videos
1507 res = self.extract_info(
1508 url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1509 except UnavailableVideoError:
1510 self.report_error('unable to download video')
1511 except MaxDownloadsReached:
1512 self.to_screen('[info] Maximum number of downloaded files reached.')
1515 if self.params.get('dump_single_json', False):
1516 self.to_stdout(json.dumps(res))
1518 return self._download_retcode
1520 def download_with_info_file(self, info_filename):
1521 with contextlib.closing(fileinput.FileInput(
1522 [info_filename], mode='r',
1523 openhook=fileinput.hook_encoded('utf-8'))) as f:
1524 # FileInput doesn't have a read method, we can't call json.load
1525 info = self.filter_requested_info(json.loads('\n'.join(f)))
1527 self.process_ie_result(info, download=True)
1528 except DownloadError:
1529 webpage_url = info.get('webpage_url')
1530 if webpage_url is not None:
1531 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1532 return self.download([webpage_url])
1535 return self._download_retcode
1538 def filter_requested_info(info_dict):
1540 (k, v) for k, v in info_dict.items()
1541 if k not in ['requested_formats', 'requested_subtitles'])
1543 def post_process(self, filename, ie_info):
1544 """Run all the postprocessors on the given file."""
1545 info = dict(ie_info)
1546 info['filepath'] = filename
1548 if ie_info.get('__postprocessors') is not None:
1549 pps_chain.extend(ie_info['__postprocessors'])
1550 pps_chain.extend(self._pps)
1551 for pp in pps_chain:
1552 files_to_delete = []
1554 files_to_delete, info = pp.run(info)
1555 except PostProcessingError as e:
1556 self.report_error(e.msg)
1557 if files_to_delete and not self.params.get('keepvideo', False):
1558 for old_filename in files_to_delete:
1559 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1561 os.remove(encodeFilename(old_filename))
1562 except (IOError, OSError):
1563 self.report_warning('Unable to remove downloaded original file')
1565 def _make_archive_id(self, info_dict):
1566 # Future-proof against any change in case
1567 # and backwards compatibility with prior versions
1568 extractor = info_dict.get('extractor_key')
1569 if extractor is None:
1570 if 'id' in info_dict:
1571 extractor = info_dict.get('ie_key') # key in a playlist
1572 if extractor is None:
1573 return None # Incomplete video information
1574 return extractor.lower() + ' ' + info_dict['id']
1576 def in_download_archive(self, info_dict):
1577 fn = self.params.get('download_archive')
1581 vid_id = self._make_archive_id(info_dict)
1583 return False # Incomplete video information
1586 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1587 for line in archive_file:
1588 if line.strip() == vid_id:
1590 except IOError as ioe:
1591 if ioe.errno != errno.ENOENT:
1595 def record_download_archive(self, info_dict):
1596 fn = self.params.get('download_archive')
1599 vid_id = self._make_archive_id(info_dict)
1601 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1602 archive_file.write(vid_id + '\n')
1605 def format_resolution(format, default='unknown'):
1606 if format.get('vcodec') == 'none':
1608 if format.get('resolution') is not None:
1609 return format['resolution']
1610 if format.get('height') is not None:
1611 if format.get('width') is not None:
1612 res = '%sx%s' % (format['width'], format['height'])
1614 res = '%sp' % format['height']
1615 elif format.get('width') is not None:
1616 res = '?x%d' % format['width']
1621 def _format_note(self, fdict):
1623 if fdict.get('ext') in ['f4f', 'f4m']:
1624 res += '(unsupported) '
1625 if fdict.get('format_note') is not None:
1626 res += fdict['format_note'] + ' '
1627 if fdict.get('tbr') is not None:
1628 res += '%4dk ' % fdict['tbr']
1629 if fdict.get('container') is not None:
1632 res += '%s container' % fdict['container']
1633 if (fdict.get('vcodec') is not None and
1634 fdict.get('vcodec') != 'none'):
1637 res += fdict['vcodec']
1638 if fdict.get('vbr') is not None:
1640 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1642 if fdict.get('vbr') is not None:
1643 res += '%4dk' % fdict['vbr']
1644 if fdict.get('fps') is not None:
1645 res += ', %sfps' % fdict['fps']
1646 if fdict.get('acodec') is not None:
1649 if fdict['acodec'] == 'none':
1652 res += '%-5s' % fdict['acodec']
1653 elif fdict.get('abr') is not None:
1657 if fdict.get('abr') is not None:
1658 res += '@%3dk' % fdict['abr']
1659 if fdict.get('asr') is not None:
1660 res += ' (%5dHz)' % fdict['asr']
1661 if fdict.get('filesize') is not None:
1664 res += format_bytes(fdict['filesize'])
1665 elif fdict.get('filesize_approx') is not None:
1668 res += '~' + format_bytes(fdict['filesize_approx'])
1671 def list_formats(self, info_dict):
1672 formats = info_dict.get('formats', [info_dict])
1674 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1676 if f.get('preference') is None or f['preference'] >= -1000]
1677 if len(formats) > 1:
1678 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1680 header_line = ['format code', 'extension', 'resolution', 'note']
1682 '[info] Available formats for %s:\n%s' %
1683 (info_dict['id'], render_table(header_line, table)))
1685 def list_thumbnails(self, info_dict):
1686 thumbnails = info_dict.get('thumbnails')
1688 tn_url = info_dict.get('thumbnail')
1690 thumbnails = [{'id': '0', 'url': tn_url}]
1693 '[info] No thumbnails present for %s' % info_dict['id'])
1697 '[info] Thumbnails for %s:' % info_dict['id'])
1698 self.to_screen(render_table(
1699 ['ID', 'width', 'height', 'URL'],
1700 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1702 def list_subtitles(self, video_id, subtitles, name='subtitles'):
1704 self.to_screen('%s has no %s' % (video_id, name))
1707 'Available %s for %s:' % (name, video_id))
1708 self.to_screen(render_table(
1709 ['Language', 'formats'],
1710 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1711 for lang, formats in subtitles.items()]))
1713 def urlopen(self, req):
1714 """ Start an HTTP download """
1716 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1717 # always respected by websites, some tend to give out URLs with non percent-encoded
1718 # non-ASCII characters (see telemb.py, ard.py [#3412])
1719 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1720 # To work around aforementioned issue we will replace request's original URL with
1721 # percent-encoded one
1722 req_is_string = isinstance(req, compat_basestring)
1723 url = req if req_is_string else req.get_full_url()
1724 url_escaped = escape_url(url)
1726 # Substitute URL if any change after escaping
1727 if url != url_escaped:
1731 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
1733 url_escaped, data=req.data, headers=req.headers,
1734 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1736 return self._opener.open(req, timeout=self._socket_timeout)
1738 def print_debug_header(self):
1739 if not self.params.get('verbose'):
1742 if type('') is not compat_str:
1743 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1744 self.report_warning(
1745 'Your Python is broken! Update to a newer and supported version')
1747 stdout_encoding = getattr(
1748 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1750 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1751 locale.getpreferredencoding(),
1752 sys.getfilesystemencoding(),
1754 self.get_encoding()))
1755 write_string(encoding_str, encoding=None)
1757 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1759 sp = subprocess.Popen(
1760 ['git', 'rev-parse', '--short', 'HEAD'],
1761 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1762 cwd=os.path.dirname(os.path.abspath(__file__)))
1763 out, err = sp.communicate()
1764 out = out.decode().strip()
1765 if re.match('[0-9a-f]+', out):
1766 self._write_string('[debug] Git HEAD: ' + out + '\n')
1772 self._write_string('[debug] Python version %s - %s\n' % (
1773 platform.python_version(), platform_name()))
1775 exe_versions = FFmpegPostProcessor.get_versions(self)
1776 exe_versions['rtmpdump'] = rtmpdump_version()
1777 exe_str = ', '.join(
1779 for exe, v in sorted(exe_versions.items())
1784 self._write_string('[debug] exe versions: %s\n' % exe_str)
1787 for handler in self._opener.handlers:
1788 if hasattr(handler, 'proxies'):
1789 proxy_map.update(handler.proxies)
1790 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1792 if self.params.get('call_home', False):
1793 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1794 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1795 latest_version = self.urlopen(
1796 'https://yt-dl.org/latest/version').read().decode('utf-8')
1797 if version_tuple(latest_version) > version_tuple(__version__):
1798 self.report_warning(
1799 'You are using an outdated version (newest version: %s)! '
1800 'See https://yt-dl.org/update if you need help updating.' %
1803 def _setup_opener(self):
1804 timeout_val = self.params.get('socket_timeout')
1805 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1807 opts_cookiefile = self.params.get('cookiefile')
1808 opts_proxy = self.params.get('proxy')
1810 if opts_cookiefile is None:
1811 self.cookiejar = compat_cookiejar.CookieJar()
1813 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1815 if os.access(opts_cookiefile, os.R_OK):
1816 self.cookiejar.load()
1818 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1820 if opts_proxy is not None:
1821 if opts_proxy == '':
1824 proxies = {'http': opts_proxy, 'https': opts_proxy}
1826 proxies = compat_urllib_request.getproxies()
1827 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1828 if 'http' in proxies and 'https' not in proxies:
1829 proxies['https'] = proxies['http']
1830 proxy_handler = PerRequestProxyHandler(proxies)
1832 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1833 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1834 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1835 opener = compat_urllib_request.build_opener(
1836 proxy_handler, https_handler, cookie_processor, ydlh)
1838 # Delete the default user-agent header, which would otherwise apply in
1839 # cases where our custom HTTP handler doesn't come into play
1840 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1841 opener.addheaders = []
1842 self._opener = opener
1844 def encode(self, s):
1845 if isinstance(s, bytes):
1846 return s # Already encoded
1849 return s.encode(self.get_encoding())
1850 except UnicodeEncodeError as err:
1851 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1854 def get_encoding(self):
1855 encoding = self.params.get('encoding')
1856 if encoding is None:
1857 encoding = preferredencoding()
1860 def _write_thumbnails(self, info_dict, filename):
1861 if self.params.get('writethumbnail', False):
1862 thumbnails = info_dict.get('thumbnails')
1864 thumbnails = [thumbnails[-1]]
1865 elif self.params.get('write_all_thumbnails', False):
1866 thumbnails = info_dict.get('thumbnails')
1871 # No thumbnails present, so return immediately
1874 for t in thumbnails:
1875 thumb_ext = determine_ext(t['url'], 'jpg')
1876 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
1877 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
1878 t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
1880 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
1881 self.to_screen('[%s] %s: Thumbnail %sis already present' %
1882 (info_dict['extractor'], info_dict['id'], thumb_display_id))
1884 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
1885 (info_dict['extractor'], info_dict['id'], thumb_display_id))
1887 uf = self.urlopen(t['url'])
1888 with open(thumb_filename, 'wb') as thumbf:
1889 shutil.copyfileobj(uf, thumbf)
1890 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
1891 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
1892 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1893 self.report_warning('Unable to download thumbnail "%s": %s' %
1894 (t['url'], compat_str(err)))