2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import, unicode_literals
34 compat_get_terminal_size,
38 compat_tokenize_tokenize,
40 compat_urllib_request,
60 PerRequestProxyHandler,
70 UnavailableVideoError,
81 from .cache import Cache
82 from .extractor import get_info_extractor, gen_extractors
83 from .downloader import get_suitable_downloader
84 from .downloader.rtmp import rtmpdump_version
85 from .postprocessor import (
87 FFmpegFixupStretchedPP,
92 from .version import __version__
95 class YoutubeDL(object):
98 YoutubeDL objects are the ones responsible of downloading the
99 actual video file and writing it to disk if the user has requested
100 it, among some other tasks. In most cases there should be one per
101 program. As, given a video URL, the downloader doesn't know how to
102 extract all the needed information, task that InfoExtractors do, it
103 has to pass the URL to one of them.
105 For this, YoutubeDL objects have a method that allows
106 InfoExtractors to be registered in a given order. When it is passed
107 a URL, the YoutubeDL object handles it to the first InfoExtractor it
108 finds that reports being able to handle it. The InfoExtractor extracts
109 all the information about the video or videos the URL refers to, and
110 YoutubeDL process the extracted information, possibly using a File
111 Downloader to download the video.
113 YoutubeDL objects accept a lot of parameters. In order not to saturate
114 the object constructor with arguments, it receives a dictionary of
115 options instead. These options are available through the params
116 attribute for the InfoExtractors to use. The YoutubeDL also
117 registers itself as the downloader in charge for the InfoExtractors
118 that are added to it, so this is a "mutual registration".
122 username: Username for authentication purposes.
123 password: Password for authentication purposes.
124 videopassword: Password for accessing a video.
125 usenetrc: Use netrc for authentication instead.
126 verbose: Print additional info to stdout.
127 quiet: Do not print messages to stdout.
128 no_warnings: Do not print out anything for warnings.
129 forceurl: Force printing final URL.
130 forcetitle: Force printing title.
131 forceid: Force printing ID.
132 forcethumbnail: Force printing thumbnail URL.
133 forcedescription: Force printing description.
134 forcefilename: Force printing final filename.
135 forceduration: Force printing duration.
136 forcejson: Force printing info_dict as JSON.
137 dump_single_json: Force printing the info_dict of the whole playlist
138 (or video) as a single JSON line.
139 simulate: Do not download the video files.
140 format: Video format code. See options.py for more information.
141 outtmpl: Template for output names.
142 restrictfilenames: Do not allow "&" and spaces in file names
143 ignoreerrors: Do not stop on download errors.
144 force_generic_extractor: Force downloader to use the generic extractor
145 nooverwrites: Prevent overwriting files.
146 playliststart: Playlist item to start at.
147 playlistend: Playlist item to end at.
148 playlist_items: Specific indices of playlist to download.
149 playlistreverse: Download playlist items in reverse order.
150 matchtitle: Download only matching titles.
151 rejecttitle: Reject downloads for matching titles.
152 logger: Log messages to a logging.Logger instance.
153 logtostderr: Log messages to stderr instead of stdout.
154 writedescription: Write the video description to a .description file
155 writeinfojson: Write the video description to a .info.json file
156 writeannotations: Write the video annotations to a .annotations.xml file
157 writethumbnail: Write the thumbnail image to a file
158 write_all_thumbnails: Write all thumbnail formats to files
159 writesubtitles: Write the video subtitles to a file
160 writeautomaticsub: Write the automatic subtitles to a file
161 allsubtitles: Downloads all the subtitles of the video
162 (requires writesubtitles or writeautomaticsub)
163 listsubtitles: Lists all available subtitles for the video
164 subtitlesformat: The format code for subtitles
165 subtitleslangs: List of languages of the subtitles to download
166 keepvideo: Keep the video file after post-processing
167 daterange: A DateRange object, download only if the upload_date is in the range.
168 skip_download: Skip the actual download of the video file
169 cachedir: Location of the cache files in the filesystem.
170 False to disable filesystem cache.
171 noplaylist: Download single video instead of a playlist if in doubt.
172 age_limit: An integer representing the user's age in years.
173 Unsuitable videos for the given age are skipped.
174 min_views: An integer representing the minimum view count the video
175 must have in order to not be skipped.
176 Videos without view count information are always
177 downloaded. None for no limit.
178 max_views: An integer representing the maximum view count.
179 Videos that are more popular than that are not
181 Videos without view count information are always
182 downloaded. None for no limit.
183 download_archive: File name of a file where all downloads are recorded.
184 Videos already present in the file are not downloaded
186 cookiefile: File name where cookies should be read from and dumped to.
187 nocheckcertificate:Do not verify SSL certificates
188 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
189 At the moment, this is only supported by YouTube.
190 proxy: URL of the proxy server to use
191 cn_verification_proxy: URL of the proxy to use for IP address verification
192 on Chinese sites. (Experimental)
193 socket_timeout: Time to wait for unresponsive hosts, in seconds
194 bidi_workaround: Work around buggy terminals without bidirectional text
195 support, using fridibi
196 debug_printtraffic:Print out sent and received HTTP traffic
197 include_ads: Download ads as well
198 default_search: Prepend this string if an input url is not valid.
199 'auto' for elaborate guessing
200 encoding: Use this encoding instead of the system-specified.
201 extract_flat: Do not resolve URLs, return the immediate result.
202 Pass in 'in_playlist' to only show this behavior for
204 postprocessors: A list of dictionaries, each with an entry
205 * key: The name of the postprocessor. See
206 youtube_dl/postprocessor/__init__.py for a list.
207 as well as any further keyword arguments for the
209 progress_hooks: A list of functions that get called on download
210 progress, with a dictionary with the entries
211 * status: One of "downloading", "error", or "finished".
212 Check this first and ignore unknown values.
214 If status is one of "downloading", or "finished", the
215 following properties may also be present:
216 * filename: The final filename (always present)
217 * tmpfilename: The filename we're currently writing to
218 * downloaded_bytes: Bytes on disk
219 * total_bytes: Size of the whole file, None if unknown
220 * total_bytes_estimate: Guess of the eventual file size,
222 * elapsed: The number of seconds since download started.
223 * eta: The estimated time in seconds, None if unknown
224 * speed: The download speed in bytes/second, None if
226 * fragment_index: The counter of the currently
227 downloaded video fragment.
228 * fragment_count: The number of fragments (= individual
229 files that will be merged)
231 Progress hooks are guaranteed to be called at least once
232 (with status "finished") if the download is successful.
233 merge_output_format: Extension to use when merging formats.
234 fixup: Automatically correct known faults of the file.
236 - "never": do nothing
237 - "warn": only emit a warning
238 - "detect_or_warn": check whether we can do anything
239 about it, warn otherwise (default)
240 source_address: (Experimental) Client-side IP address to bind to.
241 call_home: Boolean, true iff we are allowed to contact the
242 youtube-dl servers for debugging.
243 sleep_interval: Number of seconds to sleep before each download.
244 listformats: Print an overview of available video formats and exit.
245 list_thumbnails: Print a table of all thumbnails and exit.
246 match_filter: A function that gets called with the info_dict of
248 If it returns a message, the video is ignored.
249 If it returns None, the video is downloaded.
250 match_filter_func in utils.py is one example for this.
251 no_color: Do not emit color codes in output.
253 The following options determine which downloader is picked:
254 external_downloader: Executable of the external downloader to call.
255 None or unset for standard (built-in) downloader.
256 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
258 The following parameters are not used by YoutubeDL itself, they are used by
259 the downloader (see youtube_dl/downloader/common.py):
260 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
261 noresizebuffer, retries, continuedl, noprogress, consoletitle,
262 xattr_set_filesize, external_downloader_args.
264 The following options are used by the post processors:
265 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
266 otherwise prefer avconv.
267 postprocessor_args: A list of additional command-line arguments for the
274 _download_retcode = None
275 _num_downloads = None
278 def __init__(self, params=None, auto_init=True):
279 """Create a FileDownloader object with the given options."""
283 self._ies_instances = {}
285 self._progress_hooks = []
286 self._download_retcode = 0
287 self._num_downloads = 0
288 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
289 self._err_file = sys.stderr
291 self.cache = Cache(self)
293 if params.get('bidi_workaround', False):
296 master, slave = pty.openpty()
297 width = compat_get_terminal_size().columns
301 width_args = ['-w', str(width)]
303 stdin=subprocess.PIPE,
305 stderr=self._err_file)
307 self._output_process = subprocess.Popen(
308 ['bidiv'] + width_args, **sp_kwargs
311 self._output_process = subprocess.Popen(
312 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
313 self._output_channel = os.fdopen(master, 'rb')
314 except OSError as ose:
316 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
320 if (sys.version_info >= (3,) and sys.platform != 'win32' and
321 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
322 not params.get('restrictfilenames', False)):
323 # On Python 3, the Unicode filesystem API will throw errors (#1474)
325 'Assuming --restrict-filenames since file system encoding '
326 'cannot encode all characters. '
327 'Set the LC_ALL environment variable to fix this.')
328 self.params['restrictfilenames'] = True
330 if isinstance(params.get('outtmpl'), bytes):
332 'Parameter outtmpl is bytes, but should be a unicode string. '
333 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
338 self.print_debug_header()
339 self.add_default_info_extractors()
341 for pp_def_raw in self.params.get('postprocessors', []):
342 pp_class = get_postprocessor(pp_def_raw['key'])
343 pp_def = dict(pp_def_raw)
345 pp = pp_class(self, **compat_kwargs(pp_def))
346 self.add_post_processor(pp)
348 for ph in self.params.get('progress_hooks', []):
349 self.add_progress_hook(ph)
351 def warn_if_short_id(self, argv):
352 # short YouTube ID starting with dash?
354 i for i, a in enumerate(argv)
355 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
359 [a for i, a in enumerate(argv) if i not in idxs] +
360 ['--'] + [argv[i] for i in idxs]
363 'Long argument string detected. '
364 'Use -- to separate parameters and URLs, like this:\n%s\n' %
365 args_to_str(correct_argv))
367 def add_info_extractor(self, ie):
368 """Add an InfoExtractor object to the end of the list."""
370 self._ies_instances[ie.ie_key()] = ie
371 ie.set_downloader(self)
373 def get_info_extractor(self, ie_key):
375 Get an instance of an IE with name ie_key, it will try to get one from
376 the _ies list, if there's no instance it will create a new one and add
377 it to the extractor list.
379 ie = self._ies_instances.get(ie_key)
381 ie = get_info_extractor(ie_key)()
382 self.add_info_extractor(ie)
385 def add_default_info_extractors(self):
387 Add the InfoExtractors returned by gen_extractors to the end of the list
389 for ie in gen_extractors():
390 self.add_info_extractor(ie)
392 def add_post_processor(self, pp):
393 """Add a PostProcessor object to the end of the chain."""
395 pp.set_downloader(self)
397 def add_progress_hook(self, ph):
398 """Add the progress hook (currently only for the file downloader)"""
399 self._progress_hooks.append(ph)
401 def _bidi_workaround(self, message):
402 if not hasattr(self, '_output_channel'):
405 assert hasattr(self, '_output_process')
406 assert isinstance(message, compat_str)
407 line_count = message.count('\n') + 1
408 self._output_process.stdin.write((message + '\n').encode('utf-8'))
409 self._output_process.stdin.flush()
410 res = ''.join(self._output_channel.readline().decode('utf-8')
411 for _ in range(line_count))
412 return res[:-len('\n')]
414 def to_screen(self, message, skip_eol=False):
415 """Print message to stdout if not in quiet mode."""
416 return self.to_stdout(message, skip_eol, check_quiet=True)
418 def _write_string(self, s, out=None):
419 write_string(s, out=out, encoding=self.params.get('encoding'))
421 def to_stdout(self, message, skip_eol=False, check_quiet=False):
422 """Print message to stdout if not in quiet mode."""
423 if self.params.get('logger'):
424 self.params['logger'].debug(message)
425 elif not check_quiet or not self.params.get('quiet', False):
426 message = self._bidi_workaround(message)
427 terminator = ['\n', ''][skip_eol]
428 output = message + terminator
430 self._write_string(output, self._screen_file)
432 def to_stderr(self, message):
433 """Print message to stderr."""
434 assert isinstance(message, compat_str)
435 if self.params.get('logger'):
436 self.params['logger'].error(message)
438 message = self._bidi_workaround(message)
439 output = message + '\n'
440 self._write_string(output, self._err_file)
442 def to_console_title(self, message):
443 if not self.params.get('consoletitle', False):
445 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
446 # c_wchar_p() might not be necessary if `message` is
447 # already of type unicode()
448 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
449 elif 'TERM' in os.environ:
450 self._write_string('\033]0;%s\007' % message, self._screen_file)
452 def save_console_title(self):
453 if not self.params.get('consoletitle', False):
455 if 'TERM' in os.environ:
456 # Save the title on stack
457 self._write_string('\033[22;0t', self._screen_file)
459 def restore_console_title(self):
460 if not self.params.get('consoletitle', False):
462 if 'TERM' in os.environ:
463 # Restore the title from stack
464 self._write_string('\033[23;0t', self._screen_file)
467 self.save_console_title()
470 def __exit__(self, *args):
471 self.restore_console_title()
473 if self.params.get('cookiefile') is not None:
474 self.cookiejar.save()
476 def trouble(self, message=None, tb=None):
477 """Determine action to take when a download problem appears.
479 Depending on if the downloader has been configured to ignore
480 download errors or not, this method may throw an exception or
481 not when errors are found, after printing the message.
483 tb, if given, is additional traceback information.
485 if message is not None:
486 self.to_stderr(message)
487 if self.params.get('verbose'):
489 if sys.exc_info()[0]: # if .trouble has been called from an except block
491 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
492 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
493 tb += compat_str(traceback.format_exc())
495 tb_data = traceback.format_list(traceback.extract_stack())
496 tb = ''.join(tb_data)
498 if not self.params.get('ignoreerrors', False):
499 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
500 exc_info = sys.exc_info()[1].exc_info
502 exc_info = sys.exc_info()
503 raise DownloadError(message, exc_info)
504 self._download_retcode = 1
506 def report_warning(self, message):
508 Print the message to stderr, it will be prefixed with 'WARNING:'
509 If stderr is a tty file the 'WARNING:' will be colored
511 if self.params.get('logger') is not None:
512 self.params['logger'].warning(message)
514 if self.params.get('no_warnings'):
516 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
517 _msg_header = '\033[0;33mWARNING:\033[0m'
519 _msg_header = 'WARNING:'
520 warning_message = '%s %s' % (_msg_header, message)
521 self.to_stderr(warning_message)
523 def report_error(self, message, tb=None):
525 Do the same as trouble, but prefixes the message with 'ERROR:', colored
526 in red if stderr is a tty file.
528 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
529 _msg_header = '\033[0;31mERROR:\033[0m'
531 _msg_header = 'ERROR:'
532 error_message = '%s %s' % (_msg_header, message)
533 self.trouble(error_message, tb)
535 def report_file_already_downloaded(self, file_name):
536 """Report file has already been fully downloaded."""
538 self.to_screen('[download] %s has already been downloaded' % file_name)
539 except UnicodeEncodeError:
540 self.to_screen('[download] The file has already been downloaded')
542 def prepare_filename(self, info_dict):
543 """Generate the output filename."""
545 template_dict = dict(info_dict)
547 template_dict['epoch'] = int(time.time())
548 autonumber_size = self.params.get('autonumber_size')
549 if autonumber_size is None:
551 autonumber_templ = '%0' + str(autonumber_size) + 'd'
552 template_dict['autonumber'] = autonumber_templ % self._num_downloads
553 if template_dict.get('playlist_index') is not None:
554 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
555 if template_dict.get('resolution') is None:
556 if template_dict.get('width') and template_dict.get('height'):
557 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
558 elif template_dict.get('height'):
559 template_dict['resolution'] = '%sp' % template_dict['height']
560 elif template_dict.get('width'):
561 template_dict['resolution'] = '?x%d' % template_dict['width']
563 sanitize = lambda k, v: sanitize_filename(
565 restricted=self.params.get('restrictfilenames'),
567 template_dict = dict((k, sanitize(k, v))
568 for k, v in template_dict.items()
570 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
572 outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
573 tmpl = compat_expanduser(outtmpl)
574 filename = tmpl % template_dict
575 # Temporary fix for #4787
576 # 'Treat' all problem characters by passing filename through preferredencoding
577 # to workaround encoding issues with subprocess on python2 @ Windows
578 if sys.version_info < (3, 0) and sys.platform == 'win32':
579 filename = encodeFilename(filename, True).decode(preferredencoding())
581 except ValueError as err:
582 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
585 def _match_entry(self, info_dict, incomplete):
586 """ Returns None iff the file should be downloaded """
588 video_title = info_dict.get('title', info_dict.get('id', 'video'))
589 if 'title' in info_dict:
590 # This can happen when we're just evaluating the playlist
591 title = info_dict['title']
592 matchtitle = self.params.get('matchtitle', False)
594 if not re.search(matchtitle, title, re.IGNORECASE):
595 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
596 rejecttitle = self.params.get('rejecttitle', False)
598 if re.search(rejecttitle, title, re.IGNORECASE):
599 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
600 date = info_dict.get('upload_date', None)
602 dateRange = self.params.get('daterange', DateRange())
603 if date not in dateRange:
604 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
605 view_count = info_dict.get('view_count', None)
606 if view_count is not None:
607 min_views = self.params.get('min_views')
608 if min_views is not None and view_count < min_views:
609 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
610 max_views = self.params.get('max_views')
611 if max_views is not None and view_count > max_views:
612 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
613 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
614 return 'Skipping "%s" because it is age restricted' % video_title
615 if self.in_download_archive(info_dict):
616 return '%s has already been recorded in archive' % video_title
619 match_filter = self.params.get('match_filter')
620 if match_filter is not None:
621 ret = match_filter(info_dict)
628 def add_extra_info(info_dict, extra_info):
629 '''Set the keys from extra_info in info dict if they are missing'''
630 for key, value in extra_info.items():
631 info_dict.setdefault(key, value)
633 def extract_info(self, url, download=True, ie_key=None, extra_info={},
634 process=True, force_generic_extractor=False):
636 Returns a list with a dictionary for each video we find.
637 If 'download', also downloads the videos.
638 extra_info is a dict containing the extra values to add to each result
641 if not ie_key and force_generic_extractor:
645 ies = [self.get_info_extractor(ie_key)]
650 if not ie.suitable(url):
654 self.report_warning('The program functionality for this site has been marked as broken, '
655 'and will probably not work.')
658 ie_result = ie.extract(url)
659 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
661 if isinstance(ie_result, list):
662 # Backwards compatibility: old IE result format
664 '_type': 'compat_list',
665 'entries': ie_result,
667 self.add_default_extra_info(ie_result, ie, url)
669 return self.process_ie_result(ie_result, download, extra_info)
672 except ExtractorError as de: # An error we somewhat expected
673 self.report_error(compat_str(de), de.format_traceback())
675 except MaxDownloadsReached:
677 except Exception as e:
678 if self.params.get('ignoreerrors', False):
679 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
684 self.report_error('no suitable InfoExtractor for URL %s' % url)
686 def add_default_extra_info(self, ie_result, ie, url):
687 self.add_extra_info(ie_result, {
688 'extractor': ie.IE_NAME,
690 'webpage_url_basename': url_basename(url),
691 'extractor_key': ie.ie_key(),
694 def process_ie_result(self, ie_result, download=True, extra_info={}):
696 Take the result of the ie(may be modified) and resolve all unresolved
697 references (URLs, playlist items).
699 It will also download the videos if 'download'.
700 Returns the resolved ie_result.
703 result_type = ie_result.get('_type', 'video')
705 if result_type in ('url', 'url_transparent'):
706 extract_flat = self.params.get('extract_flat', False)
707 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
708 extract_flat is True):
709 if self.params.get('forcejson', False):
710 self.to_stdout(json.dumps(ie_result))
713 if result_type == 'video':
714 self.add_extra_info(ie_result, extra_info)
715 return self.process_video_result(ie_result, download=download)
716 elif result_type == 'url':
717 # We have to add extra_info to the results because it may be
718 # contained in a playlist
719 return self.extract_info(ie_result['url'],
721 ie_key=ie_result.get('ie_key'),
722 extra_info=extra_info)
723 elif result_type == 'url_transparent':
724 # Use the information from the embedding page
725 info = self.extract_info(
726 ie_result['url'], ie_key=ie_result.get('ie_key'),
727 extra_info=extra_info, download=False, process=False)
729 force_properties = dict(
730 (k, v) for k, v in ie_result.items() if v is not None)
731 for f in ('_type', 'url'):
732 if f in force_properties:
733 del force_properties[f]
734 new_result = info.copy()
735 new_result.update(force_properties)
737 assert new_result.get('_type') != 'url_transparent'
739 return self.process_ie_result(
740 new_result, download=download, extra_info=extra_info)
741 elif result_type == 'playlist' or result_type == 'multi_video':
742 # We process each entry in the playlist
743 playlist = ie_result.get('title', None) or ie_result.get('id', None)
744 self.to_screen('[download] Downloading playlist: %s' % playlist)
746 playlist_results = []
748 playliststart = self.params.get('playliststart', 1) - 1
749 playlistend = self.params.get('playlistend', None)
750 # For backwards compatibility, interpret -1 as whole list
751 if playlistend == -1:
754 playlistitems_str = self.params.get('playlist_items', None)
756 if playlistitems_str is not None:
757 def iter_playlistitems(format):
758 for string_segment in format.split(','):
759 if '-' in string_segment:
760 start, end = string_segment.split('-')
761 for item in range(int(start), int(end) + 1):
764 yield int(string_segment)
765 playlistitems = iter_playlistitems(playlistitems_str)
767 ie_entries = ie_result['entries']
768 if isinstance(ie_entries, list):
769 n_all_entries = len(ie_entries)
772 ie_entries[i - 1] for i in playlistitems
773 if -n_all_entries <= i - 1 < n_all_entries]
775 entries = ie_entries[playliststart:playlistend]
776 n_entries = len(entries)
778 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
779 (ie_result['extractor'], playlist, n_all_entries, n_entries))
780 elif isinstance(ie_entries, PagedList):
783 for item in playlistitems:
784 entries.extend(ie_entries.getslice(
788 entries = ie_entries.getslice(
789 playliststart, playlistend)
790 n_entries = len(entries)
792 "[%s] playlist %s: Downloading %d videos" %
793 (ie_result['extractor'], playlist, n_entries))
796 entry_list = list(ie_entries)
797 entries = [entry_list[i - 1] for i in playlistitems]
799 entries = list(itertools.islice(
800 ie_entries, playliststart, playlistend))
801 n_entries = len(entries)
803 "[%s] playlist %s: Downloading %d videos" %
804 (ie_result['extractor'], playlist, n_entries))
806 if self.params.get('playlistreverse', False):
807 entries = entries[::-1]
809 for i, entry in enumerate(entries, 1):
810 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
812 'n_entries': n_entries,
813 'playlist': playlist,
814 'playlist_id': ie_result.get('id'),
815 'playlist_title': ie_result.get('title'),
816 'playlist_index': i + playliststart,
817 'extractor': ie_result['extractor'],
818 'webpage_url': ie_result['webpage_url'],
819 'webpage_url_basename': url_basename(ie_result['webpage_url']),
820 'extractor_key': ie_result['extractor_key'],
823 reason = self._match_entry(entry, incomplete=True)
824 if reason is not None:
825 self.to_screen('[download] ' + reason)
828 entry_result = self.process_ie_result(entry,
831 playlist_results.append(entry_result)
832 ie_result['entries'] = playlist_results
834 elif result_type == 'compat_list':
836 'Extractor %s returned a compat_list result. '
837 'It needs to be updated.' % ie_result.get('extractor'))
843 'extractor': ie_result['extractor'],
844 'webpage_url': ie_result['webpage_url'],
845 'webpage_url_basename': url_basename(ie_result['webpage_url']),
846 'extractor_key': ie_result['extractor_key'],
850 ie_result['entries'] = [
851 self.process_ie_result(_fixup(r), download, extra_info)
852 for r in ie_result['entries']
856 raise Exception('Invalid result type: %s' % result_type)
858 def _build_format_filter(self, filter_spec):
859 " Returns a function to filter the formats according to the filter_spec "
869 operator_rex = re.compile(r'''(?x)\s*
870 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
871 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
872 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
874 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
875 m = operator_rex.search(filter_spec)
878 comparison_value = int(m.group('value'))
880 comparison_value = parse_filesize(m.group('value'))
881 if comparison_value is None:
882 comparison_value = parse_filesize(m.group('value') + 'B')
883 if comparison_value is None:
885 'Invalid value %r in format specification %r' % (
886 m.group('value'), filter_spec))
887 op = OPERATORS[m.group('op')]
894 str_operator_rex = re.compile(r'''(?x)
895 \s*(?P<key>ext|acodec|vcodec|container|protocol)
896 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
897 \s*(?P<value>[a-zA-Z0-9_-]+)
899 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
900 m = str_operator_rex.search(filter_spec)
902 comparison_value = m.group('value')
903 op = STR_OPERATORS[m.group('op')]
906 raise ValueError('Invalid filter specification %r' % filter_spec)
909 actual_value = f.get(m.group('key'))
910 if actual_value is None:
911 return m.group('none_inclusive')
912 return op(actual_value, comparison_value)
915 def build_format_selector(self, format_spec):
916 def syntax_error(note, start):
918 'Invalid format specification: '
919 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
920 return SyntaxError(message)
922 PICKFIRST = 'PICKFIRST'
926 FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
928 def _parse_filter(tokens):
930 for type, string, start, _, _ in tokens:
931 if type == tokenize.OP and string == ']':
932 return ''.join(filter_parts)
934 filter_parts.append(string)
936 def _remove_unused_ops(tokens):
937 # Remove operators that we don't use and join them with the sourrounding strings
938 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
939 ALLOWED_OPS = ('/', '+', ',', '(', ')')
940 last_string, last_start, last_end, last_line = None, None, None, None
941 for type, string, start, end, line in tokens:
942 if type == tokenize.OP and string == '[':
944 yield tokenize.NAME, last_string, last_start, last_end, last_line
946 yield type, string, start, end, line
947 # everything inside brackets will be handled by _parse_filter
948 for type, string, start, end, line in tokens:
949 yield type, string, start, end, line
950 if type == tokenize.OP and string == ']':
952 elif type == tokenize.OP and string in ALLOWED_OPS:
954 yield tokenize.NAME, last_string, last_start, last_end, last_line
956 yield type, string, start, end, line
957 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
963 last_string += string
965 yield tokenize.NAME, last_string, last_start, last_end, last_line
967 def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
969 current_selector = None
970 for type, string, start, _, _ in tokens:
971 # ENCODING is only defined in python 3.x
972 if type == getattr(tokenize, 'ENCODING', None):
974 elif type in [tokenize.NAME, tokenize.NUMBER]:
975 current_selector = FormatSelector(SINGLE, string, [])
976 elif type == tokenize.OP:
979 # ')' will be handled by the parentheses group
980 tokens.restore_last_token()
982 elif inside_merge and string in ['/', ',']:
983 tokens.restore_last_token()
985 elif inside_choice and string == ',':
986 tokens.restore_last_token()
989 if not current_selector:
990 raise syntax_error('"," must follow a format selector', start)
991 selectors.append(current_selector)
992 current_selector = None
994 if not current_selector:
995 raise syntax_error('"/" must follow a format selector', start)
996 first_choice = current_selector
997 second_choice = _parse_format_selection(tokens, inside_choice=True)
998 current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1000 if not current_selector:
1001 current_selector = FormatSelector(SINGLE, 'best', [])
1002 format_filter = _parse_filter(tokens)
1003 current_selector.filters.append(format_filter)
1005 if current_selector:
1006 raise syntax_error('Unexpected "("', start)
1007 group = _parse_format_selection(tokens, inside_group=True)
1008 current_selector = FormatSelector(GROUP, group, [])
1010 video_selector = current_selector
1011 audio_selector = _parse_format_selection(tokens, inside_merge=True)
1012 if not video_selector or not audio_selector:
1013 raise syntax_error('"+" must be between two format selectors', start)
1014 current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1016 raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1017 elif type == tokenize.ENDMARKER:
1019 if current_selector:
1020 selectors.append(current_selector)
1023 def _build_selector_function(selector):
1024 if isinstance(selector, list):
1025 fs = [_build_selector_function(s) for s in selector]
1027 def selector_function(formats):
1029 for format in f(formats):
1031 return selector_function
1032 elif selector.type == GROUP:
1033 selector_function = _build_selector_function(selector.selector)
1034 elif selector.type == PICKFIRST:
1035 fs = [_build_selector_function(s) for s in selector.selector]
1037 def selector_function(formats):
1039 picked_formats = list(f(formats))
1041 return picked_formats
1043 elif selector.type == SINGLE:
1044 format_spec = selector.selector
1046 def selector_function(formats):
1047 formats = list(formats)
1050 if format_spec == 'all':
1053 elif format_spec in ['best', 'worst', None]:
1054 format_idx = 0 if format_spec == 'worst' else -1
1055 audiovideo_formats = [
1057 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1058 if audiovideo_formats:
1059 yield audiovideo_formats[format_idx]
1060 # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
1061 elif (all(f.get('acodec') != 'none' for f in formats) or
1062 all(f.get('vcodec') != 'none' for f in formats)):
1063 yield formats[format_idx]
1064 elif format_spec == 'bestaudio':
1067 if f.get('vcodec') == 'none']
1069 yield audio_formats[-1]
1070 elif format_spec == 'worstaudio':
1073 if f.get('vcodec') == 'none']
1075 yield audio_formats[0]
1076 elif format_spec == 'bestvideo':
1079 if f.get('acodec') == 'none']
1081 yield video_formats[-1]
1082 elif format_spec == 'worstvideo':
1085 if f.get('acodec') == 'none']
1087 yield video_formats[0]
1089 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1090 if format_spec in extensions:
1091 filter_f = lambda f: f['ext'] == format_spec
1093 filter_f = lambda f: f['format_id'] == format_spec
1094 matches = list(filter(filter_f, formats))
1097 elif selector.type == MERGE:
1098 def _merge(formats_info):
1099 format_1, format_2 = [f['format_id'] for f in formats_info]
1100 # The first format must contain the video and the
1102 if formats_info[0].get('vcodec') == 'none':
1103 self.report_error('The first format must '
1104 'contain the video, try using '
1105 '"-f %s+%s"' % (format_2, format_1))
1108 formats_info[0]['ext']
1109 if self.params.get('merge_output_format') is None
1110 else self.params['merge_output_format'])
1112 'requested_formats': formats_info,
1113 'format': '%s+%s' % (formats_info[0].get('format'),
1114 formats_info[1].get('format')),
1115 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1116 formats_info[1].get('format_id')),
1117 'width': formats_info[0].get('width'),
1118 'height': formats_info[0].get('height'),
1119 'resolution': formats_info[0].get('resolution'),
1120 'fps': formats_info[0].get('fps'),
1121 'vcodec': formats_info[0].get('vcodec'),
1122 'vbr': formats_info[0].get('vbr'),
1123 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1124 'acodec': formats_info[1].get('acodec'),
1125 'abr': formats_info[1].get('abr'),
1128 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1130 def selector_function(formats):
1131 formats = list(formats)
1132 for pair in itertools.product(video_selector(formats), audio_selector(formats)):
1135 filters = [self._build_format_filter(f) for f in selector.filters]
1137 def final_selector(formats):
1138 for _filter in filters:
1139 formats = list(filter(_filter, formats))
1140 return selector_function(formats)
1141 return final_selector
1143 stream = io.BytesIO(format_spec.encode('utf-8'))
1145 tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1146 except tokenize.TokenError:
1147 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1149 class TokenIterator(object):
1150 def __init__(self, tokens):
1151 self.tokens = tokens
1158 if self.counter >= len(self.tokens):
1159 raise StopIteration()
1160 value = self.tokens[self.counter]
1166 def restore_last_token(self):
1169 parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1170 return _build_selector_function(parsed_selector)
1172 def _calc_headers(self, info_dict):
1173 res = std_headers.copy()
1175 add_headers = info_dict.get('http_headers')
1177 res.update(add_headers)
1179 cookies = self._calc_cookies(info_dict)
1181 res['Cookie'] = cookies
1185 def _calc_cookies(self, info_dict):
1186 pr = compat_urllib_request.Request(info_dict['url'])
1187 self.cookiejar.add_cookie_header(pr)
1188 return pr.get_header('Cookie')
1190 def process_video_result(self, info_dict, download=True):
1191 assert info_dict.get('_type', 'video') == 'video'
1193 if 'id' not in info_dict:
1194 raise ExtractorError('Missing "id" field in extractor result')
1195 if 'title' not in info_dict:
1196 raise ExtractorError('Missing "title" field in extractor result')
1198 if 'playlist' not in info_dict:
1199 # It isn't part of a playlist
1200 info_dict['playlist'] = None
1201 info_dict['playlist_index'] = None
1203 thumbnails = info_dict.get('thumbnails')
1204 if thumbnails is None:
1205 thumbnail = info_dict.get('thumbnail')
1207 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1209 thumbnails.sort(key=lambda t: (
1210 t.get('preference'), t.get('width'), t.get('height'),
1211 t.get('id'), t.get('url')))
1212 for i, t in enumerate(thumbnails):
1213 if t.get('width') and t.get('height'):
1214 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1215 if t.get('id') is None:
1218 if thumbnails and 'thumbnail' not in info_dict:
1219 info_dict['thumbnail'] = thumbnails[-1]['url']
1221 if 'display_id' not in info_dict and 'id' in info_dict:
1222 info_dict['display_id'] = info_dict['id']
1224 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1225 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1226 # see http://bugs.python.org/issue1646728)
1228 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1229 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1230 except (ValueError, OverflowError, OSError):
1233 if self.params.get('listsubtitles', False):
1234 if 'automatic_captions' in info_dict:
1235 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1236 self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
1238 info_dict['requested_subtitles'] = self.process_subtitles(
1239 info_dict['id'], info_dict.get('subtitles'),
1240 info_dict.get('automatic_captions'))
1242 # We now pick which formats have to be downloaded
1243 if info_dict.get('formats') is None:
1244 # There's only one format available
1245 formats = [info_dict]
1247 formats = info_dict['formats']
1250 raise ExtractorError('No video formats found!')
1254 # We check that all the formats have the format and format_id fields
1255 for i, format in enumerate(formats):
1256 if 'url' not in format:
1257 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1259 if format.get('format_id') is None:
1260 format['format_id'] = compat_str(i)
1261 format_id = format['format_id']
1262 if format_id not in formats_dict:
1263 formats_dict[format_id] = []
1264 formats_dict[format_id].append(format)
1266 # Make sure all formats have unique format_id
1267 for format_id, ambiguous_formats in formats_dict.items():
1268 if len(ambiguous_formats) > 1:
1269 for i, format in enumerate(ambiguous_formats):
1270 format['format_id'] = '%s-%d' % (format_id, i)
1272 for i, format in enumerate(formats):
1273 if format.get('format') is None:
1274 format['format'] = '{id} - {res}{note}'.format(
1275 id=format['format_id'],
1276 res=self.format_resolution(format),
1277 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1279 # Automatically determine file extension if missing
1280 if 'ext' not in format:
1281 format['ext'] = determine_ext(format['url']).lower()
1282 # Add HTTP headers, so that external programs can use them from the
1284 full_format_info = info_dict.copy()
1285 full_format_info.update(format)
1286 format['http_headers'] = self._calc_headers(full_format_info)
1288 # TODO Central sorting goes here
1290 if formats[0] is not info_dict:
1291 # only set the 'formats' fields if the original info_dict list them
1292 # otherwise we end up with a circular reference, the first (and unique)
1293 # element in the 'formats' field in info_dict is info_dict itself,
1294 # wich can't be exported to json
1295 info_dict['formats'] = formats
1296 if self.params.get('listformats'):
1297 self.list_formats(info_dict)
1299 if self.params.get('list_thumbnails'):
1300 self.list_thumbnails(info_dict)
1303 req_format = self.params.get('format')
1304 if req_format is None:
1305 req_format_list = []
1306 if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1307 info_dict['extractor'] in ['youtube', 'ted'] and
1308 not info_dict.get('is_live')):
1309 merger = FFmpegMergerPP(self)
1310 if merger.available and merger.can_merge():
1311 req_format_list.append('bestvideo+bestaudio')
1312 req_format_list.append('best')
1313 req_format = '/'.join(req_format_list)
1314 format_selector = self.build_format_selector(req_format)
1315 formats_to_download = list(format_selector(formats))
1316 if not formats_to_download:
1317 raise ExtractorError('requested format not available',
1321 if len(formats_to_download) > 1:
1322 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1323 for format in formats_to_download:
1324 new_info = dict(info_dict)
1325 new_info.update(format)
1326 self.process_info(new_info)
1327 # We update the info dict with the best quality format (backwards compatibility)
1328 info_dict.update(formats_to_download[-1])
1331 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1332 """Select the requested subtitles and their format"""
1334 if normal_subtitles and self.params.get('writesubtitles'):
1335 available_subs.update(normal_subtitles)
1336 if automatic_captions and self.params.get('writeautomaticsub'):
1337 for lang, cap_info in automatic_captions.items():
1338 if lang not in available_subs:
1339 available_subs[lang] = cap_info
1341 if (not self.params.get('writesubtitles') and not
1342 self.params.get('writeautomaticsub') or not
1346 if self.params.get('allsubtitles', False):
1347 requested_langs = available_subs.keys()
1349 if self.params.get('subtitleslangs', False):
1350 requested_langs = self.params.get('subtitleslangs')
1351 elif 'en' in available_subs:
1352 requested_langs = ['en']
1354 requested_langs = [list(available_subs.keys())[0]]
1356 formats_query = self.params.get('subtitlesformat', 'best')
1357 formats_preference = formats_query.split('/') if formats_query else []
1359 for lang in requested_langs:
1360 formats = available_subs.get(lang)
1362 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1364 for ext in formats_preference:
1368 matches = list(filter(lambda f: f['ext'] == ext, formats))
1374 self.report_warning(
1375 'No subtitle format found matching "%s" for language %s, '
1376 'using %s' % (formats_query, lang, f['ext']))
1380 def process_info(self, info_dict):
1381 """Process a single resolved IE result."""
1383 assert info_dict.get('_type', 'video') == 'video'
1385 max_downloads = self.params.get('max_downloads')
1386 if max_downloads is not None:
1387 if self._num_downloads >= int(max_downloads):
1388 raise MaxDownloadsReached()
1390 info_dict['fulltitle'] = info_dict['title']
1391 if len(info_dict['title']) > 200:
1392 info_dict['title'] = info_dict['title'][:197] + '...'
1394 if 'format' not in info_dict:
1395 info_dict['format'] = info_dict['ext']
1397 reason = self._match_entry(info_dict, incomplete=False)
1398 if reason is not None:
1399 self.to_screen('[download] ' + reason)
1402 self._num_downloads += 1
1404 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1407 if self.params.get('forcetitle', False):
1408 self.to_stdout(info_dict['fulltitle'])
1409 if self.params.get('forceid', False):
1410 self.to_stdout(info_dict['id'])
1411 if self.params.get('forceurl', False):
1412 if info_dict.get('requested_formats') is not None:
1413 for f in info_dict['requested_formats']:
1414 self.to_stdout(f['url'] + f.get('play_path', ''))
1416 # For RTMP URLs, also include the playpath
1417 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1418 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1419 self.to_stdout(info_dict['thumbnail'])
1420 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1421 self.to_stdout(info_dict['description'])
1422 if self.params.get('forcefilename', False) and filename is not None:
1423 self.to_stdout(filename)
1424 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1425 self.to_stdout(formatSeconds(info_dict['duration']))
1426 if self.params.get('forceformat', False):
1427 self.to_stdout(info_dict['format'])
1428 if self.params.get('forcejson', False):
1429 self.to_stdout(json.dumps(info_dict))
1431 # Do nothing else if in simulate mode
1432 if self.params.get('simulate', False):
1435 if filename is None:
1439 dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1440 if dn and not os.path.exists(dn):
1442 except (OSError, IOError) as err:
1443 self.report_error('unable to create directory ' + compat_str(err))
1446 if self.params.get('writedescription', False):
1447 descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1448 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1449 self.to_screen('[info] Video description is already present')
1450 elif info_dict.get('description') is None:
1451 self.report_warning('There\'s no description to write.')
1454 self.to_screen('[info] Writing video description to: ' + descfn)
1455 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1456 descfile.write(info_dict['description'])
1457 except (OSError, IOError):
1458 self.report_error('Cannot write description file ' + descfn)
1461 if self.params.get('writeannotations', False):
1462 annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1463 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1464 self.to_screen('[info] Video annotations are already present')
1467 self.to_screen('[info] Writing video annotations to: ' + annofn)
1468 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1469 annofile.write(info_dict['annotations'])
1470 except (KeyError, TypeError):
1471 self.report_warning('There are no annotations to write.')
1472 except (OSError, IOError):
1473 self.report_error('Cannot write annotations file: ' + annofn)
1476 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1477 self.params.get('writeautomaticsub')])
1479 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1480 # subtitles download errors are already managed as troubles in relevant IE
1481 # that way it will silently go on when used with unsupporting IE
1482 subtitles = info_dict['requested_subtitles']
1483 ie = self.get_info_extractor(info_dict['extractor_key'])
1484 for sub_lang, sub_info in subtitles.items():
1485 sub_format = sub_info['ext']
1486 if sub_info.get('data') is not None:
1487 sub_data = sub_info['data']
1490 sub_data = ie._download_webpage(
1491 sub_info['url'], info_dict['id'], note=False)
1492 except ExtractorError as err:
1493 self.report_warning('Unable to download subtitle for "%s": %s' %
1494 (sub_lang, compat_str(err.cause)))
1497 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1498 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1499 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1501 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1502 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1503 subfile.write(sub_data)
1504 except (OSError, IOError):
1505 self.report_error('Cannot write subtitles file ' + sub_filename)
1508 if self.params.get('writeinfojson', False):
1509 infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1510 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1511 self.to_screen('[info] Video description metadata is already present')
1513 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1515 write_json_file(self.filter_requested_info(info_dict), infofn)
1516 except (OSError, IOError):
1517 self.report_error('Cannot write metadata to JSON file ' + infofn)
1520 self._write_thumbnails(info_dict, filename)
1522 if not self.params.get('skip_download', False):
1525 fd = get_suitable_downloader(info, self.params)(self, self.params)
1526 for ph in self._progress_hooks:
1527 fd.add_progress_hook(ph)
1528 if self.params.get('verbose'):
1529 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1530 return fd.download(name, info)
1532 if info_dict.get('requested_formats') is not None:
1535 merger = FFmpegMergerPP(self)
1536 if not merger.available:
1538 self.report_warning('You have requested multiple '
1539 'formats but ffmpeg or avconv are not installed.'
1540 ' The formats won\'t be merged.')
1542 postprocessors = [merger]
1544 def compatible_formats(formats):
1545 video, audio = formats
1547 video_ext, audio_ext = audio.get('ext'), video.get('ext')
1548 if video_ext and audio_ext:
1550 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1553 for exts in COMPATIBLE_EXTS:
1554 if video_ext in exts and audio_ext in exts:
1556 # TODO: Check acodec/vcodec
1559 filename_real_ext = os.path.splitext(filename)[1][1:]
1561 os.path.splitext(filename)[0]
1562 if filename_real_ext == info_dict['ext']
1564 requested_formats = info_dict['requested_formats']
1565 if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1566 info_dict['ext'] = 'mkv'
1567 self.report_warning(
1568 'Requested formats are incompatible for merge and will be merged into mkv.')
1569 # Ensure filename always has a correct extension for successful merge
1570 filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1571 if os.path.exists(encodeFilename(filename)):
1573 '[download] %s has already been downloaded and '
1574 'merged' % filename)
1576 for f in requested_formats:
1577 new_info = dict(info_dict)
1579 fname = self.prepare_filename(new_info)
1580 fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1581 downloaded.append(fname)
1582 partial_success = dl(fname, new_info)
1583 success = success and partial_success
1584 info_dict['__postprocessors'] = postprocessors
1585 info_dict['__files_to_merge'] = downloaded
1587 # Just a single file
1588 success = dl(filename, info_dict)
1589 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1590 self.report_error('unable to download video data: %s' % str(err))
1592 except (OSError, IOError) as err:
1593 raise UnavailableVideoError(err)
1594 except (ContentTooShortError, ) as err:
1595 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1600 fixup_policy = self.params.get('fixup')
1601 if fixup_policy is None:
1602 fixup_policy = 'detect_or_warn'
1604 stretched_ratio = info_dict.get('stretched_ratio')
1605 if stretched_ratio is not None and stretched_ratio != 1:
1606 if fixup_policy == 'warn':
1607 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1608 info_dict['id'], stretched_ratio))
1609 elif fixup_policy == 'detect_or_warn':
1610 stretched_pp = FFmpegFixupStretchedPP(self)
1611 if stretched_pp.available:
1612 info_dict.setdefault('__postprocessors', [])
1613 info_dict['__postprocessors'].append(stretched_pp)
1615 self.report_warning(
1616 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1617 info_dict['id'], stretched_ratio))
1619 assert fixup_policy in ('ignore', 'never')
1621 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1622 if fixup_policy == 'warn':
1623 self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1625 elif fixup_policy == 'detect_or_warn':
1626 fixup_pp = FFmpegFixupM4aPP(self)
1627 if fixup_pp.available:
1628 info_dict.setdefault('__postprocessors', [])
1629 info_dict['__postprocessors'].append(fixup_pp)
1631 self.report_warning(
1632 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1635 assert fixup_policy in ('ignore', 'never')
1638 self.post_process(filename, info_dict)
1639 except (PostProcessingError) as err:
1640 self.report_error('postprocessing: %s' % str(err))
1642 self.record_download_archive(info_dict)
1644 def download(self, url_list):
1645 """Download a given list of URLs."""
1646 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1647 if (len(url_list) > 1 and
1648 '%' not in outtmpl and
1649 self.params.get('max_downloads') != 1):
1650 raise SameFileError(outtmpl)
1652 for url in url_list:
1654 # It also downloads the videos
1655 res = self.extract_info(
1656 url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1657 except UnavailableVideoError:
1658 self.report_error('unable to download video')
1659 except MaxDownloadsReached:
1660 self.to_screen('[info] Maximum number of downloaded files reached.')
1663 if self.params.get('dump_single_json', False):
1664 self.to_stdout(json.dumps(res))
1666 return self._download_retcode
1668 def download_with_info_file(self, info_filename):
1669 with contextlib.closing(fileinput.FileInput(
1670 [info_filename], mode='r',
1671 openhook=fileinput.hook_encoded('utf-8'))) as f:
1672 # FileInput doesn't have a read method, we can't call json.load
1673 info = self.filter_requested_info(json.loads('\n'.join(f)))
1675 self.process_ie_result(info, download=True)
1676 except DownloadError:
1677 webpage_url = info.get('webpage_url')
1678 if webpage_url is not None:
1679 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1680 return self.download([webpage_url])
1683 return self._download_retcode
1686 def filter_requested_info(info_dict):
1688 (k, v) for k, v in info_dict.items()
1689 if k not in ['requested_formats', 'requested_subtitles'])
1691 def post_process(self, filename, ie_info):
1692 """Run all the postprocessors on the given file."""
1693 info = dict(ie_info)
1694 info['filepath'] = filename
1696 if ie_info.get('__postprocessors') is not None:
1697 pps_chain.extend(ie_info['__postprocessors'])
1698 pps_chain.extend(self._pps)
1699 for pp in pps_chain:
1700 files_to_delete = []
1702 files_to_delete, info = pp.run(info)
1703 except PostProcessingError as e:
1704 self.report_error(e.msg)
1705 if files_to_delete and not self.params.get('keepvideo', False):
1706 for old_filename in files_to_delete:
1707 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1709 os.remove(encodeFilename(old_filename))
1710 except (IOError, OSError):
1711 self.report_warning('Unable to remove downloaded original file')
1713 def _make_archive_id(self, info_dict):
1714 # Future-proof against any change in case
1715 # and backwards compatibility with prior versions
1716 extractor = info_dict.get('extractor_key')
1717 if extractor is None:
1718 if 'id' in info_dict:
1719 extractor = info_dict.get('ie_key') # key in a playlist
1720 if extractor is None:
1721 return None # Incomplete video information
1722 return extractor.lower() + ' ' + info_dict['id']
1724 def in_download_archive(self, info_dict):
1725 fn = self.params.get('download_archive')
1729 vid_id = self._make_archive_id(info_dict)
1731 return False # Incomplete video information
1734 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1735 for line in archive_file:
1736 if line.strip() == vid_id:
1738 except IOError as ioe:
1739 if ioe.errno != errno.ENOENT:
1743 def record_download_archive(self, info_dict):
1744 fn = self.params.get('download_archive')
1747 vid_id = self._make_archive_id(info_dict)
1749 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1750 archive_file.write(vid_id + '\n')
1753 def format_resolution(format, default='unknown'):
1754 if format.get('vcodec') == 'none':
1756 if format.get('resolution') is not None:
1757 return format['resolution']
1758 if format.get('height') is not None:
1759 if format.get('width') is not None:
1760 res = '%sx%s' % (format['width'], format['height'])
1762 res = '%sp' % format['height']
1763 elif format.get('width') is not None:
1764 res = '?x%d' % format['width']
1769 def _format_note(self, fdict):
1771 if fdict.get('ext') in ['f4f', 'f4m']:
1772 res += '(unsupported) '
1773 if fdict.get('format_note') is not None:
1774 res += fdict['format_note'] + ' '
1775 if fdict.get('tbr') is not None:
1776 res += '%4dk ' % fdict['tbr']
1777 if fdict.get('container') is not None:
1780 res += '%s container' % fdict['container']
1781 if (fdict.get('vcodec') is not None and
1782 fdict.get('vcodec') != 'none'):
1785 res += fdict['vcodec']
1786 if fdict.get('vbr') is not None:
1788 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1790 if fdict.get('vbr') is not None:
1791 res += '%4dk' % fdict['vbr']
1792 if fdict.get('fps') is not None:
1793 res += ', %sfps' % fdict['fps']
1794 if fdict.get('acodec') is not None:
1797 if fdict['acodec'] == 'none':
1800 res += '%-5s' % fdict['acodec']
1801 elif fdict.get('abr') is not None:
1805 if fdict.get('abr') is not None:
1806 res += '@%3dk' % fdict['abr']
1807 if fdict.get('asr') is not None:
1808 res += ' (%5dHz)' % fdict['asr']
1809 if fdict.get('filesize') is not None:
1812 res += format_bytes(fdict['filesize'])
1813 elif fdict.get('filesize_approx') is not None:
1816 res += '~' + format_bytes(fdict['filesize_approx'])
1819 def list_formats(self, info_dict):
1820 formats = info_dict.get('formats', [info_dict])
1822 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1824 if f.get('preference') is None or f['preference'] >= -1000]
1825 if len(formats) > 1:
1826 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1828 header_line = ['format code', 'extension', 'resolution', 'note']
1830 '[info] Available formats for %s:\n%s' %
1831 (info_dict['id'], render_table(header_line, table)))
1833 def list_thumbnails(self, info_dict):
1834 thumbnails = info_dict.get('thumbnails')
1836 tn_url = info_dict.get('thumbnail')
1838 thumbnails = [{'id': '0', 'url': tn_url}]
1841 '[info] No thumbnails present for %s' % info_dict['id'])
1845 '[info] Thumbnails for %s:' % info_dict['id'])
1846 self.to_screen(render_table(
1847 ['ID', 'width', 'height', 'URL'],
1848 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1850 def list_subtitles(self, video_id, subtitles, name='subtitles'):
1852 self.to_screen('%s has no %s' % (video_id, name))
1855 'Available %s for %s:' % (name, video_id))
1856 self.to_screen(render_table(
1857 ['Language', 'formats'],
1858 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1859 for lang, formats in subtitles.items()]))
1861 def urlopen(self, req):
1862 """ Start an HTTP download """
1863 return self._opener.open(req, timeout=self._socket_timeout)
1865 def print_debug_header(self):
1866 if not self.params.get('verbose'):
1869 if type('') is not compat_str:
1870 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1871 self.report_warning(
1872 'Your Python is broken! Update to a newer and supported version')
1874 stdout_encoding = getattr(
1875 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1877 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1878 locale.getpreferredencoding(),
1879 sys.getfilesystemencoding(),
1881 self.get_encoding()))
1882 write_string(encoding_str, encoding=None)
1884 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1886 sp = subprocess.Popen(
1887 ['git', 'rev-parse', '--short', 'HEAD'],
1888 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1889 cwd=os.path.dirname(os.path.abspath(__file__)))
1890 out, err = sp.communicate()
1891 out = out.decode().strip()
1892 if re.match('[0-9a-f]+', out):
1893 self._write_string('[debug] Git HEAD: ' + out + '\n')
1899 self._write_string('[debug] Python version %s - %s\n' % (
1900 platform.python_version(), platform_name()))
1902 exe_versions = FFmpegPostProcessor.get_versions(self)
1903 exe_versions['rtmpdump'] = rtmpdump_version()
1904 exe_str = ', '.join(
1906 for exe, v in sorted(exe_versions.items())
1911 self._write_string('[debug] exe versions: %s\n' % exe_str)
1914 for handler in self._opener.handlers:
1915 if hasattr(handler, 'proxies'):
1916 proxy_map.update(handler.proxies)
1917 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1919 if self.params.get('call_home', False):
1920 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1921 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1922 latest_version = self.urlopen(
1923 'https://yt-dl.org/latest/version').read().decode('utf-8')
1924 if version_tuple(latest_version) > version_tuple(__version__):
1925 self.report_warning(
1926 'You are using an outdated version (newest version: %s)! '
1927 'See https://yt-dl.org/update if you need help updating.' %
1930 def _setup_opener(self):
1931 timeout_val = self.params.get('socket_timeout')
1932 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1934 opts_cookiefile = self.params.get('cookiefile')
1935 opts_proxy = self.params.get('proxy')
1937 if opts_cookiefile is None:
1938 self.cookiejar = compat_cookiejar.CookieJar()
1940 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1942 if os.access(opts_cookiefile, os.R_OK):
1943 self.cookiejar.load()
1945 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1947 if opts_proxy is not None:
1948 if opts_proxy == '':
1951 proxies = {'http': opts_proxy, 'https': opts_proxy}
1953 proxies = compat_urllib_request.getproxies()
1954 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1955 if 'http' in proxies and 'https' not in proxies:
1956 proxies['https'] = proxies['http']
1957 proxy_handler = PerRequestProxyHandler(proxies)
1959 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1960 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1961 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1962 opener = compat_urllib_request.build_opener(
1963 proxy_handler, https_handler, cookie_processor, ydlh)
1965 # Delete the default user-agent header, which would otherwise apply in
1966 # cases where our custom HTTP handler doesn't come into play
1967 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1968 opener.addheaders = []
1969 self._opener = opener
1971 def encode(self, s):
1972 if isinstance(s, bytes):
1973 return s # Already encoded
1976 return s.encode(self.get_encoding())
1977 except UnicodeEncodeError as err:
1978 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1981 def get_encoding(self):
1982 encoding = self.params.get('encoding')
1983 if encoding is None:
1984 encoding = preferredencoding()
1987 def _write_thumbnails(self, info_dict, filename):
1988 if self.params.get('writethumbnail', False):
1989 thumbnails = info_dict.get('thumbnails')
1991 thumbnails = [thumbnails[-1]]
1992 elif self.params.get('write_all_thumbnails', False):
1993 thumbnails = info_dict.get('thumbnails')
1998 # No thumbnails present, so return immediately
2001 for t in thumbnails:
2002 thumb_ext = determine_ext(t['url'], 'jpg')
2003 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2004 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2005 t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2007 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2008 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2009 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2011 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2012 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2014 uf = self.urlopen(t['url'])
2015 with open(thumb_filename, 'wb') as thumbf:
2016 shutil.copyfileobj(uf, thumbf)
2017 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2018 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2019 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2020 self.report_warning('Unable to download thumbnail "%s": %s' %
2021 (t['url'], compat_str(err)))