2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import, unicode_literals
33 compat_get_terminal_size,
37 compat_tokenize_tokenize,
39 compat_urllib_request,
57 PerRequestProxyHandler,
67 UnavailableVideoError,
78 from .cache import Cache
79 from .extractor import get_info_extractor, gen_extractors
80 from .downloader import get_suitable_downloader
81 from .downloader.rtmp import rtmpdump_version
82 from .postprocessor import (
84 FFmpegFixupStretchedPP,
89 from .version import __version__
92 class YoutubeDL(object):
95 YoutubeDL objects are the ones responsible of downloading the
96 actual video file and writing it to disk if the user has requested
97 it, among some other tasks. In most cases there should be one per
98 program. As, given a video URL, the downloader doesn't know how to
99 extract all the needed information, task that InfoExtractors do, it
100 has to pass the URL to one of them.
102 For this, YoutubeDL objects have a method that allows
103 InfoExtractors to be registered in a given order. When it is passed
104 a URL, the YoutubeDL object handles it to the first InfoExtractor it
105 finds that reports being able to handle it. The InfoExtractor extracts
106 all the information about the video or videos the URL refers to, and
107 YoutubeDL process the extracted information, possibly using a File
108 Downloader to download the video.
110 YoutubeDL objects accept a lot of parameters. In order not to saturate
111 the object constructor with arguments, it receives a dictionary of
112 options instead. These options are available through the params
113 attribute for the InfoExtractors to use. The YoutubeDL also
114 registers itself as the downloader in charge for the InfoExtractors
115 that are added to it, so this is a "mutual registration".
119 username: Username for authentication purposes.
120 password: Password for authentication purposes.
121 videopassword: Password for accessing a video.
122 usenetrc: Use netrc for authentication instead.
123 verbose: Print additional info to stdout.
124 quiet: Do not print messages to stdout.
125 no_warnings: Do not print out anything for warnings.
126 forceurl: Force printing final URL.
127 forcetitle: Force printing title.
128 forceid: Force printing ID.
129 forcethumbnail: Force printing thumbnail URL.
130 forcedescription: Force printing description.
131 forcefilename: Force printing final filename.
132 forceduration: Force printing duration.
133 forcejson: Force printing info_dict as JSON.
134 dump_single_json: Force printing the info_dict of the whole playlist
135 (or video) as a single JSON line.
136 simulate: Do not download the video files.
137 format: Video format code. See options.py for more information.
138 outtmpl: Template for output names.
139 restrictfilenames: Do not allow "&" and spaces in file names
140 ignoreerrors: Do not stop on download errors.
141 force_generic_extractor: Force downloader to use the generic extractor
142 nooverwrites: Prevent overwriting files.
143 playliststart: Playlist item to start at.
144 playlistend: Playlist item to end at.
145 playlist_items: Specific indices of playlist to download.
146 playlistreverse: Download playlist items in reverse order.
147 matchtitle: Download only matching titles.
148 rejecttitle: Reject downloads for matching titles.
149 logger: Log messages to a logging.Logger instance.
150 logtostderr: Log messages to stderr instead of stdout.
151 writedescription: Write the video description to a .description file
152 writeinfojson: Write the video description to a .info.json file
153 writeannotations: Write the video annotations to a .annotations.xml file
154 writethumbnail: Write the thumbnail image to a file
155 write_all_thumbnails: Write all thumbnail formats to files
156 writesubtitles: Write the video subtitles to a file
157 writeautomaticsub: Write the automatic subtitles to a file
158 allsubtitles: Downloads all the subtitles of the video
159 (requires writesubtitles or writeautomaticsub)
160 listsubtitles: Lists all available subtitles for the video
161 subtitlesformat: The format code for subtitles
162 subtitleslangs: List of languages of the subtitles to download
163 keepvideo: Keep the video file after post-processing
164 daterange: A DateRange object, download only if the upload_date is in the range.
165 skip_download: Skip the actual download of the video file
166 cachedir: Location of the cache files in the filesystem.
167 False to disable filesystem cache.
168 noplaylist: Download single video instead of a playlist if in doubt.
169 age_limit: An integer representing the user's age in years.
170 Unsuitable videos for the given age are skipped.
171 min_views: An integer representing the minimum view count the video
172 must have in order to not be skipped.
173 Videos without view count information are always
174 downloaded. None for no limit.
175 max_views: An integer representing the maximum view count.
176 Videos that are more popular than that are not
178 Videos without view count information are always
179 downloaded. None for no limit.
180 download_archive: File name of a file where all downloads are recorded.
181 Videos already present in the file are not downloaded
183 cookiefile: File name where cookies should be read from and dumped to.
184 nocheckcertificate:Do not verify SSL certificates
185 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
186 At the moment, this is only supported by YouTube.
187 proxy: URL of the proxy server to use
188 cn_verification_proxy: URL of the proxy to use for IP address verification
189 on Chinese sites. (Experimental)
190 socket_timeout: Time to wait for unresponsive hosts, in seconds
191 bidi_workaround: Work around buggy terminals without bidirectional text
192 support, using fridibi
193 debug_printtraffic:Print out sent and received HTTP traffic
194 include_ads: Download ads as well
195 default_search: Prepend this string if an input url is not valid.
196 'auto' for elaborate guessing
197 encoding: Use this encoding instead of the system-specified.
198 extract_flat: Do not resolve URLs, return the immediate result.
199 Pass in 'in_playlist' to only show this behavior for
201 postprocessors: A list of dictionaries, each with an entry
202 * key: The name of the postprocessor. See
203 youtube_dl/postprocessor/__init__.py for a list.
204 as well as any further keyword arguments for the
206 progress_hooks: A list of functions that get called on download
207 progress, with a dictionary with the entries
208 * status: One of "downloading", "error", or "finished".
209 Check this first and ignore unknown values.
211 If status is one of "downloading", or "finished", the
212 following properties may also be present:
213 * filename: The final filename (always present)
214 * tmpfilename: The filename we're currently writing to
215 * downloaded_bytes: Bytes on disk
216 * total_bytes: Size of the whole file, None if unknown
217 * total_bytes_estimate: Guess of the eventual file size,
219 * elapsed: The number of seconds since download started.
220 * eta: The estimated time in seconds, None if unknown
221 * speed: The download speed in bytes/second, None if
223 * fragment_index: The counter of the currently
224 downloaded video fragment.
225 * fragment_count: The number of fragments (= individual
226 files that will be merged)
228 Progress hooks are guaranteed to be called at least once
229 (with status "finished") if the download is successful.
230 merge_output_format: Extension to use when merging formats.
231 fixup: Automatically correct known faults of the file.
233 - "never": do nothing
234 - "warn": only emit a warning
235 - "detect_or_warn": check whether we can do anything
236 about it, warn otherwise (default)
237 source_address: (Experimental) Client-side IP address to bind to.
238 call_home: Boolean, true iff we are allowed to contact the
239 youtube-dl servers for debugging.
240 sleep_interval: Number of seconds to sleep before each download.
241 listformats: Print an overview of available video formats and exit.
242 list_thumbnails: Print a table of all thumbnails and exit.
243 match_filter: A function that gets called with the info_dict of
245 If it returns a message, the video is ignored.
246 If it returns None, the video is downloaded.
247 match_filter_func in utils.py is one example for this.
248 no_color: Do not emit color codes in output.
250 The following options determine which downloader is picked:
251 external_downloader: Executable of the external downloader to call.
252 None or unset for standard (built-in) downloader.
253 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
255 The following parameters are not used by YoutubeDL itself, they are used by
256 the downloader (see youtube_dl/downloader/common.py):
257 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
258 noresizebuffer, retries, continuedl, noprogress, consoletitle,
259 xattr_set_filesize, external_downloader_args.
261 The following options are used by the post processors:
262 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
263 otherwise prefer avconv.
264 postprocessor_args: A list of additional command-line arguments for the
271 _download_retcode = None
272 _num_downloads = None
275 def __init__(self, params=None, auto_init=True):
276 """Create a FileDownloader object with the given options."""
280 self._ies_instances = {}
282 self._progress_hooks = []
283 self._download_retcode = 0
284 self._num_downloads = 0
285 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
286 self._err_file = sys.stderr
288 self.cache = Cache(self)
290 if params.get('bidi_workaround', False):
293 master, slave = pty.openpty()
294 width = compat_get_terminal_size().columns
298 width_args = ['-w', str(width)]
300 stdin=subprocess.PIPE,
302 stderr=self._err_file)
304 self._output_process = subprocess.Popen(
305 ['bidiv'] + width_args, **sp_kwargs
308 self._output_process = subprocess.Popen(
309 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
310 self._output_channel = os.fdopen(master, 'rb')
311 except OSError as ose:
313 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
317 if (sys.version_info >= (3,) and sys.platform != 'win32' and
318 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
319 not params.get('restrictfilenames', False)):
320 # On Python 3, the Unicode filesystem API will throw errors (#1474)
322 'Assuming --restrict-filenames since file system encoding '
323 'cannot encode all characters. '
324 'Set the LC_ALL environment variable to fix this.')
325 self.params['restrictfilenames'] = True
327 if isinstance(params.get('outtmpl'), bytes):
329 'Parameter outtmpl is bytes, but should be a unicode string. '
330 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
335 self.print_debug_header()
336 self.add_default_info_extractors()
338 for pp_def_raw in self.params.get('postprocessors', []):
339 pp_class = get_postprocessor(pp_def_raw['key'])
340 pp_def = dict(pp_def_raw)
342 pp = pp_class(self, **compat_kwargs(pp_def))
343 self.add_post_processor(pp)
345 for ph in self.params.get('progress_hooks', []):
346 self.add_progress_hook(ph)
348 def warn_if_short_id(self, argv):
349 # short YouTube ID starting with dash?
351 i for i, a in enumerate(argv)
352 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
356 [a for i, a in enumerate(argv) if i not in idxs] +
357 ['--'] + [argv[i] for i in idxs]
360 'Long argument string detected. '
361 'Use -- to separate parameters and URLs, like this:\n%s\n' %
362 args_to_str(correct_argv))
364 def add_info_extractor(self, ie):
365 """Add an InfoExtractor object to the end of the list."""
367 self._ies_instances[ie.ie_key()] = ie
368 ie.set_downloader(self)
370 def get_info_extractor(self, ie_key):
372 Get an instance of an IE with name ie_key, it will try to get one from
373 the _ies list, if there's no instance it will create a new one and add
374 it to the extractor list.
376 ie = self._ies_instances.get(ie_key)
378 ie = get_info_extractor(ie_key)()
379 self.add_info_extractor(ie)
382 def add_default_info_extractors(self):
384 Add the InfoExtractors returned by gen_extractors to the end of the list
386 for ie in gen_extractors():
387 self.add_info_extractor(ie)
389 def add_post_processor(self, pp):
390 """Add a PostProcessor object to the end of the chain."""
392 pp.set_downloader(self)
394 def add_progress_hook(self, ph):
395 """Add the progress hook (currently only for the file downloader)"""
396 self._progress_hooks.append(ph)
398 def _bidi_workaround(self, message):
399 if not hasattr(self, '_output_channel'):
402 assert hasattr(self, '_output_process')
403 assert isinstance(message, compat_str)
404 line_count = message.count('\n') + 1
405 self._output_process.stdin.write((message + '\n').encode('utf-8'))
406 self._output_process.stdin.flush()
407 res = ''.join(self._output_channel.readline().decode('utf-8')
408 for _ in range(line_count))
409 return res[:-len('\n')]
411 def to_screen(self, message, skip_eol=False):
412 """Print message to stdout if not in quiet mode."""
413 return self.to_stdout(message, skip_eol, check_quiet=True)
415 def _write_string(self, s, out=None):
416 write_string(s, out=out, encoding=self.params.get('encoding'))
418 def to_stdout(self, message, skip_eol=False, check_quiet=False):
419 """Print message to stdout if not in quiet mode."""
420 if self.params.get('logger'):
421 self.params['logger'].debug(message)
422 elif not check_quiet or not self.params.get('quiet', False):
423 message = self._bidi_workaround(message)
424 terminator = ['\n', ''][skip_eol]
425 output = message + terminator
427 self._write_string(output, self._screen_file)
429 def to_stderr(self, message):
430 """Print message to stderr."""
431 assert isinstance(message, compat_str)
432 if self.params.get('logger'):
433 self.params['logger'].error(message)
435 message = self._bidi_workaround(message)
436 output = message + '\n'
437 self._write_string(output, self._err_file)
439 def to_console_title(self, message):
440 if not self.params.get('consoletitle', False):
442 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
443 # c_wchar_p() might not be necessary if `message` is
444 # already of type unicode()
445 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
446 elif 'TERM' in os.environ:
447 self._write_string('\033]0;%s\007' % message, self._screen_file)
449 def save_console_title(self):
450 if not self.params.get('consoletitle', False):
452 if 'TERM' in os.environ:
453 # Save the title on stack
454 self._write_string('\033[22;0t', self._screen_file)
456 def restore_console_title(self):
457 if not self.params.get('consoletitle', False):
459 if 'TERM' in os.environ:
460 # Restore the title from stack
461 self._write_string('\033[23;0t', self._screen_file)
464 self.save_console_title()
467 def __exit__(self, *args):
468 self.restore_console_title()
470 if self.params.get('cookiefile') is not None:
471 self.cookiejar.save()
473 def trouble(self, message=None, tb=None):
474 """Determine action to take when a download problem appears.
476 Depending on if the downloader has been configured to ignore
477 download errors or not, this method may throw an exception or
478 not when errors are found, after printing the message.
480 tb, if given, is additional traceback information.
482 if message is not None:
483 self.to_stderr(message)
484 if self.params.get('verbose'):
486 if sys.exc_info()[0]: # if .trouble has been called from an except block
488 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
489 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
490 tb += compat_str(traceback.format_exc())
492 tb_data = traceback.format_list(traceback.extract_stack())
493 tb = ''.join(tb_data)
495 if not self.params.get('ignoreerrors', False):
496 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
497 exc_info = sys.exc_info()[1].exc_info
499 exc_info = sys.exc_info()
500 raise DownloadError(message, exc_info)
501 self._download_retcode = 1
503 def report_warning(self, message):
505 Print the message to stderr, it will be prefixed with 'WARNING:'
506 If stderr is a tty file the 'WARNING:' will be colored
508 if self.params.get('logger') is not None:
509 self.params['logger'].warning(message)
511 if self.params.get('no_warnings'):
513 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
514 _msg_header = '\033[0;33mWARNING:\033[0m'
516 _msg_header = 'WARNING:'
517 warning_message = '%s %s' % (_msg_header, message)
518 self.to_stderr(warning_message)
520 def report_error(self, message, tb=None):
522 Do the same as trouble, but prefixes the message with 'ERROR:', colored
523 in red if stderr is a tty file.
525 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
526 _msg_header = '\033[0;31mERROR:\033[0m'
528 _msg_header = 'ERROR:'
529 error_message = '%s %s' % (_msg_header, message)
530 self.trouble(error_message, tb)
532 def report_file_already_downloaded(self, file_name):
533 """Report file has already been fully downloaded."""
535 self.to_screen('[download] %s has already been downloaded' % file_name)
536 except UnicodeEncodeError:
537 self.to_screen('[download] The file has already been downloaded')
539 def prepare_filename(self, info_dict):
540 """Generate the output filename."""
542 template_dict = dict(info_dict)
544 template_dict['epoch'] = int(time.time())
545 autonumber_size = self.params.get('autonumber_size')
546 if autonumber_size is None:
548 autonumber_templ = '%0' + str(autonumber_size) + 'd'
549 template_dict['autonumber'] = autonumber_templ % self._num_downloads
550 if template_dict.get('playlist_index') is not None:
551 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
552 if template_dict.get('resolution') is None:
553 if template_dict.get('width') and template_dict.get('height'):
554 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
555 elif template_dict.get('height'):
556 template_dict['resolution'] = '%sp' % template_dict['height']
557 elif template_dict.get('width'):
558 template_dict['resolution'] = '?x%d' % template_dict['width']
560 sanitize = lambda k, v: sanitize_filename(
562 restricted=self.params.get('restrictfilenames'),
564 template_dict = dict((k, sanitize(k, v))
565 for k, v in template_dict.items()
567 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
569 outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
570 tmpl = compat_expanduser(outtmpl)
571 filename = tmpl % template_dict
572 # Temporary fix for #4787
573 # 'Treat' all problem characters by passing filename through preferredencoding
574 # to workaround encoding issues with subprocess on python2 @ Windows
575 if sys.version_info < (3, 0) and sys.platform == 'win32':
576 filename = encodeFilename(filename, True).decode(preferredencoding())
578 except ValueError as err:
579 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
582 def _match_entry(self, info_dict, incomplete):
583 """ Returns None iff the file should be downloaded """
585 video_title = info_dict.get('title', info_dict.get('id', 'video'))
586 if 'title' in info_dict:
587 # This can happen when we're just evaluating the playlist
588 title = info_dict['title']
589 matchtitle = self.params.get('matchtitle', False)
591 if not re.search(matchtitle, title, re.IGNORECASE):
592 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
593 rejecttitle = self.params.get('rejecttitle', False)
595 if re.search(rejecttitle, title, re.IGNORECASE):
596 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
597 date = info_dict.get('upload_date', None)
599 dateRange = self.params.get('daterange', DateRange())
600 if date not in dateRange:
601 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
602 view_count = info_dict.get('view_count', None)
603 if view_count is not None:
604 min_views = self.params.get('min_views')
605 if min_views is not None and view_count < min_views:
606 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
607 max_views = self.params.get('max_views')
608 if max_views is not None and view_count > max_views:
609 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
610 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
611 return 'Skipping "%s" because it is age restricted' % video_title
612 if self.in_download_archive(info_dict):
613 return '%s has already been recorded in archive' % video_title
616 match_filter = self.params.get('match_filter')
617 if match_filter is not None:
618 ret = match_filter(info_dict)
625 def add_extra_info(info_dict, extra_info):
626 '''Set the keys from extra_info in info dict if they are missing'''
627 for key, value in extra_info.items():
628 info_dict.setdefault(key, value)
630 def extract_info(self, url, download=True, ie_key=None, extra_info={},
631 process=True, force_generic_extractor=False):
633 Returns a list with a dictionary for each video we find.
634 If 'download', also downloads the videos.
635 extra_info is a dict containing the extra values to add to each result
638 if not ie_key and force_generic_extractor:
642 ies = [self.get_info_extractor(ie_key)]
647 if not ie.suitable(url):
651 self.report_warning('The program functionality for this site has been marked as broken, '
652 'and will probably not work.')
655 ie_result = ie.extract(url)
656 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
658 if isinstance(ie_result, list):
659 # Backwards compatibility: old IE result format
661 '_type': 'compat_list',
662 'entries': ie_result,
664 self.add_default_extra_info(ie_result, ie, url)
666 return self.process_ie_result(ie_result, download, extra_info)
669 except ExtractorError as de: # An error we somewhat expected
670 self.report_error(compat_str(de), de.format_traceback())
672 except MaxDownloadsReached:
674 except Exception as e:
675 if self.params.get('ignoreerrors', False):
676 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
681 self.report_error('no suitable InfoExtractor for URL %s' % url)
683 def add_default_extra_info(self, ie_result, ie, url):
684 self.add_extra_info(ie_result, {
685 'extractor': ie.IE_NAME,
687 'webpage_url_basename': url_basename(url),
688 'extractor_key': ie.ie_key(),
691 def process_ie_result(self, ie_result, download=True, extra_info={}):
693 Take the result of the ie(may be modified) and resolve all unresolved
694 references (URLs, playlist items).
696 It will also download the videos if 'download'.
697 Returns the resolved ie_result.
700 result_type = ie_result.get('_type', 'video')
702 if result_type in ('url', 'url_transparent'):
703 extract_flat = self.params.get('extract_flat', False)
704 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
705 extract_flat is True):
706 if self.params.get('forcejson', False):
707 self.to_stdout(json.dumps(ie_result))
710 if result_type == 'video':
711 self.add_extra_info(ie_result, extra_info)
712 return self.process_video_result(ie_result, download=download)
713 elif result_type == 'url':
714 # We have to add extra_info to the results because it may be
715 # contained in a playlist
716 return self.extract_info(ie_result['url'],
718 ie_key=ie_result.get('ie_key'),
719 extra_info=extra_info)
720 elif result_type == 'url_transparent':
721 # Use the information from the embedding page
722 info = self.extract_info(
723 ie_result['url'], ie_key=ie_result.get('ie_key'),
724 extra_info=extra_info, download=False, process=False)
726 force_properties = dict(
727 (k, v) for k, v in ie_result.items() if v is not None)
728 for f in ('_type', 'url'):
729 if f in force_properties:
730 del force_properties[f]
731 new_result = info.copy()
732 new_result.update(force_properties)
734 assert new_result.get('_type') != 'url_transparent'
736 return self.process_ie_result(
737 new_result, download=download, extra_info=extra_info)
738 elif result_type == 'playlist' or result_type == 'multi_video':
739 # We process each entry in the playlist
740 playlist = ie_result.get('title', None) or ie_result.get('id', None)
741 self.to_screen('[download] Downloading playlist: %s' % playlist)
743 playlist_results = []
745 playliststart = self.params.get('playliststart', 1) - 1
746 playlistend = self.params.get('playlistend', None)
747 # For backwards compatibility, interpret -1 as whole list
748 if playlistend == -1:
751 playlistitems_str = self.params.get('playlist_items', None)
753 if playlistitems_str is not None:
754 def iter_playlistitems(format):
755 for string_segment in format.split(','):
756 if '-' in string_segment:
757 start, end = string_segment.split('-')
758 for item in range(int(start), int(end) + 1):
761 yield int(string_segment)
762 playlistitems = iter_playlistitems(playlistitems_str)
764 ie_entries = ie_result['entries']
765 if isinstance(ie_entries, list):
766 n_all_entries = len(ie_entries)
769 ie_entries[i - 1] for i in playlistitems
770 if -n_all_entries <= i - 1 < n_all_entries]
772 entries = ie_entries[playliststart:playlistend]
773 n_entries = len(entries)
775 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
776 (ie_result['extractor'], playlist, n_all_entries, n_entries))
777 elif isinstance(ie_entries, PagedList):
780 for item in playlistitems:
781 entries.extend(ie_entries.getslice(
785 entries = ie_entries.getslice(
786 playliststart, playlistend)
787 n_entries = len(entries)
789 "[%s] playlist %s: Downloading %d videos" %
790 (ie_result['extractor'], playlist, n_entries))
793 entry_list = list(ie_entries)
794 entries = [entry_list[i - 1] for i in playlistitems]
796 entries = list(itertools.islice(
797 ie_entries, playliststart, playlistend))
798 n_entries = len(entries)
800 "[%s] playlist %s: Downloading %d videos" %
801 (ie_result['extractor'], playlist, n_entries))
803 if self.params.get('playlistreverse', False):
804 entries = entries[::-1]
806 for i, entry in enumerate(entries, 1):
807 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
809 'n_entries': n_entries,
810 'playlist': playlist,
811 'playlist_id': ie_result.get('id'),
812 'playlist_title': ie_result.get('title'),
813 'playlist_index': i + playliststart,
814 'extractor': ie_result['extractor'],
815 'webpage_url': ie_result['webpage_url'],
816 'webpage_url_basename': url_basename(ie_result['webpage_url']),
817 'extractor_key': ie_result['extractor_key'],
820 reason = self._match_entry(entry, incomplete=True)
821 if reason is not None:
822 self.to_screen('[download] ' + reason)
825 entry_result = self.process_ie_result(entry,
828 playlist_results.append(entry_result)
829 ie_result['entries'] = playlist_results
831 elif result_type == 'compat_list':
833 'Extractor %s returned a compat_list result. '
834 'It needs to be updated.' % ie_result.get('extractor'))
840 'extractor': ie_result['extractor'],
841 'webpage_url': ie_result['webpage_url'],
842 'webpage_url_basename': url_basename(ie_result['webpage_url']),
843 'extractor_key': ie_result['extractor_key'],
847 ie_result['entries'] = [
848 self.process_ie_result(_fixup(r), download, extra_info)
849 for r in ie_result['entries']
853 raise Exception('Invalid result type: %s' % result_type)
855 def _build_format_filter(self, filter_spec):
856 " Returns a function to filter the formats according to the filter_spec "
866 operator_rex = re.compile(r'''(?x)\s*
867 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
868 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
869 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
871 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
872 m = operator_rex.search(filter_spec)
875 comparison_value = int(m.group('value'))
877 comparison_value = parse_filesize(m.group('value'))
878 if comparison_value is None:
879 comparison_value = parse_filesize(m.group('value') + 'B')
880 if comparison_value is None:
882 'Invalid value %r in format specification %r' % (
883 m.group('value'), filter_spec))
884 op = OPERATORS[m.group('op')]
891 str_operator_rex = re.compile(r'''(?x)
892 \s*(?P<key>ext|acodec|vcodec|container|protocol)
893 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
894 \s*(?P<value>[a-zA-Z0-9_-]+)
896 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
897 m = str_operator_rex.search(filter_spec)
899 comparison_value = m.group('value')
900 op = STR_OPERATORS[m.group('op')]
903 raise ValueError('Invalid filter specification %r' % filter_spec)
906 actual_value = f.get(m.group('key'))
907 if actual_value is None:
908 return m.group('none_inclusive')
909 return op(actual_value, comparison_value)
912 def build_format_selector(self, format_spec):
913 def syntax_error(note, start):
915 'Invalid format specification: '
916 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
917 return SyntaxError(message)
919 PICKFIRST = 'PICKFIRST'
923 FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
925 def _parse_filter(tokens):
927 for type, string, start, _, _ in tokens:
928 if type == tokenize.OP and string == ']':
929 return ''.join(filter_parts)
931 filter_parts.append(string)
933 def _remove_unused_ops(tokens):
934 # Remove operators that we don't use and join them with the sourrounding strings
935 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
936 ALLOWED_OPS = ('/', '+', ',', '(', ')')
937 last_string, last_start, last_end, last_line = None, None, None, None
938 for type, string, start, end, line in tokens:
939 if type == tokenize.OP and string == '[':
941 yield tokenize.NAME, last_string, last_start, last_end, last_line
943 yield type, string, start, end, line
944 # everything inside brackets will be handled by _parse_filter
945 for type, string, start, end, line in tokens:
946 yield type, string, start, end, line
947 if type == tokenize.OP and string == ']':
949 elif type == tokenize.OP and string in ALLOWED_OPS:
951 yield tokenize.NAME, last_string, last_start, last_end, last_line
953 yield type, string, start, end, line
954 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
960 last_string += string
962 yield tokenize.NAME, last_string, last_start, last_end, last_line
964 def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
966 current_selector = None
967 for type, string, start, _, _ in tokens:
968 # ENCODING is only defined in python 3.x
969 if type == getattr(tokenize, 'ENCODING', None):
971 elif type in [tokenize.NAME, tokenize.NUMBER]:
972 current_selector = FormatSelector(SINGLE, string, [])
973 elif type == tokenize.OP:
976 # ')' will be handled by the parentheses group
977 tokens.restore_last_token()
979 elif inside_merge and string in ['/', ',']:
980 tokens.restore_last_token()
982 elif inside_choice and string == ',':
983 tokens.restore_last_token()
986 if not current_selector:
987 raise syntax_error('"," must follow a format selector', start)
988 selectors.append(current_selector)
989 current_selector = None
991 if not current_selector:
992 raise syntax_error('"/" must follow a format selector', start)
993 first_choice = current_selector
994 second_choice = _parse_format_selection(tokens, inside_choice=True)
995 current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
997 if not current_selector:
998 current_selector = FormatSelector(SINGLE, 'best', [])
999 format_filter = _parse_filter(tokens)
1000 current_selector.filters.append(format_filter)
1002 if current_selector:
1003 raise syntax_error('Unexpected "("', start)
1004 group = _parse_format_selection(tokens, inside_group=True)
1005 current_selector = FormatSelector(GROUP, group, [])
1007 video_selector = current_selector
1008 audio_selector = _parse_format_selection(tokens, inside_merge=True)
1009 if not video_selector or not audio_selector:
1010 raise syntax_error('"+" must be between two format selectors', start)
1011 current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1013 raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1014 elif type == tokenize.ENDMARKER:
1016 if current_selector:
1017 selectors.append(current_selector)
1020 def _build_selector_function(selector):
1021 if isinstance(selector, list):
1022 fs = [_build_selector_function(s) for s in selector]
1024 def selector_function(formats):
1026 for format in f(formats):
1028 return selector_function
1029 elif selector.type == GROUP:
1030 selector_function = _build_selector_function(selector.selector)
1031 elif selector.type == PICKFIRST:
1032 fs = [_build_selector_function(s) for s in selector.selector]
1034 def selector_function(formats):
1036 picked_formats = list(f(formats))
1038 return picked_formats
1040 elif selector.type == SINGLE:
1041 format_spec = selector.selector
1043 def selector_function(formats):
1044 formats = list(formats)
1047 if format_spec == 'all':
1050 elif format_spec in ['best', 'worst', None]:
1051 format_idx = 0 if format_spec == 'worst' else -1
1052 audiovideo_formats = [
1054 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1055 if audiovideo_formats:
1056 yield audiovideo_formats[format_idx]
1057 # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
1058 elif (all(f.get('acodec') != 'none' for f in formats) or
1059 all(f.get('vcodec') != 'none' for f in formats)):
1060 yield formats[format_idx]
1061 elif format_spec == 'bestaudio':
1064 if f.get('vcodec') == 'none']
1066 yield audio_formats[-1]
1067 elif format_spec == 'worstaudio':
1070 if f.get('vcodec') == 'none']
1072 yield audio_formats[0]
1073 elif format_spec == 'bestvideo':
1076 if f.get('acodec') == 'none']
1078 yield video_formats[-1]
1079 elif format_spec == 'worstvideo':
1082 if f.get('acodec') == 'none']
1084 yield video_formats[0]
1086 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1087 if format_spec in extensions:
1088 filter_f = lambda f: f['ext'] == format_spec
1090 filter_f = lambda f: f['format_id'] == format_spec
1091 matches = list(filter(filter_f, formats))
1094 elif selector.type == MERGE:
1095 def _merge(formats_info):
1096 format_1, format_2 = [f['format_id'] for f in formats_info]
1097 # The first format must contain the video and the
1099 if formats_info[0].get('vcodec') == 'none':
1100 self.report_error('The first format must '
1101 'contain the video, try using '
1102 '"-f %s+%s"' % (format_2, format_1))
1105 formats_info[0]['ext']
1106 if self.params.get('merge_output_format') is None
1107 else self.params['merge_output_format'])
1109 'requested_formats': formats_info,
1110 'format': '%s+%s' % (formats_info[0].get('format'),
1111 formats_info[1].get('format')),
1112 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1113 formats_info[1].get('format_id')),
1114 'width': formats_info[0].get('width'),
1115 'height': formats_info[0].get('height'),
1116 'resolution': formats_info[0].get('resolution'),
1117 'fps': formats_info[0].get('fps'),
1118 'vcodec': formats_info[0].get('vcodec'),
1119 'vbr': formats_info[0].get('vbr'),
1120 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1121 'acodec': formats_info[1].get('acodec'),
1122 'abr': formats_info[1].get('abr'),
1125 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1127 def selector_function(formats):
1128 formats = list(formats)
1129 for pair in itertools.product(video_selector(formats), audio_selector(formats)):
1132 filters = [self._build_format_filter(f) for f in selector.filters]
1134 def final_selector(formats):
1135 for _filter in filters:
1136 formats = list(filter(_filter, formats))
1137 return selector_function(formats)
1138 return final_selector
1140 stream = io.BytesIO(format_spec.encode('utf-8'))
1142 tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1143 except tokenize.TokenError:
1144 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1146 class TokenIterator(object):
1147 def __init__(self, tokens):
1148 self.tokens = tokens
1155 if self.counter >= len(self.tokens):
1156 raise StopIteration()
1157 value = self.tokens[self.counter]
1163 def restore_last_token(self):
1166 parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1167 return _build_selector_function(parsed_selector)
1169 def _calc_headers(self, info_dict):
1170 res = std_headers.copy()
1172 add_headers = info_dict.get('http_headers')
1174 res.update(add_headers)
1176 cookies = self._calc_cookies(info_dict)
1178 res['Cookie'] = cookies
1182 def _calc_cookies(self, info_dict):
1183 pr = compat_urllib_request.Request(info_dict['url'])
1184 self.cookiejar.add_cookie_header(pr)
1185 return pr.get_header('Cookie')
1187 def process_video_result(self, info_dict, download=True):
1188 assert info_dict.get('_type', 'video') == 'video'
1190 if 'id' not in info_dict:
1191 raise ExtractorError('Missing "id" field in extractor result')
1192 if 'title' not in info_dict:
1193 raise ExtractorError('Missing "title" field in extractor result')
1195 if 'playlist' not in info_dict:
1196 # It isn't part of a playlist
1197 info_dict['playlist'] = None
1198 info_dict['playlist_index'] = None
1200 thumbnails = info_dict.get('thumbnails')
1201 if thumbnails is None:
1202 thumbnail = info_dict.get('thumbnail')
1204 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1206 thumbnails.sort(key=lambda t: (
1207 t.get('preference'), t.get('width'), t.get('height'),
1208 t.get('id'), t.get('url')))
1209 for i, t in enumerate(thumbnails):
1210 if t.get('width') and t.get('height'):
1211 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1212 if t.get('id') is None:
1215 if thumbnails and 'thumbnail' not in info_dict:
1216 info_dict['thumbnail'] = thumbnails[-1]['url']
1218 if 'display_id' not in info_dict and 'id' in info_dict:
1219 info_dict['display_id'] = info_dict['id']
1221 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1222 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1223 # see http://bugs.python.org/issue1646728)
1225 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1226 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1227 except (ValueError, OverflowError, OSError):
1230 if self.params.get('listsubtitles', False):
1231 if 'automatic_captions' in info_dict:
1232 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1233 self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
1235 info_dict['requested_subtitles'] = self.process_subtitles(
1236 info_dict['id'], info_dict.get('subtitles'),
1237 info_dict.get('automatic_captions'))
1239 # We now pick which formats have to be downloaded
1240 if info_dict.get('formats') is None:
1241 # There's only one format available
1242 formats = [info_dict]
1244 formats = info_dict['formats']
1247 raise ExtractorError('No video formats found!')
1251 # We check that all the formats have the format and format_id fields
1252 for i, format in enumerate(formats):
1253 if 'url' not in format:
1254 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1256 if format.get('format_id') is None:
1257 format['format_id'] = compat_str(i)
1258 format_id = format['format_id']
1259 if format_id not in formats_dict:
1260 formats_dict[format_id] = []
1261 formats_dict[format_id].append(format)
1263 # Make sure all formats have unique format_id
1264 for format_id, ambiguous_formats in formats_dict.items():
1265 if len(ambiguous_formats) > 1:
1266 for i, format in enumerate(ambiguous_formats):
1267 format['format_id'] = '%s-%d' % (format_id, i)
1269 for i, format in enumerate(formats):
1270 if format.get('format') is None:
1271 format['format'] = '{id} - {res}{note}'.format(
1272 id=format['format_id'],
1273 res=self.format_resolution(format),
1274 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1276 # Automatically determine file extension if missing
1277 if 'ext' not in format:
1278 format['ext'] = determine_ext(format['url']).lower()
1279 # Add HTTP headers, so that external programs can use them from the
1281 full_format_info = info_dict.copy()
1282 full_format_info.update(format)
1283 format['http_headers'] = self._calc_headers(full_format_info)
1285 # TODO Central sorting goes here
1287 if formats[0] is not info_dict:
1288 # only set the 'formats' fields if the original info_dict list them
1289 # otherwise we end up with a circular reference, the first (and unique)
1290 # element in the 'formats' field in info_dict is info_dict itself,
1291 # wich can't be exported to json
1292 info_dict['formats'] = formats
1293 if self.params.get('listformats'):
1294 self.list_formats(info_dict)
1296 if self.params.get('list_thumbnails'):
1297 self.list_thumbnails(info_dict)
1300 req_format = self.params.get('format')
1301 if req_format is None:
1302 req_format_list = []
1303 if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1304 info_dict['extractor'] in ['youtube', 'ted'] and
1305 not info_dict.get('is_live')):
1306 merger = FFmpegMergerPP(self)
1307 if merger.available and merger.can_merge():
1308 req_format_list.append('bestvideo+bestaudio')
1309 req_format_list.append('best')
1310 req_format = '/'.join(req_format_list)
1311 format_selector = self.build_format_selector(req_format)
1312 formats_to_download = list(format_selector(formats))
1313 if not formats_to_download:
1314 raise ExtractorError('requested format not available',
1318 if len(formats_to_download) > 1:
1319 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1320 for format in formats_to_download:
1321 new_info = dict(info_dict)
1322 new_info.update(format)
1323 self.process_info(new_info)
1324 # We update the info dict with the best quality format (backwards compatibility)
1325 info_dict.update(formats_to_download[-1])
1328 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1329 """Select the requested subtitles and their format"""
1331 if normal_subtitles and self.params.get('writesubtitles'):
1332 available_subs.update(normal_subtitles)
1333 if automatic_captions and self.params.get('writeautomaticsub'):
1334 for lang, cap_info in automatic_captions.items():
1335 if lang not in available_subs:
1336 available_subs[lang] = cap_info
1338 if (not self.params.get('writesubtitles') and not
1339 self.params.get('writeautomaticsub') or not
1343 if self.params.get('allsubtitles', False):
1344 requested_langs = available_subs.keys()
1346 if self.params.get('subtitleslangs', False):
1347 requested_langs = self.params.get('subtitleslangs')
1348 elif 'en' in available_subs:
1349 requested_langs = ['en']
1351 requested_langs = [list(available_subs.keys())[0]]
1353 formats_query = self.params.get('subtitlesformat', 'best')
1354 formats_preference = formats_query.split('/') if formats_query else []
1356 for lang in requested_langs:
1357 formats = available_subs.get(lang)
1359 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1361 for ext in formats_preference:
1365 matches = list(filter(lambda f: f['ext'] == ext, formats))
1371 self.report_warning(
1372 'No subtitle format found matching "%s" for language %s, '
1373 'using %s' % (formats_query, lang, f['ext']))
1377 def process_info(self, info_dict):
1378 """Process a single resolved IE result."""
1380 assert info_dict.get('_type', 'video') == 'video'
1382 max_downloads = self.params.get('max_downloads')
1383 if max_downloads is not None:
1384 if self._num_downloads >= int(max_downloads):
1385 raise MaxDownloadsReached()
1387 info_dict['fulltitle'] = info_dict['title']
1388 if len(info_dict['title']) > 200:
1389 info_dict['title'] = info_dict['title'][:197] + '...'
1391 if 'format' not in info_dict:
1392 info_dict['format'] = info_dict['ext']
1394 reason = self._match_entry(info_dict, incomplete=False)
1395 if reason is not None:
1396 self.to_screen('[download] ' + reason)
1399 self._num_downloads += 1
1401 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1404 if self.params.get('forcetitle', False):
1405 self.to_stdout(info_dict['fulltitle'])
1406 if self.params.get('forceid', False):
1407 self.to_stdout(info_dict['id'])
1408 if self.params.get('forceurl', False):
1409 if info_dict.get('requested_formats') is not None:
1410 for f in info_dict['requested_formats']:
1411 self.to_stdout(f['url'] + f.get('play_path', ''))
1413 # For RTMP URLs, also include the playpath
1414 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1415 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1416 self.to_stdout(info_dict['thumbnail'])
1417 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1418 self.to_stdout(info_dict['description'])
1419 if self.params.get('forcefilename', False) and filename is not None:
1420 self.to_stdout(filename)
1421 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1422 self.to_stdout(formatSeconds(info_dict['duration']))
1423 if self.params.get('forceformat', False):
1424 self.to_stdout(info_dict['format'])
1425 if self.params.get('forcejson', False):
1426 self.to_stdout(json.dumps(info_dict))
1428 # Do nothing else if in simulate mode
1429 if self.params.get('simulate', False):
1432 if filename is None:
1436 dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1437 if dn and not os.path.exists(dn):
1439 except (OSError, IOError) as err:
1440 self.report_error('unable to create directory ' + compat_str(err))
1443 if self.params.get('writedescription', False):
1444 descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1445 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1446 self.to_screen('[info] Video description is already present')
1447 elif info_dict.get('description') is None:
1448 self.report_warning('There\'s no description to write.')
1451 self.to_screen('[info] Writing video description to: ' + descfn)
1452 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1453 descfile.write(info_dict['description'])
1454 except (OSError, IOError):
1455 self.report_error('Cannot write description file ' + descfn)
1458 if self.params.get('writeannotations', False):
1459 annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1460 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1461 self.to_screen('[info] Video annotations are already present')
1464 self.to_screen('[info] Writing video annotations to: ' + annofn)
1465 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1466 annofile.write(info_dict['annotations'])
1467 except (KeyError, TypeError):
1468 self.report_warning('There are no annotations to write.')
1469 except (OSError, IOError):
1470 self.report_error('Cannot write annotations file: ' + annofn)
1473 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1474 self.params.get('writeautomaticsub')])
1476 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1477 # subtitles download errors are already managed as troubles in relevant IE
1478 # that way it will silently go on when used with unsupporting IE
1479 subtitles = info_dict['requested_subtitles']
1480 ie = self.get_info_extractor(info_dict['extractor_key'])
1481 for sub_lang, sub_info in subtitles.items():
1482 sub_format = sub_info['ext']
1483 if sub_info.get('data') is not None:
1484 sub_data = sub_info['data']
1487 sub_data = ie._download_webpage(
1488 sub_info['url'], info_dict['id'], note=False)
1489 except ExtractorError as err:
1490 self.report_warning('Unable to download subtitle for "%s": %s' %
1491 (sub_lang, compat_str(err.cause)))
1494 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1495 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1496 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1498 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1499 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1500 subfile.write(sub_data)
1501 except (OSError, IOError):
1502 self.report_error('Cannot write subtitles file ' + sub_filename)
1505 if self.params.get('writeinfojson', False):
1506 infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1507 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1508 self.to_screen('[info] Video description metadata is already present')
1510 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1512 write_json_file(self.filter_requested_info(info_dict), infofn)
1513 except (OSError, IOError):
1514 self.report_error('Cannot write metadata to JSON file ' + infofn)
1517 self._write_thumbnails(info_dict, filename)
1519 if not self.params.get('skip_download', False):
1522 fd = get_suitable_downloader(info, self.params)(self, self.params)
1523 for ph in self._progress_hooks:
1524 fd.add_progress_hook(ph)
1525 if self.params.get('verbose'):
1526 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1527 return fd.download(name, info)
1529 if info_dict.get('requested_formats') is not None:
1532 merger = FFmpegMergerPP(self)
1533 if not merger.available:
1535 self.report_warning('You have requested multiple '
1536 'formats but ffmpeg or avconv are not installed.'
1537 ' The formats won\'t be merged.')
1539 postprocessors = [merger]
1541 def compatible_formats(formats):
1542 video, audio = formats
1544 video_ext, audio_ext = audio.get('ext'), video.get('ext')
1545 if video_ext and audio_ext:
1547 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1550 for exts in COMPATIBLE_EXTS:
1551 if video_ext in exts and audio_ext in exts:
1553 # TODO: Check acodec/vcodec
1556 filename_real_ext = os.path.splitext(filename)[1][1:]
1558 os.path.splitext(filename)[0]
1559 if filename_real_ext == info_dict['ext']
1561 requested_formats = info_dict['requested_formats']
1562 if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1563 info_dict['ext'] = 'mkv'
1564 self.report_warning(
1565 'Requested formats are incompatible for merge and will be merged into mkv.')
1566 # Ensure filename always has a correct extension for successful merge
1567 filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1568 if os.path.exists(encodeFilename(filename)):
1570 '[download] %s has already been downloaded and '
1571 'merged' % filename)
1573 for f in requested_formats:
1574 new_info = dict(info_dict)
1576 fname = self.prepare_filename(new_info)
1577 fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1578 downloaded.append(fname)
1579 partial_success = dl(fname, new_info)
1580 success = success and partial_success
1581 info_dict['__postprocessors'] = postprocessors
1582 info_dict['__files_to_merge'] = downloaded
1584 # Just a single file
1585 success = dl(filename, info_dict)
1586 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1587 self.report_error('unable to download video data: %s' % str(err))
1589 except (OSError, IOError) as err:
1590 raise UnavailableVideoError(err)
1591 except (ContentTooShortError, ) as err:
1592 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1597 fixup_policy = self.params.get('fixup')
1598 if fixup_policy is None:
1599 fixup_policy = 'detect_or_warn'
1601 stretched_ratio = info_dict.get('stretched_ratio')
1602 if stretched_ratio is not None and stretched_ratio != 1:
1603 if fixup_policy == 'warn':
1604 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1605 info_dict['id'], stretched_ratio))
1606 elif fixup_policy == 'detect_or_warn':
1607 stretched_pp = FFmpegFixupStretchedPP(self)
1608 if stretched_pp.available:
1609 info_dict.setdefault('__postprocessors', [])
1610 info_dict['__postprocessors'].append(stretched_pp)
1612 self.report_warning(
1613 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1614 info_dict['id'], stretched_ratio))
1616 assert fixup_policy in ('ignore', 'never')
1618 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1619 if fixup_policy == 'warn':
1620 self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1622 elif fixup_policy == 'detect_or_warn':
1623 fixup_pp = FFmpegFixupM4aPP(self)
1624 if fixup_pp.available:
1625 info_dict.setdefault('__postprocessors', [])
1626 info_dict['__postprocessors'].append(fixup_pp)
1628 self.report_warning(
1629 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1632 assert fixup_policy in ('ignore', 'never')
1635 self.post_process(filename, info_dict)
1636 except (PostProcessingError) as err:
1637 self.report_error('postprocessing: %s' % str(err))
1639 self.record_download_archive(info_dict)
1641 def download(self, url_list):
1642 """Download a given list of URLs."""
1643 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1644 if (len(url_list) > 1 and
1645 '%' not in outtmpl and
1646 self.params.get('max_downloads') != 1):
1647 raise SameFileError(outtmpl)
1649 for url in url_list:
1651 # It also downloads the videos
1652 res = self.extract_info(
1653 url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1654 except UnavailableVideoError:
1655 self.report_error('unable to download video')
1656 except MaxDownloadsReached:
1657 self.to_screen('[info] Maximum number of downloaded files reached.')
1660 if self.params.get('dump_single_json', False):
1661 self.to_stdout(json.dumps(res))
1663 return self._download_retcode
1665 def download_with_info_file(self, info_filename):
1666 with contextlib.closing(fileinput.FileInput(
1667 [info_filename], mode='r',
1668 openhook=fileinput.hook_encoded('utf-8'))) as f:
1669 # FileInput doesn't have a read method, we can't call json.load
1670 info = self.filter_requested_info(json.loads('\n'.join(f)))
1672 self.process_ie_result(info, download=True)
1673 except DownloadError:
1674 webpage_url = info.get('webpage_url')
1675 if webpage_url is not None:
1676 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1677 return self.download([webpage_url])
1680 return self._download_retcode
1683 def filter_requested_info(info_dict):
1685 (k, v) for k, v in info_dict.items()
1686 if k not in ['requested_formats', 'requested_subtitles'])
1688 def post_process(self, filename, ie_info):
1689 """Run all the postprocessors on the given file."""
1690 info = dict(ie_info)
1691 info['filepath'] = filename
1693 if ie_info.get('__postprocessors') is not None:
1694 pps_chain.extend(ie_info['__postprocessors'])
1695 pps_chain.extend(self._pps)
1696 for pp in pps_chain:
1697 files_to_delete = []
1699 files_to_delete, info = pp.run(info)
1700 except PostProcessingError as e:
1701 self.report_error(e.msg)
1702 if files_to_delete and not self.params.get('keepvideo', False):
1703 for old_filename in files_to_delete:
1704 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1706 os.remove(encodeFilename(old_filename))
1707 except (IOError, OSError):
1708 self.report_warning('Unable to remove downloaded original file')
1710 def _make_archive_id(self, info_dict):
1711 # Future-proof against any change in case
1712 # and backwards compatibility with prior versions
1713 extractor = info_dict.get('extractor_key')
1714 if extractor is None:
1715 if 'id' in info_dict:
1716 extractor = info_dict.get('ie_key') # key in a playlist
1717 if extractor is None:
1718 return None # Incomplete video information
1719 return extractor.lower() + ' ' + info_dict['id']
1721 def in_download_archive(self, info_dict):
1722 fn = self.params.get('download_archive')
1726 vid_id = self._make_archive_id(info_dict)
1728 return False # Incomplete video information
1731 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1732 for line in archive_file:
1733 if line.strip() == vid_id:
1735 except IOError as ioe:
1736 if ioe.errno != errno.ENOENT:
1740 def record_download_archive(self, info_dict):
1741 fn = self.params.get('download_archive')
1744 vid_id = self._make_archive_id(info_dict)
1746 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1747 archive_file.write(vid_id + '\n')
1750 def format_resolution(format, default='unknown'):
1751 if format.get('vcodec') == 'none':
1753 if format.get('resolution') is not None:
1754 return format['resolution']
1755 if format.get('height') is not None:
1756 if format.get('width') is not None:
1757 res = '%sx%s' % (format['width'], format['height'])
1759 res = '%sp' % format['height']
1760 elif format.get('width') is not None:
1761 res = '?x%d' % format['width']
1766 def _format_note(self, fdict):
1768 if fdict.get('ext') in ['f4f', 'f4m']:
1769 res += '(unsupported) '
1770 if fdict.get('format_note') is not None:
1771 res += fdict['format_note'] + ' '
1772 if fdict.get('tbr') is not None:
1773 res += '%4dk ' % fdict['tbr']
1774 if fdict.get('container') is not None:
1777 res += '%s container' % fdict['container']
1778 if (fdict.get('vcodec') is not None and
1779 fdict.get('vcodec') != 'none'):
1782 res += fdict['vcodec']
1783 if fdict.get('vbr') is not None:
1785 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1787 if fdict.get('vbr') is not None:
1788 res += '%4dk' % fdict['vbr']
1789 if fdict.get('fps') is not None:
1790 res += ', %sfps' % fdict['fps']
1791 if fdict.get('acodec') is not None:
1794 if fdict['acodec'] == 'none':
1797 res += '%-5s' % fdict['acodec']
1798 elif fdict.get('abr') is not None:
1802 if fdict.get('abr') is not None:
1803 res += '@%3dk' % fdict['abr']
1804 if fdict.get('asr') is not None:
1805 res += ' (%5dHz)' % fdict['asr']
1806 if fdict.get('filesize') is not None:
1809 res += format_bytes(fdict['filesize'])
1810 elif fdict.get('filesize_approx') is not None:
1813 res += '~' + format_bytes(fdict['filesize_approx'])
1816 def list_formats(self, info_dict):
1817 formats = info_dict.get('formats', [info_dict])
1819 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1821 if f.get('preference') is None or f['preference'] >= -1000]
1822 if len(formats) > 1:
1823 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1825 header_line = ['format code', 'extension', 'resolution', 'note']
1827 '[info] Available formats for %s:\n%s' %
1828 (info_dict['id'], render_table(header_line, table)))
1830 def list_thumbnails(self, info_dict):
1831 thumbnails = info_dict.get('thumbnails')
1833 tn_url = info_dict.get('thumbnail')
1835 thumbnails = [{'id': '0', 'url': tn_url}]
1838 '[info] No thumbnails present for %s' % info_dict['id'])
1842 '[info] Thumbnails for %s:' % info_dict['id'])
1843 self.to_screen(render_table(
1844 ['ID', 'width', 'height', 'URL'],
1845 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1847 def list_subtitles(self, video_id, subtitles, name='subtitles'):
1849 self.to_screen('%s has no %s' % (video_id, name))
1852 'Available %s for %s:' % (name, video_id))
1853 self.to_screen(render_table(
1854 ['Language', 'formats'],
1855 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1856 for lang, formats in subtitles.items()]))
1858 def urlopen(self, req):
1859 """ Start an HTTP download """
1860 return self._opener.open(req, timeout=self._socket_timeout)
1862 def print_debug_header(self):
1863 if not self.params.get('verbose'):
1866 if type('') is not compat_str:
1867 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1868 self.report_warning(
1869 'Your Python is broken! Update to a newer and supported version')
1871 stdout_encoding = getattr(
1872 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1874 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1875 locale.getpreferredencoding(),
1876 sys.getfilesystemencoding(),
1878 self.get_encoding()))
1879 write_string(encoding_str, encoding=None)
1881 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1883 sp = subprocess.Popen(
1884 ['git', 'rev-parse', '--short', 'HEAD'],
1885 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1886 cwd=os.path.dirname(os.path.abspath(__file__)))
1887 out, err = sp.communicate()
1888 out = out.decode().strip()
1889 if re.match('[0-9a-f]+', out):
1890 self._write_string('[debug] Git HEAD: ' + out + '\n')
1896 self._write_string('[debug] Python version %s - %s\n' % (
1897 platform.python_version(), platform_name()))
1899 exe_versions = FFmpegPostProcessor.get_versions(self)
1900 exe_versions['rtmpdump'] = rtmpdump_version()
1901 exe_str = ', '.join(
1903 for exe, v in sorted(exe_versions.items())
1908 self._write_string('[debug] exe versions: %s\n' % exe_str)
1911 for handler in self._opener.handlers:
1912 if hasattr(handler, 'proxies'):
1913 proxy_map.update(handler.proxies)
1914 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1916 if self.params.get('call_home', False):
1917 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1918 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1919 latest_version = self.urlopen(
1920 'https://yt-dl.org/latest/version').read().decode('utf-8')
1921 if version_tuple(latest_version) > version_tuple(__version__):
1922 self.report_warning(
1923 'You are using an outdated version (newest version: %s)! '
1924 'See https://yt-dl.org/update if you need help updating.' %
1927 def _setup_opener(self):
1928 timeout_val = self.params.get('socket_timeout')
1929 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1931 opts_cookiefile = self.params.get('cookiefile')
1932 opts_proxy = self.params.get('proxy')
1934 if opts_cookiefile is None:
1935 self.cookiejar = compat_cookiejar.CookieJar()
1937 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1939 if os.access(opts_cookiefile, os.R_OK):
1940 self.cookiejar.load()
1942 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1944 if opts_proxy is not None:
1945 if opts_proxy == '':
1948 proxies = {'http': opts_proxy, 'https': opts_proxy}
1950 proxies = compat_urllib_request.getproxies()
1951 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1952 if 'http' in proxies and 'https' not in proxies:
1953 proxies['https'] = proxies['http']
1954 proxy_handler = PerRequestProxyHandler(proxies)
1956 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1957 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1958 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1959 opener = compat_urllib_request.build_opener(
1960 proxy_handler, https_handler, cookie_processor, ydlh)
1962 # Delete the default user-agent header, which would otherwise apply in
1963 # cases where our custom HTTP handler doesn't come into play
1964 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1965 opener.addheaders = []
1966 self._opener = opener
1968 def encode(self, s):
1969 if isinstance(s, bytes):
1970 return s # Already encoded
1973 return s.encode(self.get_encoding())
1974 except UnicodeEncodeError as err:
1975 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1978 def get_encoding(self):
1979 encoding = self.params.get('encoding')
1980 if encoding is None:
1981 encoding = preferredencoding()
1984 def _write_thumbnails(self, info_dict, filename):
1985 if self.params.get('writethumbnail', False):
1986 thumbnails = info_dict.get('thumbnails')
1988 thumbnails = [thumbnails[-1]]
1989 elif self.params.get('write_all_thumbnails', False):
1990 thumbnails = info_dict.get('thumbnails')
1995 # No thumbnails present, so return immediately
1998 for t in thumbnails:
1999 thumb_ext = determine_ext(t['url'], 'jpg')
2000 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2001 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2002 t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2004 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2005 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2006 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2008 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2009 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2011 uf = self.urlopen(t['url'])
2012 with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2013 shutil.copyfileobj(uf, thumbf)
2014 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2015 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2016 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2017 self.report_warning('Unable to download thumbnail "%s": %s' %
2018 (t['url'], compat_str(err)))