2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import, unicode_literals
33 compat_get_terminal_size,
38 compat_urllib_request,
57 PerRequestProxyHandler,
68 UnavailableVideoError,
78 from .cache import Cache
79 from .extractor import get_info_extractor, gen_extractors
80 from .downloader import get_suitable_downloader
81 from .downloader.rtmp import rtmpdump_version
82 from .postprocessor import (
84 FFmpegFixupStretchedPP,
89 from .version import __version__
92 class YoutubeDL(object):
95 YoutubeDL objects are the ones responsible of downloading the
96 actual video file and writing it to disk if the user has requested
97 it, among some other tasks. In most cases there should be one per
98 program. As, given a video URL, the downloader doesn't know how to
99 extract all the needed information, task that InfoExtractors do, it
100 has to pass the URL to one of them.
102 For this, YoutubeDL objects have a method that allows
103 InfoExtractors to be registered in a given order. When it is passed
104 a URL, the YoutubeDL object handles it to the first InfoExtractor it
105 finds that reports being able to handle it. The InfoExtractor extracts
106 all the information about the video or videos the URL refers to, and
107 YoutubeDL process the extracted information, possibly using a File
108 Downloader to download the video.
110 YoutubeDL objects accept a lot of parameters. In order not to saturate
111 the object constructor with arguments, it receives a dictionary of
112 options instead. These options are available through the params
113 attribute for the InfoExtractors to use. The YoutubeDL also
114 registers itself as the downloader in charge for the InfoExtractors
115 that are added to it, so this is a "mutual registration".
119 username: Username for authentication purposes.
120 password: Password for authentication purposes.
121 videopassword: Password for acces a video.
122 usenetrc: Use netrc for authentication instead.
123 verbose: Print additional info to stdout.
124 quiet: Do not print messages to stdout.
125 no_warnings: Do not print out anything for warnings.
126 forceurl: Force printing final URL.
127 forcetitle: Force printing title.
128 forceid: Force printing ID.
129 forcethumbnail: Force printing thumbnail URL.
130 forcedescription: Force printing description.
131 forcefilename: Force printing final filename.
132 forceduration: Force printing duration.
133 forcejson: Force printing info_dict as JSON.
134 dump_single_json: Force printing the info_dict of the whole playlist
135 (or video) as a single JSON line.
136 simulate: Do not download the video files.
137 format: Video format code. See options.py for more information.
138 format_limit: Highest quality format to try.
139 outtmpl: Template for output names.
140 restrictfilenames: Do not allow "&" and spaces in file names
141 ignoreerrors: Do not stop on download errors.
142 nooverwrites: Prevent overwriting files.
143 playliststart: Playlist item to start at.
144 playlistend: Playlist item to end at.
145 playlist_items: Specific indices of playlist to download.
146 playlistreverse: Download playlist items in reverse order.
147 matchtitle: Download only matching titles.
148 rejecttitle: Reject downloads for matching titles.
149 logger: Log messages to a logging.Logger instance.
150 logtostderr: Log messages to stderr instead of stdout.
151 writedescription: Write the video description to a .description file
152 writeinfojson: Write the video description to a .info.json file
153 writeannotations: Write the video annotations to a .annotations.xml file
154 writethumbnail: Write the thumbnail image to a file
155 write_all_thumbnails: Write all thumbnail formats to files
156 writesubtitles: Write the video subtitles to a file
157 writeautomaticsub: Write the automatic subtitles to a file
158 allsubtitles: Downloads all the subtitles of the video
159 (requires writesubtitles or writeautomaticsub)
160 listsubtitles: Lists all available subtitles for the video
161 subtitlesformat: The format code for subtitles
162 subtitleslangs: List of languages of the subtitles to download
163 keepvideo: Keep the video file after post-processing
164 daterange: A DateRange object, download only if the upload_date is in the range.
165 skip_download: Skip the actual download of the video file
166 cachedir: Location of the cache files in the filesystem.
167 False to disable filesystem cache.
168 noplaylist: Download single video instead of a playlist if in doubt.
169 age_limit: An integer representing the user's age in years.
170 Unsuitable videos for the given age are skipped.
171 min_views: An integer representing the minimum view count the video
172 must have in order to not be skipped.
173 Videos without view count information are always
174 downloaded. None for no limit.
175 max_views: An integer representing the maximum view count.
176 Videos that are more popular than that are not
178 Videos without view count information are always
179 downloaded. None for no limit.
180 download_archive: File name of a file where all downloads are recorded.
181 Videos already present in the file are not downloaded
183 cookiefile: File name where cookies should be read from and dumped to.
184 nocheckcertificate:Do not verify SSL certificates
185 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
186 At the moment, this is only supported by YouTube.
187 proxy: URL of the proxy server to use
188 cn_verification_proxy: URL of the proxy to use for IP address verification
189 on Chinese sites. (Experimental)
190 socket_timeout: Time to wait for unresponsive hosts, in seconds
191 bidi_workaround: Work around buggy terminals without bidirectional text
192 support, using fridibi
193 debug_printtraffic:Print out sent and received HTTP traffic
194 include_ads: Download ads as well
195 default_search: Prepend this string if an input url is not valid.
196 'auto' for elaborate guessing
197 encoding: Use this encoding instead of the system-specified.
198 extract_flat: Do not resolve URLs, return the immediate result.
199 Pass in 'in_playlist' to only show this behavior for
201 postprocessors: A list of dictionaries, each with an entry
202 * key: The name of the postprocessor. See
203 youtube_dl/postprocessor/__init__.py for a list.
204 as well as any further keyword arguments for the
206 progress_hooks: A list of functions that get called on download
207 progress, with a dictionary with the entries
208 * status: One of "downloading", "error", or "finished".
209 Check this first and ignore unknown values.
211 If status is one of "downloading", or "finished", the
212 following properties may also be present:
213 * filename: The final filename (always present)
214 * tmpfilename: The filename we're currently writing to
215 * downloaded_bytes: Bytes on disk
216 * total_bytes: Size of the whole file, None if unknown
217 * total_bytes_estimate: Guess of the eventual file size,
219 * elapsed: The number of seconds since download started.
220 * eta: The estimated time in seconds, None if unknown
221 * speed: The download speed in bytes/second, None if
223 * fragment_index: The counter of the currently
224 downloaded video fragment.
225 * fragment_count: The number of fragments (= individual
226 files that will be merged)
228 Progress hooks are guaranteed to be called at least once
229 (with status "finished") if the download is successful.
230 merge_output_format: Extension to use when merging formats.
231 fixup: Automatically correct known faults of the file.
233 - "never": do nothing
234 - "warn": only emit a warning
235 - "detect_or_warn": check whether we can do anything
236 about it, warn otherwise (default)
237 source_address: (Experimental) Client-side IP address to bind to.
238 call_home: Boolean, true iff we are allowed to contact the
239 youtube-dl servers for debugging.
240 sleep_interval: Number of seconds to sleep before each download.
241 listformats: Print an overview of available video formats and exit.
242 list_thumbnails: Print a table of all thumbnails and exit.
243 match_filter: A function that gets called with the info_dict of
245 If it returns a message, the video is ignored.
246 If it returns None, the video is downloaded.
247 match_filter_func in utils.py is one example for this.
248 no_color: Do not emit color codes in output.
250 The following options determine which downloader is picked:
251 external_downloader: Executable of the external downloader to call.
252 None or unset for standard (built-in) downloader.
253 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
255 The following parameters are not used by YoutubeDL itself, they are used by
256 the downloader (see youtube_dl/downloader/common.py):
257 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
258 noresizebuffer, retries, continuedl, noprogress, consoletitle,
259 xattr_set_filesize, external_downloader_args.
261 The following options are used by the post processors:
262 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
263 otherwise prefer avconv.
264 exec_cmd: Arbitrary command to run after downloading
270 _download_retcode = None
271 _num_downloads = None
274 def __init__(self, params=None, auto_init=True):
275 """Create a FileDownloader object with the given options."""
279 self._ies_instances = {}
281 self._progress_hooks = []
282 self._download_retcode = 0
283 self._num_downloads = 0
284 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
285 self._err_file = sys.stderr
287 self.cache = Cache(self)
289 if params.get('bidi_workaround', False):
292 master, slave = pty.openpty()
293 width = compat_get_terminal_size().columns
297 width_args = ['-w', str(width)]
299 stdin=subprocess.PIPE,
301 stderr=self._err_file)
303 self._output_process = subprocess.Popen(
304 ['bidiv'] + width_args, **sp_kwargs
307 self._output_process = subprocess.Popen(
308 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
309 self._output_channel = os.fdopen(master, 'rb')
310 except OSError as ose:
312 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
316 if (sys.version_info >= (3,) and sys.platform != 'win32' and
317 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
318 not params.get('restrictfilenames', False)):
319 # On Python 3, the Unicode filesystem API will throw errors (#1474)
321 'Assuming --restrict-filenames since file system encoding '
322 'cannot encode all characters. '
323 'Set the LC_ALL environment variable to fix this.')
324 self.params['restrictfilenames'] = True
326 if isinstance(params.get('outtmpl'), bytes):
328 'Parameter outtmpl is bytes, but should be a unicode string. '
329 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
334 self.print_debug_header()
335 self.add_default_info_extractors()
337 for pp_def_raw in self.params.get('postprocessors', []):
338 pp_class = get_postprocessor(pp_def_raw['key'])
339 pp_def = dict(pp_def_raw)
341 pp = pp_class(self, **compat_kwargs(pp_def))
342 self.add_post_processor(pp)
344 for ph in self.params.get('progress_hooks', []):
345 self.add_progress_hook(ph)
347 def warn_if_short_id(self, argv):
348 # short YouTube ID starting with dash?
350 i for i, a in enumerate(argv)
351 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
355 [a for i, a in enumerate(argv) if i not in idxs] +
356 ['--'] + [argv[i] for i in idxs]
359 'Long argument string detected. '
360 'Use -- to separate parameters and URLs, like this:\n%s\n' %
361 args_to_str(correct_argv))
363 def add_info_extractor(self, ie):
364 """Add an InfoExtractor object to the end of the list."""
366 self._ies_instances[ie.ie_key()] = ie
367 ie.set_downloader(self)
369 def get_info_extractor(self, ie_key):
371 Get an instance of an IE with name ie_key, it will try to get one from
372 the _ies list, if there's no instance it will create a new one and add
373 it to the extractor list.
375 ie = self._ies_instances.get(ie_key)
377 ie = get_info_extractor(ie_key)()
378 self.add_info_extractor(ie)
381 def add_default_info_extractors(self):
383 Add the InfoExtractors returned by gen_extractors to the end of the list
385 for ie in gen_extractors():
386 self.add_info_extractor(ie)
388 def add_post_processor(self, pp):
389 """Add a PostProcessor object to the end of the chain."""
391 pp.set_downloader(self)
393 def add_progress_hook(self, ph):
394 """Add the progress hook (currently only for the file downloader)"""
395 self._progress_hooks.append(ph)
397 def _bidi_workaround(self, message):
398 if not hasattr(self, '_output_channel'):
401 assert hasattr(self, '_output_process')
402 assert isinstance(message, compat_str)
403 line_count = message.count('\n') + 1
404 self._output_process.stdin.write((message + '\n').encode('utf-8'))
405 self._output_process.stdin.flush()
406 res = ''.join(self._output_channel.readline().decode('utf-8')
407 for _ in range(line_count))
408 return res[:-len('\n')]
410 def to_screen(self, message, skip_eol=False):
411 """Print message to stdout if not in quiet mode."""
412 return self.to_stdout(message, skip_eol, check_quiet=True)
414 def _write_string(self, s, out=None):
415 write_string(s, out=out, encoding=self.params.get('encoding'))
417 def to_stdout(self, message, skip_eol=False, check_quiet=False):
418 """Print message to stdout if not in quiet mode."""
419 if self.params.get('logger'):
420 self.params['logger'].debug(message)
421 elif not check_quiet or not self.params.get('quiet', False):
422 message = self._bidi_workaround(message)
423 terminator = ['\n', ''][skip_eol]
424 output = message + terminator
426 self._write_string(output, self._screen_file)
428 def to_stderr(self, message):
429 """Print message to stderr."""
430 assert isinstance(message, compat_str)
431 if self.params.get('logger'):
432 self.params['logger'].error(message)
434 message = self._bidi_workaround(message)
435 output = message + '\n'
436 self._write_string(output, self._err_file)
438 def to_console_title(self, message):
439 if not self.params.get('consoletitle', False):
441 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
442 # c_wchar_p() might not be necessary if `message` is
443 # already of type unicode()
444 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
445 elif 'TERM' in os.environ:
446 self._write_string('\033]0;%s\007' % message, self._screen_file)
448 def save_console_title(self):
449 if not self.params.get('consoletitle', False):
451 if 'TERM' in os.environ:
452 # Save the title on stack
453 self._write_string('\033[22;0t', self._screen_file)
455 def restore_console_title(self):
456 if not self.params.get('consoletitle', False):
458 if 'TERM' in os.environ:
459 # Restore the title from stack
460 self._write_string('\033[23;0t', self._screen_file)
463 self.save_console_title()
466 def __exit__(self, *args):
467 self.restore_console_title()
469 if self.params.get('cookiefile') is not None:
470 self.cookiejar.save()
472 def trouble(self, message=None, tb=None):
473 """Determine action to take when a download problem appears.
475 Depending on if the downloader has been configured to ignore
476 download errors or not, this method may throw an exception or
477 not when errors are found, after printing the message.
479 tb, if given, is additional traceback information.
481 if message is not None:
482 self.to_stderr(message)
483 if self.params.get('verbose'):
485 if sys.exc_info()[0]: # if .trouble has been called from an except block
487 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
488 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
489 tb += compat_str(traceback.format_exc())
491 tb_data = traceback.format_list(traceback.extract_stack())
492 tb = ''.join(tb_data)
494 if not self.params.get('ignoreerrors', False):
495 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
496 exc_info = sys.exc_info()[1].exc_info
498 exc_info = sys.exc_info()
499 raise DownloadError(message, exc_info)
500 self._download_retcode = 1
502 def report_warning(self, message):
504 Print the message to stderr, it will be prefixed with 'WARNING:'
505 If stderr is a tty file the 'WARNING:' will be colored
507 if self.params.get('logger') is not None:
508 self.params['logger'].warning(message)
510 if self.params.get('no_warnings'):
512 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
513 _msg_header = '\033[0;33mWARNING:\033[0m'
515 _msg_header = 'WARNING:'
516 warning_message = '%s %s' % (_msg_header, message)
517 self.to_stderr(warning_message)
519 def report_error(self, message, tb=None):
521 Do the same as trouble, but prefixes the message with 'ERROR:', colored
522 in red if stderr is a tty file.
524 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
525 _msg_header = '\033[0;31mERROR:\033[0m'
527 _msg_header = 'ERROR:'
528 error_message = '%s %s' % (_msg_header, message)
529 self.trouble(error_message, tb)
531 def report_file_already_downloaded(self, file_name):
532 """Report file has already been fully downloaded."""
534 self.to_screen('[download] %s has already been downloaded' % file_name)
535 except UnicodeEncodeError:
536 self.to_screen('[download] The file has already been downloaded')
538 def prepare_filename(self, info_dict):
539 """Generate the output filename."""
541 template_dict = dict(info_dict)
543 template_dict['epoch'] = int(time.time())
544 autonumber_size = self.params.get('autonumber_size')
545 if autonumber_size is None:
547 autonumber_templ = '%0' + str(autonumber_size) + 'd'
548 template_dict['autonumber'] = autonumber_templ % self._num_downloads
549 if template_dict.get('playlist_index') is not None:
550 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
551 if template_dict.get('resolution') is None:
552 if template_dict.get('width') and template_dict.get('height'):
553 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
554 elif template_dict.get('height'):
555 template_dict['resolution'] = '%sp' % template_dict['height']
556 elif template_dict.get('width'):
557 template_dict['resolution'] = '?x%d' % template_dict['width']
559 sanitize = lambda k, v: sanitize_filename(
561 restricted=self.params.get('restrictfilenames'),
563 template_dict = dict((k, sanitize(k, v))
564 for k, v in template_dict.items()
566 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
568 outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
569 tmpl = compat_expanduser(outtmpl)
570 filename = tmpl % template_dict
571 # Temporary fix for #4787
572 # 'Treat' all problem characters by passing filename through preferredencoding
573 # to workaround encoding issues with subprocess on python2 @ Windows
574 if sys.version_info < (3, 0) and sys.platform == 'win32':
575 filename = encodeFilename(filename, True).decode(preferredencoding())
577 except ValueError as err:
578 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
581 def _match_entry(self, info_dict, incomplete):
582 """ Returns None iff the file should be downloaded """
584 video_title = info_dict.get('title', info_dict.get('id', 'video'))
585 if 'title' in info_dict:
586 # This can happen when we're just evaluating the playlist
587 title = info_dict['title']
588 matchtitle = self.params.get('matchtitle', False)
590 if not re.search(matchtitle, title, re.IGNORECASE):
591 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
592 rejecttitle = self.params.get('rejecttitle', False)
594 if re.search(rejecttitle, title, re.IGNORECASE):
595 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
596 date = info_dict.get('upload_date', None)
598 dateRange = self.params.get('daterange', DateRange())
599 if date not in dateRange:
600 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
601 view_count = info_dict.get('view_count', None)
602 if view_count is not None:
603 min_views = self.params.get('min_views')
604 if min_views is not None and view_count < min_views:
605 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
606 max_views = self.params.get('max_views')
607 if max_views is not None and view_count > max_views:
608 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
609 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
610 return 'Skipping "%s" because it is age restricted' % video_title
611 if self.in_download_archive(info_dict):
612 return '%s has already been recorded in archive' % video_title
615 match_filter = self.params.get('match_filter')
616 if match_filter is not None:
617 ret = match_filter(info_dict)
624 def add_extra_info(info_dict, extra_info):
625 '''Set the keys from extra_info in info dict if they are missing'''
626 for key, value in extra_info.items():
627 info_dict.setdefault(key, value)
629 def extract_info(self, url, download=True, ie_key=None, extra_info={},
632 Returns a list with a dictionary for each video we find.
633 If 'download', also downloads the videos.
634 extra_info is a dict containing the extra values to add to each result
638 ies = [self.get_info_extractor(ie_key)]
643 if not ie.suitable(url):
647 self.report_warning('The program functionality for this site has been marked as broken, '
648 'and will probably not work.')
651 ie_result = ie.extract(url)
652 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
654 if isinstance(ie_result, list):
655 # Backwards compatibility: old IE result format
657 '_type': 'compat_list',
658 'entries': ie_result,
660 self.add_default_extra_info(ie_result, ie, url)
662 return self.process_ie_result(ie_result, download, extra_info)
665 except ExtractorError as de: # An error we somewhat expected
666 self.report_error(compat_str(de), de.format_traceback())
668 except MaxDownloadsReached:
670 except Exception as e:
671 if self.params.get('ignoreerrors', False):
672 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
677 self.report_error('no suitable InfoExtractor for URL %s' % url)
679 def add_default_extra_info(self, ie_result, ie, url):
680 self.add_extra_info(ie_result, {
681 'extractor': ie.IE_NAME,
683 'webpage_url_basename': url_basename(url),
684 'extractor_key': ie.ie_key(),
687 def process_ie_result(self, ie_result, download=True, extra_info={}):
689 Take the result of the ie(may be modified) and resolve all unresolved
690 references (URLs, playlist items).
692 It will also download the videos if 'download'.
693 Returns the resolved ie_result.
696 result_type = ie_result.get('_type', 'video')
698 if result_type in ('url', 'url_transparent'):
699 extract_flat = self.params.get('extract_flat', False)
700 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
701 extract_flat is True):
702 if self.params.get('forcejson', False):
703 self.to_stdout(json.dumps(ie_result))
706 if result_type == 'video':
707 self.add_extra_info(ie_result, extra_info)
708 return self.process_video_result(ie_result, download=download)
709 elif result_type == 'url':
710 # We have to add extra_info to the results because it may be
711 # contained in a playlist
712 return self.extract_info(ie_result['url'],
714 ie_key=ie_result.get('ie_key'),
715 extra_info=extra_info)
716 elif result_type == 'url_transparent':
717 # Use the information from the embedding page
718 info = self.extract_info(
719 ie_result['url'], ie_key=ie_result.get('ie_key'),
720 extra_info=extra_info, download=False, process=False)
722 force_properties = dict(
723 (k, v) for k, v in ie_result.items() if v is not None)
724 for f in ('_type', 'url'):
725 if f in force_properties:
726 del force_properties[f]
727 new_result = info.copy()
728 new_result.update(force_properties)
730 assert new_result.get('_type') != 'url_transparent'
732 return self.process_ie_result(
733 new_result, download=download, extra_info=extra_info)
734 elif result_type == 'playlist' or result_type == 'multi_video':
735 # We process each entry in the playlist
736 playlist = ie_result.get('title', None) or ie_result.get('id', None)
737 self.to_screen('[download] Downloading playlist: %s' % playlist)
739 playlist_results = []
741 playliststart = self.params.get('playliststart', 1) - 1
742 playlistend = self.params.get('playlistend', None)
743 # For backwards compatibility, interpret -1 as whole list
744 if playlistend == -1:
747 playlistitems_str = self.params.get('playlist_items', None)
749 if playlistitems_str is not None:
750 def iter_playlistitems(format):
751 for string_segment in format.split(','):
752 if '-' in string_segment:
753 start, end = string_segment.split('-')
754 for item in range(int(start), int(end) + 1):
757 yield int(string_segment)
758 playlistitems = iter_playlistitems(playlistitems_str)
760 ie_entries = ie_result['entries']
761 if isinstance(ie_entries, list):
762 n_all_entries = len(ie_entries)
764 entries = [ie_entries[i - 1] for i in playlistitems]
766 entries = ie_entries[playliststart:playlistend]
767 n_entries = len(entries)
769 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
770 (ie_result['extractor'], playlist, n_all_entries, n_entries))
771 elif isinstance(ie_entries, PagedList):
774 for item in playlistitems:
775 entries.extend(ie_entries.getslice(
779 entries = ie_entries.getslice(
780 playliststart, playlistend)
781 n_entries = len(entries)
783 "[%s] playlist %s: Downloading %d videos" %
784 (ie_result['extractor'], playlist, n_entries))
787 entry_list = list(ie_entries)
788 entries = [entry_list[i - 1] for i in playlistitems]
790 entries = list(itertools.islice(
791 ie_entries, playliststart, playlistend))
792 n_entries = len(entries)
794 "[%s] playlist %s: Downloading %d videos" %
795 (ie_result['extractor'], playlist, n_entries))
797 if self.params.get('playlistreverse', False):
798 entries = entries[::-1]
800 for i, entry in enumerate(entries, 1):
801 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
803 'n_entries': n_entries,
804 'playlist': playlist,
805 'playlist_id': ie_result.get('id'),
806 'playlist_title': ie_result.get('title'),
807 'playlist_index': i + playliststart,
808 'extractor': ie_result['extractor'],
809 'webpage_url': ie_result['webpage_url'],
810 'webpage_url_basename': url_basename(ie_result['webpage_url']),
811 'extractor_key': ie_result['extractor_key'],
814 reason = self._match_entry(entry, incomplete=True)
815 if reason is not None:
816 self.to_screen('[download] ' + reason)
819 entry_result = self.process_ie_result(entry,
822 playlist_results.append(entry_result)
823 ie_result['entries'] = playlist_results
825 elif result_type == 'compat_list':
827 'Extractor %s returned a compat_list result. '
828 'It needs to be updated.' % ie_result.get('extractor'))
834 'extractor': ie_result['extractor'],
835 'webpage_url': ie_result['webpage_url'],
836 'webpage_url_basename': url_basename(ie_result['webpage_url']),
837 'extractor_key': ie_result['extractor_key'],
841 ie_result['entries'] = [
842 self.process_ie_result(_fixup(r), download, extra_info)
843 for r in ie_result['entries']
847 raise Exception('Invalid result type: %s' % result_type)
849 def _apply_format_filter(self, format_spec, available_formats):
850 " Returns a tuple of the remaining format_spec and filtered formats "
860 operator_rex = re.compile(r'''(?x)\s*\[
861 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
862 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
863 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
865 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
866 m = operator_rex.search(format_spec)
869 comparison_value = int(m.group('value'))
871 comparison_value = parse_filesize(m.group('value'))
872 if comparison_value is None:
873 comparison_value = parse_filesize(m.group('value') + 'B')
874 if comparison_value is None:
876 'Invalid value %r in format specification %r' % (
877 m.group('value'), format_spec))
878 op = OPERATORS[m.group('op')]
885 str_operator_rex = re.compile(r'''(?x)\s*\[
886 \s*(?P<key>ext|acodec|vcodec|container|protocol)
887 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
888 \s*(?P<value>[a-zA-Z0-9_-]+)
890 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
891 m = str_operator_rex.search(format_spec)
893 comparison_value = m.group('value')
894 op = STR_OPERATORS[m.group('op')]
897 raise ValueError('Invalid format specification %r' % format_spec)
900 actual_value = f.get(m.group('key'))
901 if actual_value is None:
902 return m.group('none_inclusive')
903 return op(actual_value, comparison_value)
904 new_formats = [f for f in available_formats if _filter(f)]
906 new_format_spec = format_spec[:-len(m.group(0))]
907 if not new_format_spec:
908 new_format_spec = 'best'
910 return (new_format_spec, new_formats)
912 def select_format(self, format_spec, available_formats):
913 while format_spec.endswith(']'):
914 format_spec, available_formats = self._apply_format_filter(
915 format_spec, available_formats)
916 if not available_formats:
919 if format_spec == 'best' or format_spec is None:
920 return available_formats[-1]
921 elif format_spec == 'worst':
922 audiovideo_formats = [
923 f for f in available_formats
924 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
925 if audiovideo_formats:
926 return audiovideo_formats[0]
927 return available_formats[0]
928 elif format_spec == 'bestaudio':
930 f for f in available_formats
931 if f.get('vcodec') == 'none']
933 return audio_formats[-1]
934 elif format_spec == 'worstaudio':
936 f for f in available_formats
937 if f.get('vcodec') == 'none']
939 return audio_formats[0]
940 elif format_spec == 'bestvideo':
942 f for f in available_formats
943 if f.get('acodec') == 'none']
945 return video_formats[-1]
946 elif format_spec == 'worstvideo':
948 f for f in available_formats
949 if f.get('acodec') == 'none']
951 return video_formats[0]
953 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
954 if format_spec in extensions:
955 filter_f = lambda f: f['ext'] == format_spec
957 filter_f = lambda f: f['format_id'] == format_spec
958 matches = list(filter(filter_f, available_formats))
963 def _calc_headers(self, info_dict):
964 res = std_headers.copy()
966 add_headers = info_dict.get('http_headers')
968 res.update(add_headers)
970 cookies = self._calc_cookies(info_dict)
972 res['Cookie'] = cookies
976 def _calc_cookies(self, info_dict):
977 pr = compat_urllib_request.Request(info_dict['url'])
978 self.cookiejar.add_cookie_header(pr)
979 return pr.get_header('Cookie')
981 def process_video_result(self, info_dict, download=True):
982 assert info_dict.get('_type', 'video') == 'video'
984 if 'id' not in info_dict:
985 raise ExtractorError('Missing "id" field in extractor result')
986 if 'title' not in info_dict:
987 raise ExtractorError('Missing "title" field in extractor result')
989 if 'playlist' not in info_dict:
990 # It isn't part of a playlist
991 info_dict['playlist'] = None
992 info_dict['playlist_index'] = None
994 thumbnails = info_dict.get('thumbnails')
995 if thumbnails is None:
996 thumbnail = info_dict.get('thumbnail')
998 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1000 thumbnails.sort(key=lambda t: (
1001 t.get('preference'), t.get('width'), t.get('height'),
1002 t.get('id'), t.get('url')))
1003 for i, t in enumerate(thumbnails):
1004 if 'width' in t and 'height' in t:
1005 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1006 if t.get('id') is None:
1009 if thumbnails and 'thumbnail' not in info_dict:
1010 info_dict['thumbnail'] = thumbnails[-1]['url']
1012 if 'display_id' not in info_dict and 'id' in info_dict:
1013 info_dict['display_id'] = info_dict['id']
1015 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1016 # Working around negative timestamps in Windows
1017 # (see http://bugs.python.org/issue1646728)
1018 if info_dict['timestamp'] < 0 and os.name == 'nt':
1019 info_dict['timestamp'] = 0
1020 upload_date = datetime.datetime.utcfromtimestamp(
1021 info_dict['timestamp'])
1022 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1024 if self.params.get('listsubtitles', False):
1025 if 'automatic_captions' in info_dict:
1026 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1027 self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
1029 info_dict['requested_subtitles'] = self.process_subtitles(
1030 info_dict['id'], info_dict.get('subtitles'),
1031 info_dict.get('automatic_captions'))
1033 # This extractors handle format selection themselves
1034 if info_dict['extractor'] in ['Youku']:
1036 self.process_info(info_dict)
1039 # We now pick which formats have to be downloaded
1040 if info_dict.get('formats') is None:
1041 # There's only one format available
1042 formats = [info_dict]
1044 formats = info_dict['formats']
1047 raise ExtractorError('No video formats found!')
1049 # We check that all the formats have the format and format_id fields
1050 for i, format in enumerate(formats):
1051 if 'url' not in format:
1052 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1054 if format.get('format_id') is None:
1055 format['format_id'] = compat_str(i)
1056 if format.get('format') is None:
1057 format['format'] = '{id} - {res}{note}'.format(
1058 id=format['format_id'],
1059 res=self.format_resolution(format),
1060 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1062 # Automatically determine file extension if missing
1063 if 'ext' not in format:
1064 format['ext'] = determine_ext(format['url']).lower()
1065 # Add HTTP headers, so that external programs can use them from the
1067 full_format_info = info_dict.copy()
1068 full_format_info.update(format)
1069 format['http_headers'] = self._calc_headers(full_format_info)
1071 format_limit = self.params.get('format_limit', None)
1073 formats = list(takewhile_inclusive(
1074 lambda f: f['format_id'] != format_limit, formats
1077 # TODO Central sorting goes here
1079 if formats[0] is not info_dict:
1080 # only set the 'formats' fields if the original info_dict list them
1081 # otherwise we end up with a circular reference, the first (and unique)
1082 # element in the 'formats' field in info_dict is info_dict itself,
1083 # wich can't be exported to json
1084 info_dict['formats'] = formats
1085 if self.params.get('listformats'):
1086 self.list_formats(info_dict)
1088 if self.params.get('list_thumbnails'):
1089 self.list_thumbnails(info_dict)
1092 req_format = self.params.get('format')
1093 if req_format is None:
1095 formats_to_download = []
1096 if req_format == 'all':
1097 formats_to_download = formats
1099 for rfstr in req_format.split(','):
1100 # We can accept formats requested in the format: 34/5/best, we pick
1101 # the first that is available, starting from left
1102 req_formats = rfstr.split('/')
1103 for rf in req_formats:
1104 if re.match(r'.+?\+.+?', rf) is not None:
1105 # Two formats have been requested like '137+139'
1106 format_1, format_2 = rf.split('+')
1107 formats_info = (self.select_format(format_1, formats),
1108 self.select_format(format_2, formats))
1109 if all(formats_info):
1110 # The first format must contain the video and the
1112 if formats_info[0].get('vcodec') == 'none':
1113 self.report_error('The first format must '
1114 'contain the video, try using '
1115 '"-f %s+%s"' % (format_2, format_1))
1118 formats_info[0]['ext']
1119 if self.params.get('merge_output_format') is None
1120 else self.params['merge_output_format'])
1122 'requested_formats': formats_info,
1123 'format': '%s+%s' % (formats_info[0].get('format'),
1124 formats_info[1].get('format')),
1125 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1126 formats_info[1].get('format_id')),
1127 'width': formats_info[0].get('width'),
1128 'height': formats_info[0].get('height'),
1129 'resolution': formats_info[0].get('resolution'),
1130 'fps': formats_info[0].get('fps'),
1131 'vcodec': formats_info[0].get('vcodec'),
1132 'vbr': formats_info[0].get('vbr'),
1133 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1134 'acodec': formats_info[1].get('acodec'),
1135 'abr': formats_info[1].get('abr'),
1139 selected_format = None
1141 selected_format = self.select_format(rf, formats)
1142 if selected_format is not None:
1143 formats_to_download.append(selected_format)
1145 if not formats_to_download:
1146 raise ExtractorError('requested format not available',
1150 if len(formats_to_download) > 1:
1151 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1152 for format in formats_to_download:
1153 new_info = dict(info_dict)
1154 new_info.update(format)
1155 self.process_info(new_info)
1156 # We update the info dict with the best quality format (backwards compatibility)
1157 info_dict.update(formats_to_download[-1])
1160 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1161 """Select the requested subtitles and their format"""
1163 if normal_subtitles and self.params.get('writesubtitles'):
1164 available_subs.update(normal_subtitles)
1165 if automatic_captions and self.params.get('writeautomaticsub'):
1166 for lang, cap_info in automatic_captions.items():
1167 if lang not in available_subs:
1168 available_subs[lang] = cap_info
1170 if (not self.params.get('writesubtitles') and not
1171 self.params.get('writeautomaticsub') or not
1175 if self.params.get('allsubtitles', False):
1176 requested_langs = available_subs.keys()
1178 if self.params.get('subtitleslangs', False):
1179 requested_langs = self.params.get('subtitleslangs')
1180 elif 'en' in available_subs:
1181 requested_langs = ['en']
1183 requested_langs = [list(available_subs.keys())[0]]
1185 formats_query = self.params.get('subtitlesformat', 'best')
1186 formats_preference = formats_query.split('/') if formats_query else []
1188 for lang in requested_langs:
1189 formats = available_subs.get(lang)
1191 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1193 for ext in formats_preference:
1197 matches = list(filter(lambda f: f['ext'] == ext, formats))
1203 self.report_warning(
1204 'No subtitle format found matching "%s" for language %s, '
1205 'using %s' % (formats_query, lang, f['ext']))
1209 def process_info(self, info_dict):
1210 """Process a single resolved IE result."""
1212 assert info_dict.get('_type', 'video') == 'video'
1214 max_downloads = self.params.get('max_downloads')
1215 if max_downloads is not None:
1216 if self._num_downloads >= int(max_downloads):
1217 raise MaxDownloadsReached()
1219 info_dict['fulltitle'] = info_dict['title']
1220 if len(info_dict['title']) > 200:
1221 info_dict['title'] = info_dict['title'][:197] + '...'
1223 if 'format' not in info_dict:
1224 info_dict['format'] = info_dict['ext']
1226 reason = self._match_entry(info_dict, incomplete=False)
1227 if reason is not None:
1228 self.to_screen('[download] ' + reason)
1231 self._num_downloads += 1
1233 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1236 if self.params.get('forcetitle', False):
1237 self.to_stdout(info_dict['fulltitle'])
1238 if self.params.get('forceid', False):
1239 self.to_stdout(info_dict['id'])
1240 if self.params.get('forceurl', False):
1241 if info_dict.get('requested_formats') is not None:
1242 for f in info_dict['requested_formats']:
1243 self.to_stdout(f['url'] + f.get('play_path', ''))
1245 # For RTMP URLs, also include the playpath
1246 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1247 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1248 self.to_stdout(info_dict['thumbnail'])
1249 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1250 self.to_stdout(info_dict['description'])
1251 if self.params.get('forcefilename', False) and filename is not None:
1252 self.to_stdout(filename)
1253 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1254 self.to_stdout(formatSeconds(info_dict['duration']))
1255 if self.params.get('forceformat', False):
1256 self.to_stdout(info_dict['format'])
1257 if self.params.get('forcejson', False):
1258 self.to_stdout(json.dumps(info_dict))
1260 # Do nothing else if in simulate mode
1261 if self.params.get('simulate', False):
1264 if filename is None:
1268 dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1269 if dn and not os.path.exists(dn):
1271 except (OSError, IOError) as err:
1272 self.report_error('unable to create directory ' + compat_str(err))
1275 if self.params.get('writedescription', False):
1276 descfn = filename + '.description'
1277 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1278 self.to_screen('[info] Video description is already present')
1279 elif info_dict.get('description') is None:
1280 self.report_warning('There\'s no description to write.')
1283 self.to_screen('[info] Writing video description to: ' + descfn)
1284 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1285 descfile.write(info_dict['description'])
1286 except (OSError, IOError):
1287 self.report_error('Cannot write description file ' + descfn)
1290 if self.params.get('writeannotations', False):
1291 annofn = filename + '.annotations.xml'
1292 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1293 self.to_screen('[info] Video annotations are already present')
1296 self.to_screen('[info] Writing video annotations to: ' + annofn)
1297 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1298 annofile.write(info_dict['annotations'])
1299 except (KeyError, TypeError):
1300 self.report_warning('There are no annotations to write.')
1301 except (OSError, IOError):
1302 self.report_error('Cannot write annotations file: ' + annofn)
1305 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1306 self.params.get('writeautomaticsub')])
1308 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1309 # subtitles download errors are already managed as troubles in relevant IE
1310 # that way it will silently go on when used with unsupporting IE
1311 subtitles = info_dict['requested_subtitles']
1312 ie = self.get_info_extractor(info_dict['extractor_key'])
1313 for sub_lang, sub_info in subtitles.items():
1314 sub_format = sub_info['ext']
1315 if sub_info.get('data') is not None:
1316 sub_data = sub_info['data']
1319 sub_data = ie._download_webpage(
1320 sub_info['url'], info_dict['id'], note=False)
1321 except ExtractorError as err:
1322 self.report_warning('Unable to download subtitle for "%s": %s' %
1323 (sub_lang, compat_str(err.cause)))
1326 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1327 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1328 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1330 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1331 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1332 subfile.write(sub_data)
1333 except (OSError, IOError):
1334 self.report_error('Cannot write subtitles file ' + sub_filename)
1337 if self.params.get('writeinfojson', False):
1338 infofn = os.path.splitext(filename)[0] + '.info.json'
1339 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1340 self.to_screen('[info] Video description metadata is already present')
1342 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1344 write_json_file(info_dict, infofn)
1345 except (OSError, IOError):
1346 self.report_error('Cannot write metadata to JSON file ' + infofn)
1349 self._write_thumbnails(info_dict, filename)
1351 if not self.params.get('skip_download', False):
1354 fd = get_suitable_downloader(info, self.params)(self, self.params)
1355 for ph in self._progress_hooks:
1356 fd.add_progress_hook(ph)
1357 if self.params.get('verbose'):
1358 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1359 return fd.download(name, info)
1361 if info_dict.get('requested_formats') is not None:
1364 merger = FFmpegMergerPP(self)
1365 if not merger.available:
1367 self.report_warning('You have requested multiple '
1368 'formats but ffmpeg or avconv are not installed.'
1369 ' The formats won\'t be merged')
1371 postprocessors = [merger]
1372 if os.path.exists(encodeFilename(filename)):
1374 '[download] %s has already been downloaded and '
1375 'merged' % filename)
1377 for f in info_dict['requested_formats']:
1378 new_info = dict(info_dict)
1380 fname = self.prepare_filename(new_info)
1381 fname = prepend_extension(fname, 'f%s' % f['format_id'])
1382 downloaded.append(fname)
1383 partial_success = dl(fname, new_info)
1384 success = success and partial_success
1385 info_dict['__postprocessors'] = postprocessors
1386 info_dict['__files_to_merge'] = downloaded
1388 # Just a single file
1389 success = dl(filename, info_dict)
1390 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1391 self.report_error('unable to download video data: %s' % str(err))
1393 except (OSError, IOError) as err:
1394 raise UnavailableVideoError(err)
1395 except (ContentTooShortError, ) as err:
1396 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1401 fixup_policy = self.params.get('fixup')
1402 if fixup_policy is None:
1403 fixup_policy = 'detect_or_warn'
1405 stretched_ratio = info_dict.get('stretched_ratio')
1406 if stretched_ratio is not None and stretched_ratio != 1:
1407 if fixup_policy == 'warn':
1408 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1409 info_dict['id'], stretched_ratio))
1410 elif fixup_policy == 'detect_or_warn':
1411 stretched_pp = FFmpegFixupStretchedPP(self)
1412 if stretched_pp.available:
1413 info_dict.setdefault('__postprocessors', [])
1414 info_dict['__postprocessors'].append(stretched_pp)
1416 self.report_warning(
1417 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1418 info_dict['id'], stretched_ratio))
1420 assert fixup_policy in ('ignore', 'never')
1422 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1423 if fixup_policy == 'warn':
1424 self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1426 elif fixup_policy == 'detect_or_warn':
1427 fixup_pp = FFmpegFixupM4aPP(self)
1428 if fixup_pp.available:
1429 info_dict.setdefault('__postprocessors', [])
1430 info_dict['__postprocessors'].append(fixup_pp)
1432 self.report_warning(
1433 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1436 assert fixup_policy in ('ignore', 'never')
1439 self.post_process(filename, info_dict)
1440 except (PostProcessingError) as err:
1441 self.report_error('postprocessing: %s' % str(err))
1443 self.record_download_archive(info_dict)
1445 def download(self, url_list):
1446 """Download a given list of URLs."""
1447 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1448 if (len(url_list) > 1 and
1449 '%' not in outtmpl and
1450 self.params.get('max_downloads') != 1):
1451 raise SameFileError(outtmpl)
1453 for url in url_list:
1455 # It also downloads the videos
1456 res = self.extract_info(url)
1457 except UnavailableVideoError:
1458 self.report_error('unable to download video')
1459 except MaxDownloadsReached:
1460 self.to_screen('[info] Maximum number of downloaded files reached.')
1463 if self.params.get('dump_single_json', False):
1464 self.to_stdout(json.dumps(res))
1466 return self._download_retcode
1468 def download_with_info_file(self, info_filename):
1469 with contextlib.closing(fileinput.FileInput(
1470 [info_filename], mode='r',
1471 openhook=fileinput.hook_encoded('utf-8'))) as f:
1472 # FileInput doesn't have a read method, we can't call json.load
1473 info = json.loads('\n'.join(f))
1475 self.process_ie_result(info, download=True)
1476 except DownloadError:
1477 webpage_url = info.get('webpage_url')
1478 if webpage_url is not None:
1479 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1480 return self.download([webpage_url])
1483 return self._download_retcode
1485 def post_process(self, filename, ie_info):
1486 """Run all the postprocessors on the given file."""
1487 info = dict(ie_info)
1488 info['filepath'] = filename
1490 if ie_info.get('__postprocessors') is not None:
1491 pps_chain.extend(ie_info['__postprocessors'])
1492 pps_chain.extend(self._pps)
1493 for pp in pps_chain:
1494 old_filename = info['filepath']
1496 files_to_delete, info = pp.run(info)
1497 except PostProcessingError as e:
1498 self.report_error(e.msg)
1499 if files_to_delete and not self.params.get('keepvideo', False):
1500 for old_filename in files_to_delete:
1501 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1503 os.remove(encodeFilename(old_filename))
1504 except (IOError, OSError):
1505 self.report_warning('Unable to remove downloaded original file')
1507 def _make_archive_id(self, info_dict):
1508 # Future-proof against any change in case
1509 # and backwards compatibility with prior versions
1510 extractor = info_dict.get('extractor_key')
1511 if extractor is None:
1512 if 'id' in info_dict:
1513 extractor = info_dict.get('ie_key') # key in a playlist
1514 if extractor is None:
1515 return None # Incomplete video information
1516 return extractor.lower() + ' ' + info_dict['id']
1518 def in_download_archive(self, info_dict):
1519 fn = self.params.get('download_archive')
1523 vid_id = self._make_archive_id(info_dict)
1525 return False # Incomplete video information
1528 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1529 for line in archive_file:
1530 if line.strip() == vid_id:
1532 except IOError as ioe:
1533 if ioe.errno != errno.ENOENT:
1537 def record_download_archive(self, info_dict):
1538 fn = self.params.get('download_archive')
1541 vid_id = self._make_archive_id(info_dict)
1543 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1544 archive_file.write(vid_id + '\n')
1547 def format_resolution(format, default='unknown'):
1548 if format.get('vcodec') == 'none':
1550 if format.get('resolution') is not None:
1551 return format['resolution']
1552 if format.get('height') is not None:
1553 if format.get('width') is not None:
1554 res = '%sx%s' % (format['width'], format['height'])
1556 res = '%sp' % format['height']
1557 elif format.get('width') is not None:
1558 res = '?x%d' % format['width']
1563 def _format_note(self, fdict):
1565 if fdict.get('ext') in ['f4f', 'f4m']:
1566 res += '(unsupported) '
1567 if fdict.get('format_note') is not None:
1568 res += fdict['format_note'] + ' '
1569 if fdict.get('tbr') is not None:
1570 res += '%4dk ' % fdict['tbr']
1571 if fdict.get('container') is not None:
1574 res += '%s container' % fdict['container']
1575 if (fdict.get('vcodec') is not None and
1576 fdict.get('vcodec') != 'none'):
1579 res += fdict['vcodec']
1580 if fdict.get('vbr') is not None:
1582 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1584 if fdict.get('vbr') is not None:
1585 res += '%4dk' % fdict['vbr']
1586 if fdict.get('fps') is not None:
1587 res += ', %sfps' % fdict['fps']
1588 if fdict.get('acodec') is not None:
1591 if fdict['acodec'] == 'none':
1594 res += '%-5s' % fdict['acodec']
1595 elif fdict.get('abr') is not None:
1599 if fdict.get('abr') is not None:
1600 res += '@%3dk' % fdict['abr']
1601 if fdict.get('asr') is not None:
1602 res += ' (%5dHz)' % fdict['asr']
1603 if fdict.get('filesize') is not None:
1606 res += format_bytes(fdict['filesize'])
1607 elif fdict.get('filesize_approx') is not None:
1610 res += '~' + format_bytes(fdict['filesize_approx'])
1613 def list_formats(self, info_dict):
1614 formats = info_dict.get('formats', [info_dict])
1616 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1618 if f.get('preference') is None or f['preference'] >= -1000]
1619 if len(formats) > 1:
1620 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1622 header_line = ['format code', 'extension', 'resolution', 'note']
1624 '[info] Available formats for %s:\n%s' %
1625 (info_dict['id'], render_table(header_line, table)))
1627 def list_thumbnails(self, info_dict):
1628 thumbnails = info_dict.get('thumbnails')
1630 tn_url = info_dict.get('thumbnail')
1632 thumbnails = [{'id': '0', 'url': tn_url}]
1635 '[info] No thumbnails present for %s' % info_dict['id'])
1639 '[info] Thumbnails for %s:' % info_dict['id'])
1640 self.to_screen(render_table(
1641 ['ID', 'width', 'height', 'URL'],
1642 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1644 def list_subtitles(self, video_id, subtitles, name='subtitles'):
1646 self.to_screen('%s has no %s' % (video_id, name))
1649 'Available %s for %s:' % (name, video_id))
1650 self.to_screen(render_table(
1651 ['Language', 'formats'],
1652 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1653 for lang, formats in subtitles.items()]))
1655 def urlopen(self, req):
1656 """ Start an HTTP download """
1658 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1659 # always respected by websites, some tend to give out URLs with non percent-encoded
1660 # non-ASCII characters (see telemb.py, ard.py [#3412])
1661 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1662 # To work around aforementioned issue we will replace request's original URL with
1663 # percent-encoded one
1664 req_is_string = isinstance(req, compat_basestring)
1665 url = req if req_is_string else req.get_full_url()
1666 url_escaped = escape_url(url)
1668 # Substitute URL if any change after escaping
1669 if url != url_escaped:
1673 req = compat_urllib_request.Request(
1674 url_escaped, data=req.data, headers=req.headers,
1675 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1677 return self._opener.open(req, timeout=self._socket_timeout)
1679 def print_debug_header(self):
1680 if not self.params.get('verbose'):
1683 if type('') is not compat_str:
1684 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1685 self.report_warning(
1686 'Your Python is broken! Update to a newer and supported version')
1688 stdout_encoding = getattr(
1689 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1691 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1692 locale.getpreferredencoding(),
1693 sys.getfilesystemencoding(),
1695 self.get_encoding()))
1696 write_string(encoding_str, encoding=None)
1698 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1700 sp = subprocess.Popen(
1701 ['git', 'rev-parse', '--short', 'HEAD'],
1702 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1703 cwd=os.path.dirname(os.path.abspath(__file__)))
1704 out, err = sp.communicate()
1705 out = out.decode().strip()
1706 if re.match('[0-9a-f]+', out):
1707 self._write_string('[debug] Git HEAD: ' + out + '\n')
1713 self._write_string('[debug] Python version %s - %s\n' % (
1714 platform.python_version(), platform_name()))
1716 exe_versions = FFmpegPostProcessor.get_versions(self)
1717 exe_versions['rtmpdump'] = rtmpdump_version()
1718 exe_str = ', '.join(
1720 for exe, v in sorted(exe_versions.items())
1725 self._write_string('[debug] exe versions: %s\n' % exe_str)
1728 for handler in self._opener.handlers:
1729 if hasattr(handler, 'proxies'):
1730 proxy_map.update(handler.proxies)
1731 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1733 if self.params.get('call_home', False):
1734 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1735 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1736 latest_version = self.urlopen(
1737 'https://yt-dl.org/latest/version').read().decode('utf-8')
1738 if version_tuple(latest_version) > version_tuple(__version__):
1739 self.report_warning(
1740 'You are using an outdated version (newest version: %s)! '
1741 'See https://yt-dl.org/update if you need help updating.' %
1744 def _setup_opener(self):
1745 timeout_val = self.params.get('socket_timeout')
1746 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1748 opts_cookiefile = self.params.get('cookiefile')
1749 opts_proxy = self.params.get('proxy')
1751 if opts_cookiefile is None:
1752 self.cookiejar = compat_cookiejar.CookieJar()
1754 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1756 if os.access(opts_cookiefile, os.R_OK):
1757 self.cookiejar.load()
1759 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1761 if opts_proxy is not None:
1762 if opts_proxy == '':
1765 proxies = {'http': opts_proxy, 'https': opts_proxy}
1767 proxies = compat_urllib_request.getproxies()
1768 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1769 if 'http' in proxies and 'https' not in proxies:
1770 proxies['https'] = proxies['http']
1771 proxy_handler = PerRequestProxyHandler(proxies)
1773 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1774 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1775 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1776 opener = compat_urllib_request.build_opener(
1777 proxy_handler, https_handler, cookie_processor, ydlh)
1779 # Delete the default user-agent header, which would otherwise apply in
1780 # cases where our custom HTTP handler doesn't come into play
1781 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1782 opener.addheaders = []
1783 self._opener = opener
1785 def encode(self, s):
1786 if isinstance(s, bytes):
1787 return s # Already encoded
1790 return s.encode(self.get_encoding())
1791 except UnicodeEncodeError as err:
1792 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1795 def get_encoding(self):
1796 encoding = self.params.get('encoding')
1797 if encoding is None:
1798 encoding = preferredencoding()
1801 def _write_thumbnails(self, info_dict, filename):
1802 if self.params.get('writethumbnail', False):
1803 thumbnails = info_dict.get('thumbnails')
1805 thumbnails = [thumbnails[-1]]
1806 elif self.params.get('write_all_thumbnails', False):
1807 thumbnails = info_dict.get('thumbnails')
1812 # No thumbnails present, so return immediately
1815 for t in thumbnails:
1816 thumb_ext = determine_ext(t['url'], 'jpg')
1817 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
1818 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
1819 thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
1821 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
1822 self.to_screen('[%s] %s: Thumbnail %sis already present' %
1823 (info_dict['extractor'], info_dict['id'], thumb_display_id))
1825 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
1826 (info_dict['extractor'], info_dict['id'], thumb_display_id))
1828 uf = self.urlopen(t['url'])
1829 with open(thumb_filename, 'wb') as thumbf:
1830 shutil.copyfileobj(uf, thumbf)
1831 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
1832 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
1833 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1834 self.report_warning('Unable to download thumbnail "%s": %s' %
1835 (t['url'], compat_str(err)))