2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import, unicode_literals
32 compat_urllib_request,
58 UnavailableVideoError,
66 from .cache import Cache
67 from .extractor import get_info_extractor, gen_extractors
68 from .downloader import get_suitable_downloader
69 from .downloader.rtmp import rtmpdump_version
70 from .postprocessor import FFmpegMergerPP, FFmpegPostProcessor
71 from .version import __version__
74 class YoutubeDL(object):
77 YoutubeDL objects are the ones responsible of downloading the
78 actual video file and writing it to disk if the user has requested
79 it, among some other tasks. In most cases there should be one per
80 program. As, given a video URL, the downloader doesn't know how to
81 extract all the needed information, task that InfoExtractors do, it
82 has to pass the URL to one of them.
84 For this, YoutubeDL objects have a method that allows
85 InfoExtractors to be registered in a given order. When it is passed
86 a URL, the YoutubeDL object handles it to the first InfoExtractor it
87 finds that reports being able to handle it. The InfoExtractor extracts
88 all the information about the video or videos the URL refers to, and
89 YoutubeDL process the extracted information, possibly using a File
90 Downloader to download the video.
92 YoutubeDL objects accept a lot of parameters. In order not to saturate
93 the object constructor with arguments, it receives a dictionary of
94 options instead. These options are available through the params
95 attribute for the InfoExtractors to use. The YoutubeDL also
96 registers itself as the downloader in charge for the InfoExtractors
97 that are added to it, so this is a "mutual registration".
101 username: Username for authentication purposes.
102 password: Password for authentication purposes.
103 videopassword: Password for acces a video.
104 usenetrc: Use netrc for authentication instead.
105 verbose: Print additional info to stdout.
106 quiet: Do not print messages to stdout.
107 no_warnings: Do not print out anything for warnings.
108 forceurl: Force printing final URL.
109 forcetitle: Force printing title.
110 forceid: Force printing ID.
111 forcethumbnail: Force printing thumbnail URL.
112 forcedescription: Force printing description.
113 forcefilename: Force printing final filename.
114 forceduration: Force printing duration.
115 forcejson: Force printing info_dict as JSON.
116 dump_single_json: Force printing the info_dict of the whole playlist
117 (or video) as a single JSON line.
118 simulate: Do not download the video files.
119 format: Video format code.
120 format_limit: Highest quality format to try.
121 outtmpl: Template for output names.
122 restrictfilenames: Do not allow "&" and spaces in file names
123 ignoreerrors: Do not stop on download errors.
124 nooverwrites: Prevent overwriting files.
125 playliststart: Playlist item to start at.
126 playlistend: Playlist item to end at.
127 matchtitle: Download only matching titles.
128 rejecttitle: Reject downloads for matching titles.
129 logger: Log messages to a logging.Logger instance.
130 logtostderr: Log messages to stderr instead of stdout.
131 writedescription: Write the video description to a .description file
132 writeinfojson: Write the video description to a .info.json file
133 writeannotations: Write the video annotations to a .annotations.xml file
134 writethumbnail: Write the thumbnail image to a file
135 writesubtitles: Write the video subtitles to a file
136 writeautomaticsub: Write the automatic subtitles to a file
137 allsubtitles: Downloads all the subtitles of the video
138 (requires writesubtitles or writeautomaticsub)
139 listsubtitles: Lists all available subtitles for the video
140 subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
141 subtitleslangs: List of languages of the subtitles to download
142 keepvideo: Keep the video file after post-processing
143 daterange: A DateRange object, download only if the upload_date is in the range.
144 skip_download: Skip the actual download of the video file
145 cachedir: Location of the cache files in the filesystem.
146 False to disable filesystem cache.
147 noplaylist: Download single video instead of a playlist if in doubt.
148 age_limit: An integer representing the user's age in years.
149 Unsuitable videos for the given age are skipped.
150 min_views: An integer representing the minimum view count the video
151 must have in order to not be skipped.
152 Videos without view count information are always
153 downloaded. None for no limit.
154 max_views: An integer representing the maximum view count.
155 Videos that are more popular than that are not
157 Videos without view count information are always
158 downloaded. None for no limit.
159 download_archive: File name of a file where all downloads are recorded.
160 Videos already present in the file are not downloaded
162 cookiefile: File name where cookies should be read from and dumped to.
163 nocheckcertificate:Do not verify SSL certificates
164 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
165 At the moment, this is only supported by YouTube.
166 proxy: URL of the proxy server to use
167 socket_timeout: Time to wait for unresponsive hosts, in seconds
168 bidi_workaround: Work around buggy terminals without bidirectional text
169 support, using fridibi
170 debug_printtraffic:Print out sent and received HTTP traffic
171 include_ads: Download ads as well
172 default_search: Prepend this string if an input url is not valid.
173 'auto' for elaborate guessing
174 encoding: Use this encoding instead of the system-specified.
175 extract_flat: Do not resolve URLs, return the immediate result.
176 Pass in 'in_playlist' to only show this behavior for
179 The following parameters are not used by YoutubeDL itself, they are used by
181 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
182 noresizebuffer, retries, continuedl, noprogress, consoletitle
184 The following options are used by the post processors:
185 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
186 otherwise prefer avconv.
187 exec_cmd: Arbitrary command to run after downloading
193 _download_retcode = None
194 _num_downloads = None
197 def __init__(self, params=None, auto_init=True):
198 """Create a FileDownloader object with the given options."""
202 self._ies_instances = {}
204 self._progress_hooks = []
205 self._download_retcode = 0
206 self._num_downloads = 0
207 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
208 self._err_file = sys.stderr
210 self.cache = Cache(self)
212 if params.get('bidi_workaround', False):
215 master, slave = pty.openpty()
216 width = get_term_width()
220 width_args = ['-w', str(width)]
222 stdin=subprocess.PIPE,
224 stderr=self._err_file)
226 self._output_process = subprocess.Popen(
227 ['bidiv'] + width_args, **sp_kwargs
230 self._output_process = subprocess.Popen(
231 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
232 self._output_channel = os.fdopen(master, 'rb')
233 except OSError as ose:
235 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
239 if (sys.version_info >= (3,) and sys.platform != 'win32' and
240 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
241 and not params.get('restrictfilenames', False)):
242 # On Python 3, the Unicode filesystem API will throw errors (#1474)
244 'Assuming --restrict-filenames since file system encoding '
245 'cannot encode all characters. '
246 'Set the LC_ALL environment variable to fix this.')
247 self.params['restrictfilenames'] = True
249 if '%(stitle)s' in self.params.get('outtmpl', ''):
250 self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
255 self.print_debug_header()
256 self.add_default_info_extractors()
258 def warn_if_short_id(self, argv):
259 # short YouTube ID starting with dash?
261 i for i, a in enumerate(argv)
262 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
266 [a for i, a in enumerate(argv) if i not in idxs] +
267 ['--'] + [argv[i] for i in idxs]
270 'Long argument string detected. '
271 'Use -- to separate parameters and URLs, like this:\n%s\n' %
272 args_to_str(correct_argv))
274 def add_info_extractor(self, ie):
275 """Add an InfoExtractor object to the end of the list."""
277 self._ies_instances[ie.ie_key()] = ie
278 ie.set_downloader(self)
280 def get_info_extractor(self, ie_key):
282 Get an instance of an IE with name ie_key, it will try to get one from
283 the _ies list, if there's no instance it will create a new one and add
284 it to the extractor list.
286 ie = self._ies_instances.get(ie_key)
288 ie = get_info_extractor(ie_key)()
289 self.add_info_extractor(ie)
292 def add_default_info_extractors(self):
294 Add the InfoExtractors returned by gen_extractors to the end of the list
296 for ie in gen_extractors():
297 self.add_info_extractor(ie)
299 def add_post_processor(self, pp):
300 """Add a PostProcessor object to the end of the chain."""
302 pp.set_downloader(self)
304 def add_progress_hook(self, ph):
305 """Add the progress hook (currently only for the file downloader)"""
306 self._progress_hooks.append(ph)
308 def _bidi_workaround(self, message):
309 if not hasattr(self, '_output_channel'):
312 assert hasattr(self, '_output_process')
313 assert isinstance(message, compat_str)
314 line_count = message.count('\n') + 1
315 self._output_process.stdin.write((message + '\n').encode('utf-8'))
316 self._output_process.stdin.flush()
317 res = ''.join(self._output_channel.readline().decode('utf-8')
318 for _ in range(line_count))
319 return res[:-len('\n')]
321 def to_screen(self, message, skip_eol=False):
322 """Print message to stdout if not in quiet mode."""
323 return self.to_stdout(message, skip_eol, check_quiet=True)
325 def _write_string(self, s, out=None):
326 write_string(s, out=out, encoding=self.params.get('encoding'))
328 def to_stdout(self, message, skip_eol=False, check_quiet=False):
329 """Print message to stdout if not in quiet mode."""
330 if self.params.get('logger'):
331 self.params['logger'].debug(message)
332 elif not check_quiet or not self.params.get('quiet', False):
333 message = self._bidi_workaround(message)
334 terminator = ['\n', ''][skip_eol]
335 output = message + terminator
337 self._write_string(output, self._screen_file)
339 def to_stderr(self, message):
340 """Print message to stderr."""
341 assert isinstance(message, compat_str)
342 if self.params.get('logger'):
343 self.params['logger'].error(message)
345 message = self._bidi_workaround(message)
346 output = message + '\n'
347 self._write_string(output, self._err_file)
349 def to_console_title(self, message):
350 if not self.params.get('consoletitle', False):
352 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
353 # c_wchar_p() might not be necessary if `message` is
354 # already of type unicode()
355 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
356 elif 'TERM' in os.environ:
357 self._write_string('\033]0;%s\007' % message, self._screen_file)
359 def save_console_title(self):
360 if not self.params.get('consoletitle', False):
362 if 'TERM' in os.environ:
363 # Save the title on stack
364 self._write_string('\033[22;0t', self._screen_file)
366 def restore_console_title(self):
367 if not self.params.get('consoletitle', False):
369 if 'TERM' in os.environ:
370 # Restore the title from stack
371 self._write_string('\033[23;0t', self._screen_file)
374 self.save_console_title()
377 def __exit__(self, *args):
378 self.restore_console_title()
380 if self.params.get('cookiefile') is not None:
381 self.cookiejar.save()
383 def trouble(self, message=None, tb=None):
384 """Determine action to take when a download problem appears.
386 Depending on if the downloader has been configured to ignore
387 download errors or not, this method may throw an exception or
388 not when errors are found, after printing the message.
390 tb, if given, is additional traceback information.
392 if message is not None:
393 self.to_stderr(message)
394 if self.params.get('verbose'):
396 if sys.exc_info()[0]: # if .trouble has been called from an except block
398 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
399 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
400 tb += compat_str(traceback.format_exc())
402 tb_data = traceback.format_list(traceback.extract_stack())
403 tb = ''.join(tb_data)
405 if not self.params.get('ignoreerrors', False):
406 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
407 exc_info = sys.exc_info()[1].exc_info
409 exc_info = sys.exc_info()
410 raise DownloadError(message, exc_info)
411 self._download_retcode = 1
413 def report_warning(self, message):
415 Print the message to stderr, it will be prefixed with 'WARNING:'
416 If stderr is a tty file the 'WARNING:' will be colored
418 if self.params.get('logger') is not None:
419 self.params['logger'].warning(message)
421 if self.params.get('no_warnings'):
423 if self._err_file.isatty() and os.name != 'nt':
424 _msg_header = '\033[0;33mWARNING:\033[0m'
426 _msg_header = 'WARNING:'
427 warning_message = '%s %s' % (_msg_header, message)
428 self.to_stderr(warning_message)
430 def report_error(self, message, tb=None):
432 Do the same as trouble, but prefixes the message with 'ERROR:', colored
433 in red if stderr is a tty file.
435 if self._err_file.isatty() and os.name != 'nt':
436 _msg_header = '\033[0;31mERROR:\033[0m'
438 _msg_header = 'ERROR:'
439 error_message = '%s %s' % (_msg_header, message)
440 self.trouble(error_message, tb)
442 def report_file_already_downloaded(self, file_name):
443 """Report file has already been fully downloaded."""
445 self.to_screen('[download] %s has already been downloaded' % file_name)
446 except UnicodeEncodeError:
447 self.to_screen('[download] The file has already been downloaded')
449 def prepare_filename(self, info_dict):
450 """Generate the output filename."""
452 template_dict = dict(info_dict)
454 template_dict['epoch'] = int(time.time())
455 autonumber_size = self.params.get('autonumber_size')
456 if autonumber_size is None:
458 autonumber_templ = '%0' + str(autonumber_size) + 'd'
459 template_dict['autonumber'] = autonumber_templ % self._num_downloads
460 if template_dict.get('playlist_index') is not None:
461 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
462 if template_dict.get('resolution') is None:
463 if template_dict.get('width') and template_dict.get('height'):
464 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
465 elif template_dict.get('height'):
466 template_dict['resolution'] = '%sp' % template_dict['height']
467 elif template_dict.get('width'):
468 template_dict['resolution'] = '?x%d' % template_dict['width']
470 sanitize = lambda k, v: sanitize_filename(
472 restricted=self.params.get('restrictfilenames'),
474 template_dict = dict((k, sanitize(k, v))
475 for k, v in template_dict.items()
477 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
479 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
480 tmpl = compat_expanduser(outtmpl)
481 filename = tmpl % template_dict
483 except ValueError as err:
484 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
487 def _match_entry(self, info_dict):
488 """ Returns None iff the file should be downloaded """
490 video_title = info_dict.get('title', info_dict.get('id', 'video'))
491 if 'title' in info_dict:
492 # This can happen when we're just evaluating the playlist
493 title = info_dict['title']
494 matchtitle = self.params.get('matchtitle', False)
496 if not re.search(matchtitle, title, re.IGNORECASE):
497 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
498 rejecttitle = self.params.get('rejecttitle', False)
500 if re.search(rejecttitle, title, re.IGNORECASE):
501 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
502 date = info_dict.get('upload_date', None)
504 dateRange = self.params.get('daterange', DateRange())
505 if date not in dateRange:
506 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
507 view_count = info_dict.get('view_count', None)
508 if view_count is not None:
509 min_views = self.params.get('min_views')
510 if min_views is not None and view_count < min_views:
511 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
512 max_views = self.params.get('max_views')
513 if max_views is not None and view_count > max_views:
514 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
515 age_limit = self.params.get('age_limit')
516 if age_limit is not None:
517 actual_age_limit = info_dict.get('age_limit')
518 if actual_age_limit is None:
520 if age_limit < actual_age_limit:
521 return 'Skipping "' + title + '" because it is age restricted'
522 if self.in_download_archive(info_dict):
523 return '%s has already been recorded in archive' % video_title
527 def add_extra_info(info_dict, extra_info):
528 '''Set the keys from extra_info in info dict if they are missing'''
529 for key, value in extra_info.items():
530 info_dict.setdefault(key, value)
532 def extract_info(self, url, download=True, ie_key=None, extra_info={},
535 Returns a list with a dictionary for each video we find.
536 If 'download', also downloads the videos.
537 extra_info is a dict containing the extra values to add to each result
541 ies = [self.get_info_extractor(ie_key)]
546 if not ie.suitable(url):
550 self.report_warning('The program functionality for this site has been marked as broken, '
551 'and will probably not work.')
554 ie_result = ie.extract(url)
555 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
557 if isinstance(ie_result, list):
558 # Backwards compatibility: old IE result format
560 '_type': 'compat_list',
561 'entries': ie_result,
563 self.add_default_extra_info(ie_result, ie, url)
565 return self.process_ie_result(ie_result, download, extra_info)
568 except ExtractorError as de: # An error we somewhat expected
569 self.report_error(compat_str(de), de.format_traceback())
571 except MaxDownloadsReached:
573 except Exception as e:
574 if self.params.get('ignoreerrors', False):
575 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
580 self.report_error('no suitable InfoExtractor for URL %s' % url)
582 def add_default_extra_info(self, ie_result, ie, url):
583 self.add_extra_info(ie_result, {
584 'extractor': ie.IE_NAME,
586 'webpage_url_basename': url_basename(url),
587 'extractor_key': ie.ie_key(),
590 def process_ie_result(self, ie_result, download=True, extra_info={}):
592 Take the result of the ie(may be modified) and resolve all unresolved
593 references (URLs, playlist items).
595 It will also download the videos if 'download'.
596 Returns the resolved ie_result.
599 result_type = ie_result.get('_type', 'video')
601 if result_type in ('url', 'url_transparent'):
602 extract_flat = self.params.get('extract_flat', False)
603 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
604 extract_flat is True):
605 if self.params.get('forcejson', False):
606 self.to_stdout(json.dumps(ie_result))
609 if result_type == 'video':
610 self.add_extra_info(ie_result, extra_info)
611 return self.process_video_result(ie_result, download=download)
612 elif result_type == 'url':
613 # We have to add extra_info to the results because it may be
614 # contained in a playlist
615 return self.extract_info(ie_result['url'],
617 ie_key=ie_result.get('ie_key'),
618 extra_info=extra_info)
619 elif result_type == 'url_transparent':
620 # Use the information from the embedding page
621 info = self.extract_info(
622 ie_result['url'], ie_key=ie_result.get('ie_key'),
623 extra_info=extra_info, download=False, process=False)
625 def make_result(embedded_info):
626 new_result = ie_result.copy()
627 for f in ('_type', 'url', 'ext', 'player_url', 'formats',
628 'entries', 'ie_key', 'duration',
629 'subtitles', 'annotations', 'format',
630 'thumbnail', 'thumbnails'):
633 if f in embedded_info:
634 new_result[f] = embedded_info[f]
636 new_result = make_result(info)
638 assert new_result.get('_type') != 'url_transparent'
639 if new_result.get('_type') == 'compat_list':
640 new_result['entries'] = [
641 make_result(e) for e in new_result['entries']]
643 return self.process_ie_result(
644 new_result, download=download, extra_info=extra_info)
645 elif result_type == 'playlist' or result_type == 'multi_video':
646 # We process each entry in the playlist
647 playlist = ie_result.get('title', None) or ie_result.get('id', None)
648 self.to_screen('[download] Downloading playlist: %s' % playlist)
650 playlist_results = []
652 playliststart = self.params.get('playliststart', 1) - 1
653 playlistend = self.params.get('playlistend', None)
654 # For backwards compatibility, interpret -1 as whole list
655 if playlistend == -1:
658 ie_entries = ie_result['entries']
659 if isinstance(ie_entries, list):
660 n_all_entries = len(ie_entries)
661 entries = ie_entries[playliststart:playlistend]
662 n_entries = len(entries)
664 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
665 (ie_result['extractor'], playlist, n_all_entries, n_entries))
666 elif isinstance(ie_entries, PagedList):
667 entries = ie_entries.getslice(
668 playliststart, playlistend)
669 n_entries = len(entries)
671 "[%s] playlist %s: Downloading %d videos" %
672 (ie_result['extractor'], playlist, n_entries))
674 entries = list(itertools.islice(
675 ie_entries, playliststart, playlistend))
676 n_entries = len(entries)
678 "[%s] playlist %s: Downloading %d videos" %
679 (ie_result['extractor'], playlist, n_entries))
681 for i, entry in enumerate(entries, 1):
682 self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries))
684 'n_entries': n_entries,
685 'playlist': playlist,
686 'playlist_id': ie_result.get('id'),
687 'playlist_title': ie_result.get('title'),
688 'playlist_index': i + playliststart,
689 'extractor': ie_result['extractor'],
690 'webpage_url': ie_result['webpage_url'],
691 'webpage_url_basename': url_basename(ie_result['webpage_url']),
692 'extractor_key': ie_result['extractor_key'],
695 reason = self._match_entry(entry)
696 if reason is not None:
697 self.to_screen('[download] ' + reason)
700 entry_result = self.process_ie_result(entry,
703 playlist_results.append(entry_result)
704 ie_result['entries'] = playlist_results
706 elif result_type == 'compat_list':
708 'Extractor %s returned a compat_list result. '
709 'It needs to be updated.' % ie_result.get('extractor'))
715 'extractor': ie_result['extractor'],
716 'webpage_url': ie_result['webpage_url'],
717 'webpage_url_basename': url_basename(ie_result['webpage_url']),
718 'extractor_key': ie_result['extractor_key'],
722 ie_result['entries'] = [
723 self.process_ie_result(_fixup(r), download, extra_info)
724 for r in ie_result['entries']
728 raise Exception('Invalid result type: %s' % result_type)
730 def select_format(self, format_spec, available_formats):
731 if format_spec == 'best' or format_spec is None:
732 return available_formats[-1]
733 elif format_spec == 'worst':
734 return available_formats[0]
735 elif format_spec == 'bestaudio':
737 f for f in available_formats
738 if f.get('vcodec') == 'none']
740 return audio_formats[-1]
741 elif format_spec == 'worstaudio':
743 f for f in available_formats
744 if f.get('vcodec') == 'none']
746 return audio_formats[0]
747 elif format_spec == 'bestvideo':
749 f for f in available_formats
750 if f.get('acodec') == 'none']
752 return video_formats[-1]
753 elif format_spec == 'worstvideo':
755 f for f in available_formats
756 if f.get('acodec') == 'none']
758 return video_formats[0]
760 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a']
761 if format_spec in extensions:
762 filter_f = lambda f: f['ext'] == format_spec
764 filter_f = lambda f: f['format_id'] == format_spec
765 matches = list(filter(filter_f, available_formats))
770 def process_video_result(self, info_dict, download=True):
771 assert info_dict.get('_type', 'video') == 'video'
773 if 'id' not in info_dict:
774 raise ExtractorError('Missing "id" field in extractor result')
775 if 'title' not in info_dict:
776 raise ExtractorError('Missing "title" field in extractor result')
778 if 'playlist' not in info_dict:
779 # It isn't part of a playlist
780 info_dict['playlist'] = None
781 info_dict['playlist_index'] = None
783 thumbnails = info_dict.get('thumbnails')
785 thumbnails.sort(key=lambda t: (
786 t.get('width'), t.get('height'), t.get('url')))
788 if 'width' in t and 'height' in t:
789 t['resolution'] = '%dx%d' % (t['width'], t['height'])
791 if thumbnails and 'thumbnail' not in info_dict:
792 info_dict['thumbnail'] = thumbnails[-1]['url']
794 if 'display_id' not in info_dict and 'id' in info_dict:
795 info_dict['display_id'] = info_dict['id']
797 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
798 # Working around negative timestamps in Windows
799 # (see http://bugs.python.org/issue1646728)
800 if info_dict['timestamp'] < 0 and os.name == 'nt':
801 info_dict['timestamp'] = 0
802 upload_date = datetime.datetime.utcfromtimestamp(
803 info_dict['timestamp'])
804 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
806 # This extractors handle format selection themselves
807 if info_dict['extractor'] in ['Youku']:
809 self.process_info(info_dict)
812 # We now pick which formats have to be downloaded
813 if info_dict.get('formats') is None:
814 # There's only one format available
815 formats = [info_dict]
817 formats = info_dict['formats']
820 raise ExtractorError('No video formats found!')
822 # We check that all the formats have the format and format_id fields
823 for i, format in enumerate(formats):
824 if 'url' not in format:
825 raise ExtractorError('Missing "url" key in result (index %d)' % i)
827 if format.get('format_id') is None:
828 format['format_id'] = compat_str(i)
829 if format.get('format') is None:
830 format['format'] = '{id} - {res}{note}'.format(
831 id=format['format_id'],
832 res=self.format_resolution(format),
833 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
835 # Automatically determine file extension if missing
836 if 'ext' not in format:
837 format['ext'] = determine_ext(format['url']).lower()
839 format_limit = self.params.get('format_limit', None)
841 formats = list(takewhile_inclusive(
842 lambda f: f['format_id'] != format_limit, formats
845 # TODO Central sorting goes here
847 if formats[0] is not info_dict:
848 # only set the 'formats' fields if the original info_dict list them
849 # otherwise we end up with a circular reference, the first (and unique)
850 # element in the 'formats' field in info_dict is info_dict itself,
851 # wich can't be exported to json
852 info_dict['formats'] = formats
853 if self.params.get('listformats', None):
854 self.list_formats(info_dict)
857 req_format = self.params.get('format')
858 if req_format is None:
860 formats_to_download = []
861 # The -1 is for supporting YoutubeIE
862 if req_format in ('-1', 'all'):
863 formats_to_download = formats
865 for rfstr in req_format.split(','):
866 # We can accept formats requested in the format: 34/5/best, we pick
867 # the first that is available, starting from left
868 req_formats = rfstr.split('/')
869 for rf in req_formats:
870 if re.match(r'.+?\+.+?', rf) is not None:
871 # Two formats have been requested like '137+139'
872 format_1, format_2 = rf.split('+')
873 formats_info = (self.select_format(format_1, formats),
874 self.select_format(format_2, formats))
875 if all(formats_info):
876 # The first format must contain the video and the
878 if formats_info[0].get('vcodec') == 'none':
879 self.report_error('The first format must '
880 'contain the video, try using '
881 '"-f %s+%s"' % (format_2, format_1))
884 'requested_formats': formats_info,
886 'ext': formats_info[0]['ext'],
889 selected_format = None
891 selected_format = self.select_format(rf, formats)
892 if selected_format is not None:
893 formats_to_download.append(selected_format)
895 if not formats_to_download:
896 raise ExtractorError('requested format not available',
900 if len(formats_to_download) > 1:
901 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
902 for format in formats_to_download:
903 new_info = dict(info_dict)
904 new_info.update(format)
905 self.process_info(new_info)
906 # We update the info dict with the best quality format (backwards compatibility)
907 info_dict.update(formats_to_download[-1])
910 def process_info(self, info_dict):
911 """Process a single resolved IE result."""
913 assert info_dict.get('_type', 'video') == 'video'
915 max_downloads = self.params.get('max_downloads')
916 if max_downloads is not None:
917 if self._num_downloads >= int(max_downloads):
918 raise MaxDownloadsReached()
920 info_dict['fulltitle'] = info_dict['title']
921 if len(info_dict['title']) > 200:
922 info_dict['title'] = info_dict['title'][:197] + '...'
924 # Keep for backwards compatibility
925 info_dict['stitle'] = info_dict['title']
927 if 'format' not in info_dict:
928 info_dict['format'] = info_dict['ext']
930 reason = self._match_entry(info_dict)
931 if reason is not None:
932 self.to_screen('[download] ' + reason)
935 self._num_downloads += 1
937 filename = self.prepare_filename(info_dict)
940 if self.params.get('forcetitle', False):
941 self.to_stdout(info_dict['fulltitle'])
942 if self.params.get('forceid', False):
943 self.to_stdout(info_dict['id'])
944 if self.params.get('forceurl', False):
945 if info_dict.get('requested_formats') is not None:
946 for f in info_dict['requested_formats']:
947 self.to_stdout(f['url'] + f.get('play_path', ''))
949 # For RTMP URLs, also include the playpath
950 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
951 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
952 self.to_stdout(info_dict['thumbnail'])
953 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
954 self.to_stdout(info_dict['description'])
955 if self.params.get('forcefilename', False) and filename is not None:
956 self.to_stdout(filename)
957 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
958 self.to_stdout(formatSeconds(info_dict['duration']))
959 if self.params.get('forceformat', False):
960 self.to_stdout(info_dict['format'])
961 if self.params.get('forcejson', False):
962 info_dict['_filename'] = filename
963 self.to_stdout(json.dumps(info_dict))
964 if self.params.get('dump_single_json', False):
965 info_dict['_filename'] = filename
967 # Do nothing else if in simulate mode
968 if self.params.get('simulate', False):
975 dn = os.path.dirname(encodeFilename(filename))
976 if dn and not os.path.exists(dn):
978 except (OSError, IOError) as err:
979 self.report_error('unable to create directory ' + compat_str(err))
982 if self.params.get('writedescription', False):
983 descfn = filename + '.description'
984 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
985 self.to_screen('[info] Video description is already present')
988 self.to_screen('[info] Writing video description to: ' + descfn)
989 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
990 descfile.write(info_dict['description'])
991 except (KeyError, TypeError):
992 self.report_warning('There\'s no description to write.')
993 except (OSError, IOError):
994 self.report_error('Cannot write description file ' + descfn)
997 if self.params.get('writeannotations', False):
998 annofn = filename + '.annotations.xml'
999 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1000 self.to_screen('[info] Video annotations are already present')
1003 self.to_screen('[info] Writing video annotations to: ' + annofn)
1004 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1005 annofile.write(info_dict['annotations'])
1006 except (KeyError, TypeError):
1007 self.report_warning('There are no annotations to write.')
1008 except (OSError, IOError):
1009 self.report_error('Cannot write annotations file: ' + annofn)
1012 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1013 self.params.get('writeautomaticsub')])
1015 if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
1016 # subtitles download errors are already managed as troubles in relevant IE
1017 # that way it will silently go on when used with unsupporting IE
1018 subtitles = info_dict['subtitles']
1019 sub_format = self.params.get('subtitlesformat', 'srt')
1020 for sub_lang in subtitles.keys():
1021 sub = subtitles[sub_lang]
1025 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1026 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1027 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1029 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1030 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1032 except (OSError, IOError):
1033 self.report_error('Cannot write subtitles file ' + sub_filename)
1036 if self.params.get('writeinfojson', False):
1037 infofn = os.path.splitext(filename)[0] + '.info.json'
1038 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1039 self.to_screen('[info] Video description metadata is already present')
1041 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1043 write_json_file(info_dict, infofn)
1044 except (OSError, IOError):
1045 self.report_error('Cannot write metadata to JSON file ' + infofn)
1048 if self.params.get('writethumbnail', False):
1049 if info_dict.get('thumbnail') is not None:
1050 thumb_format = determine_ext(info_dict['thumbnail'], 'jpg')
1051 thumb_filename = os.path.splitext(filename)[0] + '.' + thumb_format
1052 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
1053 self.to_screen('[%s] %s: Thumbnail is already present' %
1054 (info_dict['extractor'], info_dict['id']))
1056 self.to_screen('[%s] %s: Downloading thumbnail ...' %
1057 (info_dict['extractor'], info_dict['id']))
1059 uf = self.urlopen(info_dict['thumbnail'])
1060 with open(thumb_filename, 'wb') as thumbf:
1061 shutil.copyfileobj(uf, thumbf)
1062 self.to_screen('[%s] %s: Writing thumbnail to: %s' %
1063 (info_dict['extractor'], info_dict['id'], thumb_filename))
1064 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1065 self.report_warning('Unable to download thumbnail "%s": %s' %
1066 (info_dict['thumbnail'], compat_str(err)))
1068 if not self.params.get('skip_download', False):
1069 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
1074 fd = get_suitable_downloader(info)(self, self.params)
1075 for ph in self._progress_hooks:
1076 fd.add_progress_hook(ph)
1077 if self.params.get('verbose'):
1078 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1079 return fd.download(name, info)
1080 if info_dict.get('requested_formats') is not None:
1083 merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
1084 if not merger._executable:
1086 self.report_warning('You have requested multiple '
1087 'formats but ffmpeg or avconv are not installed.'
1088 ' The formats won\'t be merged')
1090 postprocessors = [merger]
1091 for f in info_dict['requested_formats']:
1092 new_info = dict(info_dict)
1094 fname = self.prepare_filename(new_info)
1095 fname = prepend_extension(fname, 'f%s' % f['format_id'])
1096 downloaded.append(fname)
1097 partial_success = dl(fname, new_info)
1098 success = success and partial_success
1099 info_dict['__postprocessors'] = postprocessors
1100 info_dict['__files_to_merge'] = downloaded
1102 # Just a single file
1103 success = dl(filename, info_dict)
1104 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1105 self.report_error('unable to download video data: %s' % str(err))
1107 except (OSError, IOError) as err:
1108 raise UnavailableVideoError(err)
1109 except (ContentTooShortError, ) as err:
1110 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1115 self.post_process(filename, info_dict)
1116 except (PostProcessingError) as err:
1117 self.report_error('postprocessing: %s' % str(err))
1120 self.record_download_archive(info_dict)
1122 def download(self, url_list):
1123 """Download a given list of URLs."""
1124 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1125 if (len(url_list) > 1 and
1127 and self.params.get('max_downloads') != 1):
1128 raise SameFileError(outtmpl)
1130 for url in url_list:
1132 # It also downloads the videos
1133 res = self.extract_info(url)
1134 except UnavailableVideoError:
1135 self.report_error('unable to download video')
1136 except MaxDownloadsReached:
1137 self.to_screen('[info] Maximum number of downloaded files reached.')
1140 if self.params.get('dump_single_json', False):
1141 self.to_stdout(json.dumps(res))
1143 return self._download_retcode
1145 def download_with_info_file(self, info_filename):
1146 with io.open(info_filename, 'r', encoding='utf-8') as f:
1149 self.process_ie_result(info, download=True)
1150 except DownloadError:
1151 webpage_url = info.get('webpage_url')
1152 if webpage_url is not None:
1153 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1154 return self.download([webpage_url])
1157 return self._download_retcode
1159 def post_process(self, filename, ie_info):
1160 """Run all the postprocessors on the given file."""
1161 info = dict(ie_info)
1162 info['filepath'] = filename
1165 if ie_info.get('__postprocessors') is not None:
1166 pps_chain.extend(ie_info['__postprocessors'])
1167 pps_chain.extend(self._pps)
1168 for pp in pps_chain:
1170 keep_video_wish, new_info = pp.run(info)
1171 if keep_video_wish is not None:
1173 keep_video = keep_video_wish
1174 elif keep_video is None:
1175 # No clear decision yet, let IE decide
1176 keep_video = keep_video_wish
1177 except PostProcessingError as e:
1178 self.report_error(e.msg)
1179 if keep_video is False and not self.params.get('keepvideo', False):
1181 self.to_screen('Deleting original file %s (pass -k to keep)' % filename)
1182 os.remove(encodeFilename(filename))
1183 except (IOError, OSError):
1184 self.report_warning('Unable to remove downloaded video file')
1186 def _make_archive_id(self, info_dict):
1187 # Future-proof against any change in case
1188 # and backwards compatibility with prior versions
1189 extractor = info_dict.get('extractor_key')
1190 if extractor is None:
1191 if 'id' in info_dict:
1192 extractor = info_dict.get('ie_key') # key in a playlist
1193 if extractor is None:
1194 return None # Incomplete video information
1195 return extractor.lower() + ' ' + info_dict['id']
1197 def in_download_archive(self, info_dict):
1198 fn = self.params.get('download_archive')
1202 vid_id = self._make_archive_id(info_dict)
1204 return False # Incomplete video information
1207 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1208 for line in archive_file:
1209 if line.strip() == vid_id:
1211 except IOError as ioe:
1212 if ioe.errno != errno.ENOENT:
1216 def record_download_archive(self, info_dict):
1217 fn = self.params.get('download_archive')
1220 vid_id = self._make_archive_id(info_dict)
1222 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1223 archive_file.write(vid_id + '\n')
1226 def format_resolution(format, default='unknown'):
1227 if format.get('vcodec') == 'none':
1229 if format.get('resolution') is not None:
1230 return format['resolution']
1231 if format.get('height') is not None:
1232 if format.get('width') is not None:
1233 res = '%sx%s' % (format['width'], format['height'])
1235 res = '%sp' % format['height']
1236 elif format.get('width') is not None:
1237 res = '?x%d' % format['width']
1242 def _format_note(self, fdict):
1244 if fdict.get('ext') in ['f4f', 'f4m']:
1245 res += '(unsupported) '
1246 if fdict.get('format_note') is not None:
1247 res += fdict['format_note'] + ' '
1248 if fdict.get('tbr') is not None:
1249 res += '%4dk ' % fdict['tbr']
1250 if fdict.get('container') is not None:
1253 res += '%s container' % fdict['container']
1254 if (fdict.get('vcodec') is not None and
1255 fdict.get('vcodec') != 'none'):
1258 res += fdict['vcodec']
1259 if fdict.get('vbr') is not None:
1261 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1263 if fdict.get('vbr') is not None:
1264 res += '%4dk' % fdict['vbr']
1265 if fdict.get('fps') is not None:
1266 res += ', %sfps' % fdict['fps']
1267 if fdict.get('acodec') is not None:
1270 if fdict['acodec'] == 'none':
1273 res += '%-5s' % fdict['acodec']
1274 elif fdict.get('abr') is not None:
1278 if fdict.get('abr') is not None:
1279 res += '@%3dk' % fdict['abr']
1280 if fdict.get('asr') is not None:
1281 res += ' (%5dHz)' % fdict['asr']
1282 if fdict.get('filesize') is not None:
1285 res += format_bytes(fdict['filesize'])
1286 elif fdict.get('filesize_approx') is not None:
1289 res += '~' + format_bytes(fdict['filesize_approx'])
1292 def list_formats(self, info_dict):
1293 def line(format, idlen=20):
1294 return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
1295 format['format_id'],
1297 self.format_resolution(format),
1298 self._format_note(format),
1301 formats = info_dict.get('formats', [info_dict])
1302 idlen = max(len('format code'),
1303 max(len(f['format_id']) for f in formats))
1304 formats_s = [line(f, idlen) for f in formats]
1305 if len(formats) > 1:
1306 formats_s[0] += (' ' if self._format_note(formats[0]) else '') + '(worst)'
1307 formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
1309 header_line = line({
1310 'format_id': 'format code', 'ext': 'extension',
1311 'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
1312 self.to_screen('[info] Available formats for %s:\n%s\n%s' %
1313 (info_dict['id'], header_line, '\n'.join(formats_s)))
1315 def urlopen(self, req):
1316 """ Start an HTTP download """
1318 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1319 # always respected by websites, some tend to give out URLs with non percent-encoded
1320 # non-ASCII characters (see telemb.py, ard.py [#3412])
1321 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1322 # To work around aforementioned issue we will replace request's original URL with
1323 # percent-encoded one
1324 req_is_string = isinstance(req, basestring if sys.version_info < (3, 0) else compat_str)
1325 url = req if req_is_string else req.get_full_url()
1326 url_escaped = escape_url(url)
1328 # Substitute URL if any change after escaping
1329 if url != url_escaped:
1333 req = compat_urllib_request.Request(
1334 url_escaped, data=req.data, headers=req.headers,
1335 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1337 return self._opener.open(req, timeout=self._socket_timeout)
1339 def print_debug_header(self):
1340 if not self.params.get('verbose'):
1343 if type('') is not compat_str:
1344 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1345 self.report_warning(
1346 'Your Python is broken! Update to a newer and supported version')
1348 stdout_encoding = getattr(
1349 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1351 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1352 locale.getpreferredencoding(),
1353 sys.getfilesystemencoding(),
1355 self.get_encoding()))
1356 write_string(encoding_str, encoding=None)
1358 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1360 sp = subprocess.Popen(
1361 ['git', 'rev-parse', '--short', 'HEAD'],
1362 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1363 cwd=os.path.dirname(os.path.abspath(__file__)))
1364 out, err = sp.communicate()
1365 out = out.decode().strip()
1366 if re.match('[0-9a-f]+', out):
1367 self._write_string('[debug] Git HEAD: ' + out + '\n')
1373 self._write_string('[debug] Python version %s - %s\n' % (
1374 platform.python_version(), platform_name()))
1376 exe_versions = FFmpegPostProcessor.get_versions()
1377 exe_versions['rtmpdump'] = rtmpdump_version()
1378 exe_str = ', '.join(
1380 for exe, v in sorted(exe_versions.items())
1385 self._write_string('[debug] exe versions: %s\n' % exe_str)
1388 for handler in self._opener.handlers:
1389 if hasattr(handler, 'proxies'):
1390 proxy_map.update(handler.proxies)
1391 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1393 def _setup_opener(self):
1394 timeout_val = self.params.get('socket_timeout')
1395 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1397 opts_cookiefile = self.params.get('cookiefile')
1398 opts_proxy = self.params.get('proxy')
1400 if opts_cookiefile is None:
1401 self.cookiejar = compat_cookiejar.CookieJar()
1403 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1405 if os.access(opts_cookiefile, os.R_OK):
1406 self.cookiejar.load()
1408 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1410 if opts_proxy is not None:
1411 if opts_proxy == '':
1414 proxies = {'http': opts_proxy, 'https': opts_proxy}
1416 proxies = compat_urllib_request.getproxies()
1417 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1418 if 'http' in proxies and 'https' not in proxies:
1419 proxies['https'] = proxies['http']
1420 proxy_handler = compat_urllib_request.ProxyHandler(proxies)
1422 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1423 https_handler = make_HTTPS_handler(
1424 self.params.get('nocheckcertificate', False), debuglevel=debuglevel)
1425 ydlh = YoutubeDLHandler(debuglevel=debuglevel)
1426 opener = compat_urllib_request.build_opener(
1427 https_handler, proxy_handler, cookie_processor, ydlh)
1428 # Delete the default user-agent header, which would otherwise apply in
1429 # cases where our custom HTTP handler doesn't come into play
1430 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1431 opener.addheaders = []
1432 self._opener = opener
1434 def encode(self, s):
1435 if isinstance(s, bytes):
1436 return s # Already encoded
1439 return s.encode(self.get_encoding())
1440 except UnicodeEncodeError as err:
1441 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1444 def get_encoding(self):
1445 encoding = self.params.get('encoding')
1446 if encoding is None:
1447 encoding = preferredencoding()