2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import, unicode_literals
33 compat_urllib_request,
59 UnavailableVideoError,
68 from .cache import Cache
69 from .extractor import get_info_extractor, gen_extractors
70 from .downloader import get_suitable_downloader
71 from .downloader.rtmp import rtmpdump_version
72 from .postprocessor import (
73 FFmpegFixupStretchedPP,
78 from .version import __version__
81 class YoutubeDL(object):
84 YoutubeDL objects are the ones responsible of downloading the
85 actual video file and writing it to disk if the user has requested
86 it, among some other tasks. In most cases there should be one per
87 program. As, given a video URL, the downloader doesn't know how to
88 extract all the needed information, task that InfoExtractors do, it
89 has to pass the URL to one of them.
91 For this, YoutubeDL objects have a method that allows
92 InfoExtractors to be registered in a given order. When it is passed
93 a URL, the YoutubeDL object handles it to the first InfoExtractor it
94 finds that reports being able to handle it. The InfoExtractor extracts
95 all the information about the video or videos the URL refers to, and
96 YoutubeDL process the extracted information, possibly using a File
97 Downloader to download the video.
99 YoutubeDL objects accept a lot of parameters. In order not to saturate
100 the object constructor with arguments, it receives a dictionary of
101 options instead. These options are available through the params
102 attribute for the InfoExtractors to use. The YoutubeDL also
103 registers itself as the downloader in charge for the InfoExtractors
104 that are added to it, so this is a "mutual registration".
108 username: Username for authentication purposes.
109 password: Password for authentication purposes.
110 videopassword: Password for acces a video.
111 usenetrc: Use netrc for authentication instead.
112 verbose: Print additional info to stdout.
113 quiet: Do not print messages to stdout.
114 no_warnings: Do not print out anything for warnings.
115 forceurl: Force printing final URL.
116 forcetitle: Force printing title.
117 forceid: Force printing ID.
118 forcethumbnail: Force printing thumbnail URL.
119 forcedescription: Force printing description.
120 forcefilename: Force printing final filename.
121 forceduration: Force printing duration.
122 forcejson: Force printing info_dict as JSON.
123 dump_single_json: Force printing the info_dict of the whole playlist
124 (or video) as a single JSON line.
125 simulate: Do not download the video files.
126 format: Video format code. See options.py for more information.
127 format_limit: Highest quality format to try.
128 outtmpl: Template for output names.
129 restrictfilenames: Do not allow "&" and spaces in file names
130 ignoreerrors: Do not stop on download errors.
131 nooverwrites: Prevent overwriting files.
132 playliststart: Playlist item to start at.
133 playlistend: Playlist item to end at.
134 playlistreverse: Download playlist items in reverse order.
135 matchtitle: Download only matching titles.
136 rejecttitle: Reject downloads for matching titles.
137 logger: Log messages to a logging.Logger instance.
138 logtostderr: Log messages to stderr instead of stdout.
139 writedescription: Write the video description to a .description file
140 writeinfojson: Write the video description to a .info.json file
141 writeannotations: Write the video annotations to a .annotations.xml file
142 writethumbnail: Write the thumbnail image to a file
143 writesubtitles: Write the video subtitles to a file
144 writeautomaticsub: Write the automatic subtitles to a file
145 allsubtitles: Downloads all the subtitles of the video
146 (requires writesubtitles or writeautomaticsub)
147 listsubtitles: Lists all available subtitles for the video
148 subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
149 subtitleslangs: List of languages of the subtitles to download
150 keepvideo: Keep the video file after post-processing
151 daterange: A DateRange object, download only if the upload_date is in the range.
152 skip_download: Skip the actual download of the video file
153 cachedir: Location of the cache files in the filesystem.
154 False to disable filesystem cache.
155 noplaylist: Download single video instead of a playlist if in doubt.
156 age_limit: An integer representing the user's age in years.
157 Unsuitable videos for the given age are skipped.
158 min_views: An integer representing the minimum view count the video
159 must have in order to not be skipped.
160 Videos without view count information are always
161 downloaded. None for no limit.
162 max_views: An integer representing the maximum view count.
163 Videos that are more popular than that are not
165 Videos without view count information are always
166 downloaded. None for no limit.
167 download_archive: File name of a file where all downloads are recorded.
168 Videos already present in the file are not downloaded
170 cookiefile: File name where cookies should be read from and dumped to.
171 nocheckcertificate:Do not verify SSL certificates
172 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
173 At the moment, this is only supported by YouTube.
174 proxy: URL of the proxy server to use
175 socket_timeout: Time to wait for unresponsive hosts, in seconds
176 bidi_workaround: Work around buggy terminals without bidirectional text
177 support, using fridibi
178 debug_printtraffic:Print out sent and received HTTP traffic
179 include_ads: Download ads as well
180 default_search: Prepend this string if an input url is not valid.
181 'auto' for elaborate guessing
182 encoding: Use this encoding instead of the system-specified.
183 extract_flat: Do not resolve URLs, return the immediate result.
184 Pass in 'in_playlist' to only show this behavior for
186 postprocessors: A list of dictionaries, each with an entry
187 * key: The name of the postprocessor. See
188 youtube_dl/postprocessor/__init__.py for a list.
189 as well as any further keyword arguments for the
191 progress_hooks: A list of functions that get called on download
192 progress, with a dictionary with the entries
193 * filename: The final filename
194 * status: One of "downloading" and "finished"
196 The dict may also have some of the following entries:
198 * downloaded_bytes: Bytes on disk
199 * total_bytes: Size of the whole file, None if unknown
200 * tmpfilename: The filename we're currently writing to
201 * eta: The estimated time in seconds, None if unknown
202 * speed: The download speed in bytes/second, None if
205 Progress hooks are guaranteed to be called at least once
206 (with status "finished") if the download is successful.
207 merge_output_format: Extension to use when merging formats.
208 fixup: Automatically correct known faults of the file.
210 - "never": do nothing
211 - "warn": only emit a warning
212 - "detect_or_warn": check whether we can do anything
213 about it, warn otherwise
216 The following parameters are not used by YoutubeDL itself, they are used by
218 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
219 noresizebuffer, retries, continuedl, noprogress, consoletitle
221 The following options are used by the post processors:
222 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
223 otherwise prefer avconv.
224 exec_cmd: Arbitrary command to run after downloading
230 _download_retcode = None
231 _num_downloads = None
234 def __init__(self, params=None, auto_init=True):
235 """Create a FileDownloader object with the given options."""
239 self._ies_instances = {}
241 self._progress_hooks = []
242 self._download_retcode = 0
243 self._num_downloads = 0
244 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
245 self._err_file = sys.stderr
247 self.cache = Cache(self)
249 if params.get('bidi_workaround', False):
252 master, slave = pty.openpty()
253 width = get_term_width()
257 width_args = ['-w', str(width)]
259 stdin=subprocess.PIPE,
261 stderr=self._err_file)
263 self._output_process = subprocess.Popen(
264 ['bidiv'] + width_args, **sp_kwargs
267 self._output_process = subprocess.Popen(
268 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
269 self._output_channel = os.fdopen(master, 'rb')
270 except OSError as ose:
272 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
276 if (sys.version_info >= (3,) and sys.platform != 'win32' and
277 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
278 and not params.get('restrictfilenames', False)):
279 # On Python 3, the Unicode filesystem API will throw errors (#1474)
281 'Assuming --restrict-filenames since file system encoding '
282 'cannot encode all characters. '
283 'Set the LC_ALL environment variable to fix this.')
284 self.params['restrictfilenames'] = True
286 if '%(stitle)s' in self.params.get('outtmpl', ''):
287 self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
292 self.print_debug_header()
293 self.add_default_info_extractors()
295 for pp_def_raw in self.params.get('postprocessors', []):
296 pp_class = get_postprocessor(pp_def_raw['key'])
297 pp_def = dict(pp_def_raw)
299 pp = pp_class(self, **compat_kwargs(pp_def))
300 self.add_post_processor(pp)
302 for ph in self.params.get('progress_hooks', []):
303 self.add_progress_hook(ph)
305 def warn_if_short_id(self, argv):
306 # short YouTube ID starting with dash?
308 i for i, a in enumerate(argv)
309 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
313 [a for i, a in enumerate(argv) if i not in idxs] +
314 ['--'] + [argv[i] for i in idxs]
317 'Long argument string detected. '
318 'Use -- to separate parameters and URLs, like this:\n%s\n' %
319 args_to_str(correct_argv))
321 def add_info_extractor(self, ie):
322 """Add an InfoExtractor object to the end of the list."""
324 self._ies_instances[ie.ie_key()] = ie
325 ie.set_downloader(self)
327 def get_info_extractor(self, ie_key):
329 Get an instance of an IE with name ie_key, it will try to get one from
330 the _ies list, if there's no instance it will create a new one and add
331 it to the extractor list.
333 ie = self._ies_instances.get(ie_key)
335 ie = get_info_extractor(ie_key)()
336 self.add_info_extractor(ie)
339 def add_default_info_extractors(self):
341 Add the InfoExtractors returned by gen_extractors to the end of the list
343 for ie in gen_extractors():
344 self.add_info_extractor(ie)
346 def add_post_processor(self, pp):
347 """Add a PostProcessor object to the end of the chain."""
349 pp.set_downloader(self)
351 def add_progress_hook(self, ph):
352 """Add the progress hook (currently only for the file downloader)"""
353 self._progress_hooks.append(ph)
355 def _bidi_workaround(self, message):
356 if not hasattr(self, '_output_channel'):
359 assert hasattr(self, '_output_process')
360 assert isinstance(message, compat_str)
361 line_count = message.count('\n') + 1
362 self._output_process.stdin.write((message + '\n').encode('utf-8'))
363 self._output_process.stdin.flush()
364 res = ''.join(self._output_channel.readline().decode('utf-8')
365 for _ in range(line_count))
366 return res[:-len('\n')]
368 def to_screen(self, message, skip_eol=False):
369 """Print message to stdout if not in quiet mode."""
370 return self.to_stdout(message, skip_eol, check_quiet=True)
372 def _write_string(self, s, out=None):
373 write_string(s, out=out, encoding=self.params.get('encoding'))
375 def to_stdout(self, message, skip_eol=False, check_quiet=False):
376 """Print message to stdout if not in quiet mode."""
377 if self.params.get('logger'):
378 self.params['logger'].debug(message)
379 elif not check_quiet or not self.params.get('quiet', False):
380 message = self._bidi_workaround(message)
381 terminator = ['\n', ''][skip_eol]
382 output = message + terminator
384 self._write_string(output, self._screen_file)
386 def to_stderr(self, message):
387 """Print message to stderr."""
388 assert isinstance(message, compat_str)
389 if self.params.get('logger'):
390 self.params['logger'].error(message)
392 message = self._bidi_workaround(message)
393 output = message + '\n'
394 self._write_string(output, self._err_file)
396 def to_console_title(self, message):
397 if not self.params.get('consoletitle', False):
399 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
400 # c_wchar_p() might not be necessary if `message` is
401 # already of type unicode()
402 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
403 elif 'TERM' in os.environ:
404 self._write_string('\033]0;%s\007' % message, self._screen_file)
406 def save_console_title(self):
407 if not self.params.get('consoletitle', False):
409 if 'TERM' in os.environ:
410 # Save the title on stack
411 self._write_string('\033[22;0t', self._screen_file)
413 def restore_console_title(self):
414 if not self.params.get('consoletitle', False):
416 if 'TERM' in os.environ:
417 # Restore the title from stack
418 self._write_string('\033[23;0t', self._screen_file)
421 self.save_console_title()
424 def __exit__(self, *args):
425 self.restore_console_title()
427 if self.params.get('cookiefile') is not None:
428 self.cookiejar.save()
430 def trouble(self, message=None, tb=None):
431 """Determine action to take when a download problem appears.
433 Depending on if the downloader has been configured to ignore
434 download errors or not, this method may throw an exception or
435 not when errors are found, after printing the message.
437 tb, if given, is additional traceback information.
439 if message is not None:
440 self.to_stderr(message)
441 if self.params.get('verbose'):
443 if sys.exc_info()[0]: # if .trouble has been called from an except block
445 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
446 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
447 tb += compat_str(traceback.format_exc())
449 tb_data = traceback.format_list(traceback.extract_stack())
450 tb = ''.join(tb_data)
452 if not self.params.get('ignoreerrors', False):
453 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
454 exc_info = sys.exc_info()[1].exc_info
456 exc_info = sys.exc_info()
457 raise DownloadError(message, exc_info)
458 self._download_retcode = 1
460 def report_warning(self, message):
462 Print the message to stderr, it will be prefixed with 'WARNING:'
463 If stderr is a tty file the 'WARNING:' will be colored
465 if self.params.get('logger') is not None:
466 self.params['logger'].warning(message)
468 if self.params.get('no_warnings'):
470 if self._err_file.isatty() and os.name != 'nt':
471 _msg_header = '\033[0;33mWARNING:\033[0m'
473 _msg_header = 'WARNING:'
474 warning_message = '%s %s' % (_msg_header, message)
475 self.to_stderr(warning_message)
477 def report_error(self, message, tb=None):
479 Do the same as trouble, but prefixes the message with 'ERROR:', colored
480 in red if stderr is a tty file.
482 if self._err_file.isatty() and os.name != 'nt':
483 _msg_header = '\033[0;31mERROR:\033[0m'
485 _msg_header = 'ERROR:'
486 error_message = '%s %s' % (_msg_header, message)
487 self.trouble(error_message, tb)
489 def report_file_already_downloaded(self, file_name):
490 """Report file has already been fully downloaded."""
492 self.to_screen('[download] %s has already been downloaded' % file_name)
493 except UnicodeEncodeError:
494 self.to_screen('[download] The file has already been downloaded')
496 def prepare_filename(self, info_dict):
497 """Generate the output filename."""
499 template_dict = dict(info_dict)
501 template_dict['epoch'] = int(time.time())
502 autonumber_size = self.params.get('autonumber_size')
503 if autonumber_size is None:
505 autonumber_templ = '%0' + str(autonumber_size) + 'd'
506 template_dict['autonumber'] = autonumber_templ % self._num_downloads
507 if template_dict.get('playlist_index') is not None:
508 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
509 if template_dict.get('resolution') is None:
510 if template_dict.get('width') and template_dict.get('height'):
511 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
512 elif template_dict.get('height'):
513 template_dict['resolution'] = '%sp' % template_dict['height']
514 elif template_dict.get('width'):
515 template_dict['resolution'] = '?x%d' % template_dict['width']
517 sanitize = lambda k, v: sanitize_filename(
519 restricted=self.params.get('restrictfilenames'),
521 template_dict = dict((k, sanitize(k, v))
522 for k, v in template_dict.items()
524 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
526 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
527 tmpl = compat_expanduser(outtmpl)
528 filename = tmpl % template_dict
530 except ValueError as err:
531 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
534 def _match_entry(self, info_dict):
535 """ Returns None iff the file should be downloaded """
537 video_title = info_dict.get('title', info_dict.get('id', 'video'))
538 if 'title' in info_dict:
539 # This can happen when we're just evaluating the playlist
540 title = info_dict['title']
541 matchtitle = self.params.get('matchtitle', False)
543 if not re.search(matchtitle, title, re.IGNORECASE):
544 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
545 rejecttitle = self.params.get('rejecttitle', False)
547 if re.search(rejecttitle, title, re.IGNORECASE):
548 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
549 date = info_dict.get('upload_date', None)
551 dateRange = self.params.get('daterange', DateRange())
552 if date not in dateRange:
553 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
554 view_count = info_dict.get('view_count', None)
555 if view_count is not None:
556 min_views = self.params.get('min_views')
557 if min_views is not None and view_count < min_views:
558 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
559 max_views = self.params.get('max_views')
560 if max_views is not None and view_count > max_views:
561 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
562 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
563 return 'Skipping "%s" because it is age restricted' % title
564 if self.in_download_archive(info_dict):
565 return '%s has already been recorded in archive' % video_title
569 def add_extra_info(info_dict, extra_info):
570 '''Set the keys from extra_info in info dict if they are missing'''
571 for key, value in extra_info.items():
572 info_dict.setdefault(key, value)
574 def extract_info(self, url, download=True, ie_key=None, extra_info={},
577 Returns a list with a dictionary for each video we find.
578 If 'download', also downloads the videos.
579 extra_info is a dict containing the extra values to add to each result
583 ies = [self.get_info_extractor(ie_key)]
588 if not ie.suitable(url):
592 self.report_warning('The program functionality for this site has been marked as broken, '
593 'and will probably not work.')
596 ie_result = ie.extract(url)
597 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
599 if isinstance(ie_result, list):
600 # Backwards compatibility: old IE result format
602 '_type': 'compat_list',
603 'entries': ie_result,
605 self.add_default_extra_info(ie_result, ie, url)
607 return self.process_ie_result(ie_result, download, extra_info)
610 except ExtractorError as de: # An error we somewhat expected
611 self.report_error(compat_str(de), de.format_traceback())
613 except MaxDownloadsReached:
615 except Exception as e:
616 if self.params.get('ignoreerrors', False):
617 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
622 self.report_error('no suitable InfoExtractor for URL %s' % url)
624 def add_default_extra_info(self, ie_result, ie, url):
625 self.add_extra_info(ie_result, {
626 'extractor': ie.IE_NAME,
628 'webpage_url_basename': url_basename(url),
629 'extractor_key': ie.ie_key(),
632 def process_ie_result(self, ie_result, download=True, extra_info={}):
634 Take the result of the ie(may be modified) and resolve all unresolved
635 references (URLs, playlist items).
637 It will also download the videos if 'download'.
638 Returns the resolved ie_result.
641 result_type = ie_result.get('_type', 'video')
643 if result_type in ('url', 'url_transparent'):
644 extract_flat = self.params.get('extract_flat', False)
645 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
646 extract_flat is True):
647 if self.params.get('forcejson', False):
648 self.to_stdout(json.dumps(ie_result))
651 if result_type == 'video':
652 self.add_extra_info(ie_result, extra_info)
653 return self.process_video_result(ie_result, download=download)
654 elif result_type == 'url':
655 # We have to add extra_info to the results because it may be
656 # contained in a playlist
657 return self.extract_info(ie_result['url'],
659 ie_key=ie_result.get('ie_key'),
660 extra_info=extra_info)
661 elif result_type == 'url_transparent':
662 # Use the information from the embedding page
663 info = self.extract_info(
664 ie_result['url'], ie_key=ie_result.get('ie_key'),
665 extra_info=extra_info, download=False, process=False)
667 force_properties = dict(
668 (k, v) for k, v in ie_result.items() if v is not None)
669 for f in ('_type', 'url'):
670 if f in force_properties:
671 del force_properties[f]
672 new_result = info.copy()
673 new_result.update(force_properties)
675 assert new_result.get('_type') != 'url_transparent'
677 return self.process_ie_result(
678 new_result, download=download, extra_info=extra_info)
679 elif result_type == 'playlist' or result_type == 'multi_video':
680 # We process each entry in the playlist
681 playlist = ie_result.get('title', None) or ie_result.get('id', None)
682 self.to_screen('[download] Downloading playlist: %s' % playlist)
684 playlist_results = []
686 playliststart = self.params.get('playliststart', 1) - 1
687 playlistend = self.params.get('playlistend', None)
688 # For backwards compatibility, interpret -1 as whole list
689 if playlistend == -1:
692 ie_entries = ie_result['entries']
693 if isinstance(ie_entries, list):
694 n_all_entries = len(ie_entries)
695 entries = ie_entries[playliststart:playlistend]
696 n_entries = len(entries)
698 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
699 (ie_result['extractor'], playlist, n_all_entries, n_entries))
700 elif isinstance(ie_entries, PagedList):
701 entries = ie_entries.getslice(
702 playliststart, playlistend)
703 n_entries = len(entries)
705 "[%s] playlist %s: Downloading %d videos" %
706 (ie_result['extractor'], playlist, n_entries))
708 entries = list(itertools.islice(
709 ie_entries, playliststart, playlistend))
710 n_entries = len(entries)
712 "[%s] playlist %s: Downloading %d videos" %
713 (ie_result['extractor'], playlist, n_entries))
715 if self.params.get('playlistreverse', False):
716 entries = entries[::-1]
718 for i, entry in enumerate(entries, 1):
719 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
721 'n_entries': n_entries,
722 'playlist': playlist,
723 'playlist_id': ie_result.get('id'),
724 'playlist_title': ie_result.get('title'),
725 'playlist_index': i + playliststart,
726 'extractor': ie_result['extractor'],
727 'webpage_url': ie_result['webpage_url'],
728 'webpage_url_basename': url_basename(ie_result['webpage_url']),
729 'extractor_key': ie_result['extractor_key'],
732 reason = self._match_entry(entry)
733 if reason is not None:
734 self.to_screen('[download] ' + reason)
737 entry_result = self.process_ie_result(entry,
740 playlist_results.append(entry_result)
741 ie_result['entries'] = playlist_results
743 elif result_type == 'compat_list':
745 'Extractor %s returned a compat_list result. '
746 'It needs to be updated.' % ie_result.get('extractor'))
752 'extractor': ie_result['extractor'],
753 'webpage_url': ie_result['webpage_url'],
754 'webpage_url_basename': url_basename(ie_result['webpage_url']),
755 'extractor_key': ie_result['extractor_key'],
759 ie_result['entries'] = [
760 self.process_ie_result(_fixup(r), download, extra_info)
761 for r in ie_result['entries']
765 raise Exception('Invalid result type: %s' % result_type)
767 def select_format(self, format_spec, available_formats):
768 if format_spec == 'best' or format_spec is None:
769 return available_formats[-1]
770 elif format_spec == 'worst':
771 return available_formats[0]
772 elif format_spec == 'bestaudio':
774 f for f in available_formats
775 if f.get('vcodec') == 'none']
777 return audio_formats[-1]
778 elif format_spec == 'worstaudio':
780 f for f in available_formats
781 if f.get('vcodec') == 'none']
783 return audio_formats[0]
784 elif format_spec == 'bestvideo':
786 f for f in available_formats
787 if f.get('acodec') == 'none']
789 return video_formats[-1]
790 elif format_spec == 'worstvideo':
792 f for f in available_formats
793 if f.get('acodec') == 'none']
795 return video_formats[0]
797 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
798 if format_spec in extensions:
799 filter_f = lambda f: f['ext'] == format_spec
801 filter_f = lambda f: f['format_id'] == format_spec
802 matches = list(filter(filter_f, available_formats))
807 def process_video_result(self, info_dict, download=True):
808 assert info_dict.get('_type', 'video') == 'video'
810 if 'id' not in info_dict:
811 raise ExtractorError('Missing "id" field in extractor result')
812 if 'title' not in info_dict:
813 raise ExtractorError('Missing "title" field in extractor result')
815 if 'playlist' not in info_dict:
816 # It isn't part of a playlist
817 info_dict['playlist'] = None
818 info_dict['playlist_index'] = None
820 thumbnails = info_dict.get('thumbnails')
822 thumbnails.sort(key=lambda t: (
823 t.get('width'), t.get('height'), t.get('url')))
825 if 'width' in t and 'height' in t:
826 t['resolution'] = '%dx%d' % (t['width'], t['height'])
828 if thumbnails and 'thumbnail' not in info_dict:
829 info_dict['thumbnail'] = thumbnails[-1]['url']
831 if 'display_id' not in info_dict and 'id' in info_dict:
832 info_dict['display_id'] = info_dict['id']
834 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
835 # Working around negative timestamps in Windows
836 # (see http://bugs.python.org/issue1646728)
837 if info_dict['timestamp'] < 0 and os.name == 'nt':
838 info_dict['timestamp'] = 0
839 upload_date = datetime.datetime.utcfromtimestamp(
840 info_dict['timestamp'])
841 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
843 # This extractors handle format selection themselves
844 if info_dict['extractor'] in ['Youku']:
846 self.process_info(info_dict)
849 # We now pick which formats have to be downloaded
850 if info_dict.get('formats') is None:
851 # There's only one format available
852 formats = [info_dict]
854 formats = info_dict['formats']
857 raise ExtractorError('No video formats found!')
859 # We check that all the formats have the format and format_id fields
860 for i, format in enumerate(formats):
861 if 'url' not in format:
862 raise ExtractorError('Missing "url" key in result (index %d)' % i)
864 if format.get('format_id') is None:
865 format['format_id'] = compat_str(i)
866 if format.get('format') is None:
867 format['format'] = '{id} - {res}{note}'.format(
868 id=format['format_id'],
869 res=self.format_resolution(format),
870 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
872 # Automatically determine file extension if missing
873 if 'ext' not in format:
874 format['ext'] = determine_ext(format['url']).lower()
876 format_limit = self.params.get('format_limit', None)
878 formats = list(takewhile_inclusive(
879 lambda f: f['format_id'] != format_limit, formats
882 # TODO Central sorting goes here
884 if formats[0] is not info_dict:
885 # only set the 'formats' fields if the original info_dict list them
886 # otherwise we end up with a circular reference, the first (and unique)
887 # element in the 'formats' field in info_dict is info_dict itself,
888 # wich can't be exported to json
889 info_dict['formats'] = formats
890 if self.params.get('listformats', None):
891 self.list_formats(info_dict)
894 req_format = self.params.get('format')
895 if req_format is None:
897 formats_to_download = []
898 # The -1 is for supporting YoutubeIE
899 if req_format in ('-1', 'all'):
900 formats_to_download = formats
902 for rfstr in req_format.split(','):
903 # We can accept formats requested in the format: 34/5/best, we pick
904 # the first that is available, starting from left
905 req_formats = rfstr.split('/')
906 for rf in req_formats:
907 if re.match(r'.+?\+.+?', rf) is not None:
908 # Two formats have been requested like '137+139'
909 format_1, format_2 = rf.split('+')
910 formats_info = (self.select_format(format_1, formats),
911 self.select_format(format_2, formats))
912 if all(formats_info):
913 # The first format must contain the video and the
915 if formats_info[0].get('vcodec') == 'none':
916 self.report_error('The first format must '
917 'contain the video, try using '
918 '"-f %s+%s"' % (format_2, format_1))
921 formats_info[0]['ext']
922 if self.params.get('merge_output_format') is None
923 else self.params['merge_output_format'])
925 'requested_formats': formats_info,
927 'ext': formats_info[0]['ext'],
928 'width': formats_info[0].get('width'),
929 'height': formats_info[0].get('height'),
930 'resolution': formats_info[0].get('resolution'),
931 'fps': formats_info[0].get('fps'),
932 'vcodec': formats_info[0].get('vcodec'),
933 'vbr': formats_info[0].get('vbr'),
934 'stretched_ratio': formats_info[0].get('stretched_ratio'),
935 'acodec': formats_info[1].get('acodec'),
936 'abr': formats_info[1].get('abr'),
940 selected_format = None
942 selected_format = self.select_format(rf, formats)
943 if selected_format is not None:
944 formats_to_download.append(selected_format)
946 if not formats_to_download:
947 raise ExtractorError('requested format not available',
951 if len(formats_to_download) > 1:
952 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
953 for format in formats_to_download:
954 new_info = dict(info_dict)
955 new_info.update(format)
956 self.process_info(new_info)
957 # We update the info dict with the best quality format (backwards compatibility)
958 info_dict.update(formats_to_download[-1])
961 def process_info(self, info_dict):
962 """Process a single resolved IE result."""
964 assert info_dict.get('_type', 'video') == 'video'
966 max_downloads = self.params.get('max_downloads')
967 if max_downloads is not None:
968 if self._num_downloads >= int(max_downloads):
969 raise MaxDownloadsReached()
971 info_dict['fulltitle'] = info_dict['title']
972 if len(info_dict['title']) > 200:
973 info_dict['title'] = info_dict['title'][:197] + '...'
975 # Keep for backwards compatibility
976 info_dict['stitle'] = info_dict['title']
978 if 'format' not in info_dict:
979 info_dict['format'] = info_dict['ext']
981 reason = self._match_entry(info_dict)
982 if reason is not None:
983 self.to_screen('[download] ' + reason)
986 self._num_downloads += 1
988 filename = self.prepare_filename(info_dict)
991 if self.params.get('forcetitle', False):
992 self.to_stdout(info_dict['fulltitle'])
993 if self.params.get('forceid', False):
994 self.to_stdout(info_dict['id'])
995 if self.params.get('forceurl', False):
996 if info_dict.get('requested_formats') is not None:
997 for f in info_dict['requested_formats']:
998 self.to_stdout(f['url'] + f.get('play_path', ''))
1000 # For RTMP URLs, also include the playpath
1001 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1002 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1003 self.to_stdout(info_dict['thumbnail'])
1004 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1005 self.to_stdout(info_dict['description'])
1006 if self.params.get('forcefilename', False) and filename is not None:
1007 self.to_stdout(filename)
1008 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1009 self.to_stdout(formatSeconds(info_dict['duration']))
1010 if self.params.get('forceformat', False):
1011 self.to_stdout(info_dict['format'])
1012 if self.params.get('forcejson', False):
1013 info_dict['_filename'] = filename
1014 self.to_stdout(json.dumps(info_dict))
1015 if self.params.get('dump_single_json', False):
1016 info_dict['_filename'] = filename
1018 # Do nothing else if in simulate mode
1019 if self.params.get('simulate', False):
1022 if filename is None:
1026 dn = os.path.dirname(encodeFilename(filename))
1027 if dn and not os.path.exists(dn):
1029 except (OSError, IOError) as err:
1030 self.report_error('unable to create directory ' + compat_str(err))
1033 if self.params.get('writedescription', False):
1034 descfn = filename + '.description'
1035 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1036 self.to_screen('[info] Video description is already present')
1037 elif info_dict.get('description') is None:
1038 self.report_warning('There\'s no description to write.')
1041 self.to_screen('[info] Writing video description to: ' + descfn)
1042 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1043 descfile.write(info_dict['description'])
1044 except (OSError, IOError):
1045 self.report_error('Cannot write description file ' + descfn)
1048 if self.params.get('writeannotations', False):
1049 annofn = filename + '.annotations.xml'
1050 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1051 self.to_screen('[info] Video annotations are already present')
1054 self.to_screen('[info] Writing video annotations to: ' + annofn)
1055 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1056 annofile.write(info_dict['annotations'])
1057 except (KeyError, TypeError):
1058 self.report_warning('There are no annotations to write.')
1059 except (OSError, IOError):
1060 self.report_error('Cannot write annotations file: ' + annofn)
1063 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1064 self.params.get('writeautomaticsub')])
1066 if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
1067 # subtitles download errors are already managed as troubles in relevant IE
1068 # that way it will silently go on when used with unsupporting IE
1069 subtitles = info_dict['subtitles']
1070 sub_format = self.params.get('subtitlesformat', 'srt')
1071 for sub_lang in subtitles.keys():
1072 sub = subtitles[sub_lang]
1076 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1077 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1078 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1080 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1081 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1083 except (OSError, IOError):
1084 self.report_error('Cannot write subtitles file ' + sub_filename)
1087 if self.params.get('writeinfojson', False):
1088 infofn = os.path.splitext(filename)[0] + '.info.json'
1089 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1090 self.to_screen('[info] Video description metadata is already present')
1092 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1094 write_json_file(info_dict, infofn)
1095 except (OSError, IOError):
1096 self.report_error('Cannot write metadata to JSON file ' + infofn)
1099 if self.params.get('writethumbnail', False):
1100 if info_dict.get('thumbnail') is not None:
1101 thumb_format = determine_ext(info_dict['thumbnail'], 'jpg')
1102 thumb_filename = os.path.splitext(filename)[0] + '.' + thumb_format
1103 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
1104 self.to_screen('[%s] %s: Thumbnail is already present' %
1105 (info_dict['extractor'], info_dict['id']))
1107 self.to_screen('[%s] %s: Downloading thumbnail ...' %
1108 (info_dict['extractor'], info_dict['id']))
1110 uf = self.urlopen(info_dict['thumbnail'])
1111 with open(thumb_filename, 'wb') as thumbf:
1112 shutil.copyfileobj(uf, thumbf)
1113 self.to_screen('[%s] %s: Writing thumbnail to: %s' %
1114 (info_dict['extractor'], info_dict['id'], thumb_filename))
1115 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1116 self.report_warning('Unable to download thumbnail "%s": %s' %
1117 (info_dict['thumbnail'], compat_str(err)))
1119 if not self.params.get('skip_download', False):
1120 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
1125 fd = get_suitable_downloader(info)(self, self.params)
1126 for ph in self._progress_hooks:
1127 fd.add_progress_hook(ph)
1128 if self.params.get('verbose'):
1129 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1130 return fd.download(name, info)
1131 if info_dict.get('requested_formats') is not None:
1134 merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
1135 if not merger._executable:
1137 self.report_warning('You have requested multiple '
1138 'formats but ffmpeg or avconv are not installed.'
1139 ' The formats won\'t be merged')
1141 postprocessors = [merger]
1142 for f in info_dict['requested_formats']:
1143 new_info = dict(info_dict)
1145 fname = self.prepare_filename(new_info)
1146 fname = prepend_extension(fname, 'f%s' % f['format_id'])
1147 downloaded.append(fname)
1148 partial_success = dl(fname, new_info)
1149 success = success and partial_success
1150 info_dict['__postprocessors'] = postprocessors
1151 info_dict['__files_to_merge'] = downloaded
1153 # Just a single file
1154 success = dl(filename, info_dict)
1155 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1156 self.report_error('unable to download video data: %s' % str(err))
1158 except (OSError, IOError) as err:
1159 raise UnavailableVideoError(err)
1160 except (ContentTooShortError, ) as err:
1161 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1166 stretched_ratio = info_dict.get('stretched_ratio')
1167 if stretched_ratio is not None and stretched_ratio != 1:
1168 fixup_policy = self.params.get('fixup')
1169 if fixup_policy is None:
1170 fixup_policy = 'detect_or_warn'
1171 if fixup_policy == 'warn':
1172 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1173 info_dict['id'], stretched_ratio))
1174 elif fixup_policy == 'detect_or_warn':
1175 stretched_pp = FFmpegFixupStretchedPP(self)
1176 if stretched_pp.available:
1177 info_dict.setdefault('__postprocessors', [])
1178 info_dict['__postprocessors'].append(stretched_pp)
1180 self.report_warning(
1181 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1182 info_dict['id'], stretched_ratio))
1184 assert fixup_policy == 'ignore'
1187 self.post_process(filename, info_dict)
1188 except (PostProcessingError) as err:
1189 self.report_error('postprocessing: %s' % str(err))
1191 self.record_download_archive(info_dict)
1193 def download(self, url_list):
1194 """Download a given list of URLs."""
1195 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1196 if (len(url_list) > 1 and
1198 and self.params.get('max_downloads') != 1):
1199 raise SameFileError(outtmpl)
1201 for url in url_list:
1203 # It also downloads the videos
1204 res = self.extract_info(url)
1205 except UnavailableVideoError:
1206 self.report_error('unable to download video')
1207 except MaxDownloadsReached:
1208 self.to_screen('[info] Maximum number of downloaded files reached.')
1211 if self.params.get('dump_single_json', False):
1212 self.to_stdout(json.dumps(res))
1214 return self._download_retcode
1216 def download_with_info_file(self, info_filename):
1217 with io.open(info_filename, 'r', encoding='utf-8') as f:
1220 self.process_ie_result(info, download=True)
1221 except DownloadError:
1222 webpage_url = info.get('webpage_url')
1223 if webpage_url is not None:
1224 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1225 return self.download([webpage_url])
1228 return self._download_retcode
1230 def post_process(self, filename, ie_info):
1231 """Run all the postprocessors on the given file."""
1232 info = dict(ie_info)
1233 info['filepath'] = filename
1236 if ie_info.get('__postprocessors') is not None:
1237 pps_chain.extend(ie_info['__postprocessors'])
1238 pps_chain.extend(self._pps)
1239 for pp in pps_chain:
1241 keep_video_wish, new_info = pp.run(info)
1242 if keep_video_wish is not None:
1244 keep_video = keep_video_wish
1245 elif keep_video is None:
1246 # No clear decision yet, let IE decide
1247 keep_video = keep_video_wish
1248 except PostProcessingError as e:
1249 self.report_error(e.msg)
1250 if keep_video is False and not self.params.get('keepvideo', False):
1252 self.to_screen('Deleting original file %s (pass -k to keep)' % filename)
1253 os.remove(encodeFilename(filename))
1254 except (IOError, OSError):
1255 self.report_warning('Unable to remove downloaded video file')
1257 def _make_archive_id(self, info_dict):
1258 # Future-proof against any change in case
1259 # and backwards compatibility with prior versions
1260 extractor = info_dict.get('extractor_key')
1261 if extractor is None:
1262 if 'id' in info_dict:
1263 extractor = info_dict.get('ie_key') # key in a playlist
1264 if extractor is None:
1265 return None # Incomplete video information
1266 return extractor.lower() + ' ' + info_dict['id']
1268 def in_download_archive(self, info_dict):
1269 fn = self.params.get('download_archive')
1273 vid_id = self._make_archive_id(info_dict)
1275 return False # Incomplete video information
1278 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1279 for line in archive_file:
1280 if line.strip() == vid_id:
1282 except IOError as ioe:
1283 if ioe.errno != errno.ENOENT:
1287 def record_download_archive(self, info_dict):
1288 fn = self.params.get('download_archive')
1291 vid_id = self._make_archive_id(info_dict)
1293 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1294 archive_file.write(vid_id + '\n')
1297 def format_resolution(format, default='unknown'):
1298 if format.get('vcodec') == 'none':
1300 if format.get('resolution') is not None:
1301 return format['resolution']
1302 if format.get('height') is not None:
1303 if format.get('width') is not None:
1304 res = '%sx%s' % (format['width'], format['height'])
1306 res = '%sp' % format['height']
1307 elif format.get('width') is not None:
1308 res = '?x%d' % format['width']
1313 def _format_note(self, fdict):
1315 if fdict.get('ext') in ['f4f', 'f4m']:
1316 res += '(unsupported) '
1317 if fdict.get('format_note') is not None:
1318 res += fdict['format_note'] + ' '
1319 if fdict.get('tbr') is not None:
1320 res += '%4dk ' % fdict['tbr']
1321 if fdict.get('container') is not None:
1324 res += '%s container' % fdict['container']
1325 if (fdict.get('vcodec') is not None and
1326 fdict.get('vcodec') != 'none'):
1329 res += fdict['vcodec']
1330 if fdict.get('vbr') is not None:
1332 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1334 if fdict.get('vbr') is not None:
1335 res += '%4dk' % fdict['vbr']
1336 if fdict.get('fps') is not None:
1337 res += ', %sfps' % fdict['fps']
1338 if fdict.get('acodec') is not None:
1341 if fdict['acodec'] == 'none':
1344 res += '%-5s' % fdict['acodec']
1345 elif fdict.get('abr') is not None:
1349 if fdict.get('abr') is not None:
1350 res += '@%3dk' % fdict['abr']
1351 if fdict.get('asr') is not None:
1352 res += ' (%5dHz)' % fdict['asr']
1353 if fdict.get('filesize') is not None:
1356 res += format_bytes(fdict['filesize'])
1357 elif fdict.get('filesize_approx') is not None:
1360 res += '~' + format_bytes(fdict['filesize_approx'])
1363 def list_formats(self, info_dict):
1364 def line(format, idlen=20):
1365 return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
1366 format['format_id'],
1368 self.format_resolution(format),
1369 self._format_note(format),
1372 formats = info_dict.get('formats', [info_dict])
1373 idlen = max(len('format code'),
1374 max(len(f['format_id']) for f in formats))
1376 line(f, idlen) for f in formats
1377 if f.get('preference') is None or f['preference'] >= -1000]
1378 if len(formats) > 1:
1379 formats_s[0] += (' ' if self._format_note(formats[0]) else '') + '(worst)'
1380 formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
1382 header_line = line({
1383 'format_id': 'format code', 'ext': 'extension',
1384 'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
1385 self.to_screen('[info] Available formats for %s:\n%s\n%s' %
1386 (info_dict['id'], header_line, '\n'.join(formats_s)))
1388 def urlopen(self, req):
1389 """ Start an HTTP download """
1391 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1392 # always respected by websites, some tend to give out URLs with non percent-encoded
1393 # non-ASCII characters (see telemb.py, ard.py [#3412])
1394 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1395 # To work around aforementioned issue we will replace request's original URL with
1396 # percent-encoded one
1397 req_is_string = isinstance(req, basestring if sys.version_info < (3, 0) else compat_str)
1398 url = req if req_is_string else req.get_full_url()
1399 url_escaped = escape_url(url)
1401 # Substitute URL if any change after escaping
1402 if url != url_escaped:
1406 req = compat_urllib_request.Request(
1407 url_escaped, data=req.data, headers=req.headers,
1408 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1410 return self._opener.open(req, timeout=self._socket_timeout)
1412 def print_debug_header(self):
1413 if not self.params.get('verbose'):
1416 if type('') is not compat_str:
1417 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1418 self.report_warning(
1419 'Your Python is broken! Update to a newer and supported version')
1421 stdout_encoding = getattr(
1422 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1424 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1425 locale.getpreferredencoding(),
1426 sys.getfilesystemencoding(),
1428 self.get_encoding()))
1429 write_string(encoding_str, encoding=None)
1431 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1433 sp = subprocess.Popen(
1434 ['git', 'rev-parse', '--short', 'HEAD'],
1435 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1436 cwd=os.path.dirname(os.path.abspath(__file__)))
1437 out, err = sp.communicate()
1438 out = out.decode().strip()
1439 if re.match('[0-9a-f]+', out):
1440 self._write_string('[debug] Git HEAD: ' + out + '\n')
1446 self._write_string('[debug] Python version %s - %s\n' % (
1447 platform.python_version(), platform_name()))
1449 exe_versions = FFmpegPostProcessor.get_versions()
1450 exe_versions['rtmpdump'] = rtmpdump_version()
1451 exe_str = ', '.join(
1453 for exe, v in sorted(exe_versions.items())
1458 self._write_string('[debug] exe versions: %s\n' % exe_str)
1461 for handler in self._opener.handlers:
1462 if hasattr(handler, 'proxies'):
1463 proxy_map.update(handler.proxies)
1464 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1466 def _setup_opener(self):
1467 timeout_val = self.params.get('socket_timeout')
1468 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1470 opts_cookiefile = self.params.get('cookiefile')
1471 opts_proxy = self.params.get('proxy')
1473 if opts_cookiefile is None:
1474 self.cookiejar = compat_cookiejar.CookieJar()
1476 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1478 if os.access(opts_cookiefile, os.R_OK):
1479 self.cookiejar.load()
1481 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1483 if opts_proxy is not None:
1484 if opts_proxy == '':
1487 proxies = {'http': opts_proxy, 'https': opts_proxy}
1489 proxies = compat_urllib_request.getproxies()
1490 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1491 if 'http' in proxies and 'https' not in proxies:
1492 proxies['https'] = proxies['http']
1493 proxy_handler = compat_urllib_request.ProxyHandler(proxies)
1495 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1496 https_handler = make_HTTPS_handler(
1497 self.params.get('nocheckcertificate', False), debuglevel=debuglevel)
1498 ydlh = YoutubeDLHandler(debuglevel=debuglevel)
1499 opener = compat_urllib_request.build_opener(
1500 https_handler, proxy_handler, cookie_processor, ydlh)
1501 # Delete the default user-agent header, which would otherwise apply in
1502 # cases where our custom HTTP handler doesn't come into play
1503 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1504 opener.addheaders = []
1505 self._opener = opener
1507 def encode(self, s):
1508 if isinstance(s, bytes):
1509 return s # Already encoded
1512 return s.encode(self.get_encoding())
1513 except UnicodeEncodeError as err:
1514 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1517 def get_encoding(self):
1518 encoding = self.params.get('encoding')
1519 if encoding is None:
1520 encoding = preferredencoding()