2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import, unicode_literals
33 compat_urllib_request,
59 UnavailableVideoError,
68 from .cache import Cache
69 from .extractor import get_info_extractor, gen_extractors
70 from .downloader import get_suitable_downloader
71 from .downloader.rtmp import rtmpdump_version
72 from .postprocessor import (
73 FFmpegFixupStretchedPP,
78 from .version import __version__
81 class YoutubeDL(object):
84 YoutubeDL objects are the ones responsible of downloading the
85 actual video file and writing it to disk if the user has requested
86 it, among some other tasks. In most cases there should be one per
87 program. As, given a video URL, the downloader doesn't know how to
88 extract all the needed information, task that InfoExtractors do, it
89 has to pass the URL to one of them.
91 For this, YoutubeDL objects have a method that allows
92 InfoExtractors to be registered in a given order. When it is passed
93 a URL, the YoutubeDL object handles it to the first InfoExtractor it
94 finds that reports being able to handle it. The InfoExtractor extracts
95 all the information about the video or videos the URL refers to, and
96 YoutubeDL process the extracted information, possibly using a File
97 Downloader to download the video.
99 YoutubeDL objects accept a lot of parameters. In order not to saturate
100 the object constructor with arguments, it receives a dictionary of
101 options instead. These options are available through the params
102 attribute for the InfoExtractors to use. The YoutubeDL also
103 registers itself as the downloader in charge for the InfoExtractors
104 that are added to it, so this is a "mutual registration".
108 username: Username for authentication purposes.
109 password: Password for authentication purposes.
110 videopassword: Password for acces a video.
111 usenetrc: Use netrc for authentication instead.
112 verbose: Print additional info to stdout.
113 quiet: Do not print messages to stdout.
114 no_warnings: Do not print out anything for warnings.
115 forceurl: Force printing final URL.
116 forcetitle: Force printing title.
117 forceid: Force printing ID.
118 forcethumbnail: Force printing thumbnail URL.
119 forcedescription: Force printing description.
120 forcefilename: Force printing final filename.
121 forceduration: Force printing duration.
122 forcejson: Force printing info_dict as JSON.
123 dump_single_json: Force printing the info_dict of the whole playlist
124 (or video) as a single JSON line.
125 simulate: Do not download the video files.
126 format: Video format code. See options.py for more information.
127 format_limit: Highest quality format to try.
128 outtmpl: Template for output names.
129 restrictfilenames: Do not allow "&" and spaces in file names
130 ignoreerrors: Do not stop on download errors.
131 nooverwrites: Prevent overwriting files.
132 playliststart: Playlist item to start at.
133 playlistend: Playlist item to end at.
134 playlistreverse: Download playlist items in reverse order.
135 matchtitle: Download only matching titles.
136 rejecttitle: Reject downloads for matching titles.
137 logger: Log messages to a logging.Logger instance.
138 logtostderr: Log messages to stderr instead of stdout.
139 writedescription: Write the video description to a .description file
140 writeinfojson: Write the video description to a .info.json file
141 writeannotations: Write the video annotations to a .annotations.xml file
142 writethumbnail: Write the thumbnail image to a file
143 writesubtitles: Write the video subtitles to a file
144 writeautomaticsub: Write the automatic subtitles to a file
145 allsubtitles: Downloads all the subtitles of the video
146 (requires writesubtitles or writeautomaticsub)
147 listsubtitles: Lists all available subtitles for the video
148 subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
149 subtitleslangs: List of languages of the subtitles to download
150 keepvideo: Keep the video file after post-processing
151 daterange: A DateRange object, download only if the upload_date is in the range.
152 skip_download: Skip the actual download of the video file
153 cachedir: Location of the cache files in the filesystem.
154 False to disable filesystem cache.
155 noplaylist: Download single video instead of a playlist if in doubt.
156 age_limit: An integer representing the user's age in years.
157 Unsuitable videos for the given age are skipped.
158 min_views: An integer representing the minimum view count the video
159 must have in order to not be skipped.
160 Videos without view count information are always
161 downloaded. None for no limit.
162 max_views: An integer representing the maximum view count.
163 Videos that are more popular than that are not
165 Videos without view count information are always
166 downloaded. None for no limit.
167 download_archive: File name of a file where all downloads are recorded.
168 Videos already present in the file are not downloaded
170 cookiefile: File name where cookies should be read from and dumped to.
171 nocheckcertificate:Do not verify SSL certificates
172 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
173 At the moment, this is only supported by YouTube.
174 proxy: URL of the proxy server to use
175 socket_timeout: Time to wait for unresponsive hosts, in seconds
176 bidi_workaround: Work around buggy terminals without bidirectional text
177 support, using fridibi
178 debug_printtraffic:Print out sent and received HTTP traffic
179 include_ads: Download ads as well
180 default_search: Prepend this string if an input url is not valid.
181 'auto' for elaborate guessing
182 encoding: Use this encoding instead of the system-specified.
183 extract_flat: Do not resolve URLs, return the immediate result.
184 Pass in 'in_playlist' to only show this behavior for
186 postprocessors: A list of dictionaries, each with an entry
187 * key: The name of the postprocessor. See
188 youtube_dl/postprocessor/__init__.py for a list.
189 as well as any further keyword arguments for the
191 progress_hooks: A list of functions that get called on download
192 progress, with a dictionary with the entries
193 * filename: The final filename
194 * status: One of "downloading" and "finished"
196 The dict may also have some of the following entries:
198 * downloaded_bytes: Bytes on disk
199 * total_bytes: Size of the whole file, None if unknown
200 * tmpfilename: The filename we're currently writing to
201 * eta: The estimated time in seconds, None if unknown
202 * speed: The download speed in bytes/second, None if
205 Progress hooks are guaranteed to be called at least once
206 (with status "finished") if the download is successful.
207 merge_output_format: Extension to use when merging formats.
208 fixup: Automatically correct known faults of the file.
210 - "never": do nothing
211 - "warn": only emit a warning
212 - "detect_or_warn": check whether we can do anything
213 about it, warn otherwise
214 source_address: (Experimental) Client-side IP address to bind to.
217 The following parameters are not used by YoutubeDL itself, they are used by
219 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
220 noresizebuffer, retries, continuedl, noprogress, consoletitle
222 The following options are used by the post processors:
223 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
224 otherwise prefer avconv.
225 exec_cmd: Arbitrary command to run after downloading
231 _download_retcode = None
232 _num_downloads = None
235 def __init__(self, params=None, auto_init=True):
236 """Create a FileDownloader object with the given options."""
240 self._ies_instances = {}
242 self._progress_hooks = []
243 self._download_retcode = 0
244 self._num_downloads = 0
245 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
246 self._err_file = sys.stderr
248 self.cache = Cache(self)
250 if params.get('bidi_workaround', False):
253 master, slave = pty.openpty()
254 width = get_term_width()
258 width_args = ['-w', str(width)]
260 stdin=subprocess.PIPE,
262 stderr=self._err_file)
264 self._output_process = subprocess.Popen(
265 ['bidiv'] + width_args, **sp_kwargs
268 self._output_process = subprocess.Popen(
269 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
270 self._output_channel = os.fdopen(master, 'rb')
271 except OSError as ose:
273 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
277 if (sys.version_info >= (3,) and sys.platform != 'win32' and
278 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
279 and not params.get('restrictfilenames', False)):
280 # On Python 3, the Unicode filesystem API will throw errors (#1474)
282 'Assuming --restrict-filenames since file system encoding '
283 'cannot encode all characters. '
284 'Set the LC_ALL environment variable to fix this.')
285 self.params['restrictfilenames'] = True
287 if '%(stitle)s' in self.params.get('outtmpl', ''):
288 self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
293 self.print_debug_header()
294 self.add_default_info_extractors()
296 for pp_def_raw in self.params.get('postprocessors', []):
297 pp_class = get_postprocessor(pp_def_raw['key'])
298 pp_def = dict(pp_def_raw)
300 pp = pp_class(self, **compat_kwargs(pp_def))
301 self.add_post_processor(pp)
303 for ph in self.params.get('progress_hooks', []):
304 self.add_progress_hook(ph)
306 def warn_if_short_id(self, argv):
307 # short YouTube ID starting with dash?
309 i for i, a in enumerate(argv)
310 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
314 [a for i, a in enumerate(argv) if i not in idxs] +
315 ['--'] + [argv[i] for i in idxs]
318 'Long argument string detected. '
319 'Use -- to separate parameters and URLs, like this:\n%s\n' %
320 args_to_str(correct_argv))
322 def add_info_extractor(self, ie):
323 """Add an InfoExtractor object to the end of the list."""
325 self._ies_instances[ie.ie_key()] = ie
326 ie.set_downloader(self)
328 def get_info_extractor(self, ie_key):
330 Get an instance of an IE with name ie_key, it will try to get one from
331 the _ies list, if there's no instance it will create a new one and add
332 it to the extractor list.
334 ie = self._ies_instances.get(ie_key)
336 ie = get_info_extractor(ie_key)()
337 self.add_info_extractor(ie)
340 def add_default_info_extractors(self):
342 Add the InfoExtractors returned by gen_extractors to the end of the list
344 for ie in gen_extractors():
345 self.add_info_extractor(ie)
347 def add_post_processor(self, pp):
348 """Add a PostProcessor object to the end of the chain."""
350 pp.set_downloader(self)
352 def add_progress_hook(self, ph):
353 """Add the progress hook (currently only for the file downloader)"""
354 self._progress_hooks.append(ph)
356 def _bidi_workaround(self, message):
357 if not hasattr(self, '_output_channel'):
360 assert hasattr(self, '_output_process')
361 assert isinstance(message, compat_str)
362 line_count = message.count('\n') + 1
363 self._output_process.stdin.write((message + '\n').encode('utf-8'))
364 self._output_process.stdin.flush()
365 res = ''.join(self._output_channel.readline().decode('utf-8')
366 for _ in range(line_count))
367 return res[:-len('\n')]
369 def to_screen(self, message, skip_eol=False):
370 """Print message to stdout if not in quiet mode."""
371 return self.to_stdout(message, skip_eol, check_quiet=True)
373 def _write_string(self, s, out=None):
374 write_string(s, out=out, encoding=self.params.get('encoding'))
376 def to_stdout(self, message, skip_eol=False, check_quiet=False):
377 """Print message to stdout if not in quiet mode."""
378 if self.params.get('logger'):
379 self.params['logger'].debug(message)
380 elif not check_quiet or not self.params.get('quiet', False):
381 message = self._bidi_workaround(message)
382 terminator = ['\n', ''][skip_eol]
383 output = message + terminator
385 self._write_string(output, self._screen_file)
387 def to_stderr(self, message):
388 """Print message to stderr."""
389 assert isinstance(message, compat_str)
390 if self.params.get('logger'):
391 self.params['logger'].error(message)
393 message = self._bidi_workaround(message)
394 output = message + '\n'
395 self._write_string(output, self._err_file)
397 def to_console_title(self, message):
398 if not self.params.get('consoletitle', False):
400 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
401 # c_wchar_p() might not be necessary if `message` is
402 # already of type unicode()
403 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
404 elif 'TERM' in os.environ:
405 self._write_string('\033]0;%s\007' % message, self._screen_file)
407 def save_console_title(self):
408 if not self.params.get('consoletitle', False):
410 if 'TERM' in os.environ:
411 # Save the title on stack
412 self._write_string('\033[22;0t', self._screen_file)
414 def restore_console_title(self):
415 if not self.params.get('consoletitle', False):
417 if 'TERM' in os.environ:
418 # Restore the title from stack
419 self._write_string('\033[23;0t', self._screen_file)
422 self.save_console_title()
425 def __exit__(self, *args):
426 self.restore_console_title()
428 if self.params.get('cookiefile') is not None:
429 self.cookiejar.save()
431 def trouble(self, message=None, tb=None):
432 """Determine action to take when a download problem appears.
434 Depending on if the downloader has been configured to ignore
435 download errors or not, this method may throw an exception or
436 not when errors are found, after printing the message.
438 tb, if given, is additional traceback information.
440 if message is not None:
441 self.to_stderr(message)
442 if self.params.get('verbose'):
444 if sys.exc_info()[0]: # if .trouble has been called from an except block
446 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
447 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
448 tb += compat_str(traceback.format_exc())
450 tb_data = traceback.format_list(traceback.extract_stack())
451 tb = ''.join(tb_data)
453 if not self.params.get('ignoreerrors', False):
454 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
455 exc_info = sys.exc_info()[1].exc_info
457 exc_info = sys.exc_info()
458 raise DownloadError(message, exc_info)
459 self._download_retcode = 1
461 def report_warning(self, message):
463 Print the message to stderr, it will be prefixed with 'WARNING:'
464 If stderr is a tty file the 'WARNING:' will be colored
466 if self.params.get('logger') is not None:
467 self.params['logger'].warning(message)
469 if self.params.get('no_warnings'):
471 if self._err_file.isatty() and os.name != 'nt':
472 _msg_header = '\033[0;33mWARNING:\033[0m'
474 _msg_header = 'WARNING:'
475 warning_message = '%s %s' % (_msg_header, message)
476 self.to_stderr(warning_message)
478 def report_error(self, message, tb=None):
480 Do the same as trouble, but prefixes the message with 'ERROR:', colored
481 in red if stderr is a tty file.
483 if self._err_file.isatty() and os.name != 'nt':
484 _msg_header = '\033[0;31mERROR:\033[0m'
486 _msg_header = 'ERROR:'
487 error_message = '%s %s' % (_msg_header, message)
488 self.trouble(error_message, tb)
490 def report_file_already_downloaded(self, file_name):
491 """Report file has already been fully downloaded."""
493 self.to_screen('[download] %s has already been downloaded' % file_name)
494 except UnicodeEncodeError:
495 self.to_screen('[download] The file has already been downloaded')
497 def prepare_filename(self, info_dict):
498 """Generate the output filename."""
500 template_dict = dict(info_dict)
502 template_dict['epoch'] = int(time.time())
503 autonumber_size = self.params.get('autonumber_size')
504 if autonumber_size is None:
506 autonumber_templ = '%0' + str(autonumber_size) + 'd'
507 template_dict['autonumber'] = autonumber_templ % self._num_downloads
508 if template_dict.get('playlist_index') is not None:
509 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
510 if template_dict.get('resolution') is None:
511 if template_dict.get('width') and template_dict.get('height'):
512 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
513 elif template_dict.get('height'):
514 template_dict['resolution'] = '%sp' % template_dict['height']
515 elif template_dict.get('width'):
516 template_dict['resolution'] = '?x%d' % template_dict['width']
518 sanitize = lambda k, v: sanitize_filename(
520 restricted=self.params.get('restrictfilenames'),
522 template_dict = dict((k, sanitize(k, v))
523 for k, v in template_dict.items()
525 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
527 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
528 tmpl = compat_expanduser(outtmpl)
529 filename = tmpl % template_dict
531 except ValueError as err:
532 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
535 def _match_entry(self, info_dict):
536 """ Returns None iff the file should be downloaded """
538 video_title = info_dict.get('title', info_dict.get('id', 'video'))
539 if 'title' in info_dict:
540 # This can happen when we're just evaluating the playlist
541 title = info_dict['title']
542 matchtitle = self.params.get('matchtitle', False)
544 if not re.search(matchtitle, title, re.IGNORECASE):
545 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
546 rejecttitle = self.params.get('rejecttitle', False)
548 if re.search(rejecttitle, title, re.IGNORECASE):
549 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
550 date = info_dict.get('upload_date', None)
552 dateRange = self.params.get('daterange', DateRange())
553 if date not in dateRange:
554 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
555 view_count = info_dict.get('view_count', None)
556 if view_count is not None:
557 min_views = self.params.get('min_views')
558 if min_views is not None and view_count < min_views:
559 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
560 max_views = self.params.get('max_views')
561 if max_views is not None and view_count > max_views:
562 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
563 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
564 return 'Skipping "%s" because it is age restricted' % title
565 if self.in_download_archive(info_dict):
566 return '%s has already been recorded in archive' % video_title
570 def add_extra_info(info_dict, extra_info):
571 '''Set the keys from extra_info in info dict if they are missing'''
572 for key, value in extra_info.items():
573 info_dict.setdefault(key, value)
575 def extract_info(self, url, download=True, ie_key=None, extra_info={},
578 Returns a list with a dictionary for each video we find.
579 If 'download', also downloads the videos.
580 extra_info is a dict containing the extra values to add to each result
584 ies = [self.get_info_extractor(ie_key)]
589 if not ie.suitable(url):
593 self.report_warning('The program functionality for this site has been marked as broken, '
594 'and will probably not work.')
597 ie_result = ie.extract(url)
598 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
600 if isinstance(ie_result, list):
601 # Backwards compatibility: old IE result format
603 '_type': 'compat_list',
604 'entries': ie_result,
606 self.add_default_extra_info(ie_result, ie, url)
608 return self.process_ie_result(ie_result, download, extra_info)
611 except ExtractorError as de: # An error we somewhat expected
612 self.report_error(compat_str(de), de.format_traceback())
614 except MaxDownloadsReached:
616 except Exception as e:
617 if self.params.get('ignoreerrors', False):
618 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
623 self.report_error('no suitable InfoExtractor for URL %s' % url)
625 def add_default_extra_info(self, ie_result, ie, url):
626 self.add_extra_info(ie_result, {
627 'extractor': ie.IE_NAME,
629 'webpage_url_basename': url_basename(url),
630 'extractor_key': ie.ie_key(),
633 def process_ie_result(self, ie_result, download=True, extra_info={}):
635 Take the result of the ie(may be modified) and resolve all unresolved
636 references (URLs, playlist items).
638 It will also download the videos if 'download'.
639 Returns the resolved ie_result.
642 result_type = ie_result.get('_type', 'video')
644 if result_type in ('url', 'url_transparent'):
645 extract_flat = self.params.get('extract_flat', False)
646 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
647 extract_flat is True):
648 if self.params.get('forcejson', False):
649 self.to_stdout(json.dumps(ie_result))
652 if result_type == 'video':
653 self.add_extra_info(ie_result, extra_info)
654 return self.process_video_result(ie_result, download=download)
655 elif result_type == 'url':
656 # We have to add extra_info to the results because it may be
657 # contained in a playlist
658 return self.extract_info(ie_result['url'],
660 ie_key=ie_result.get('ie_key'),
661 extra_info=extra_info)
662 elif result_type == 'url_transparent':
663 # Use the information from the embedding page
664 info = self.extract_info(
665 ie_result['url'], ie_key=ie_result.get('ie_key'),
666 extra_info=extra_info, download=False, process=False)
668 force_properties = dict(
669 (k, v) for k, v in ie_result.items() if v is not None)
670 for f in ('_type', 'url'):
671 if f in force_properties:
672 del force_properties[f]
673 new_result = info.copy()
674 new_result.update(force_properties)
676 assert new_result.get('_type') != 'url_transparent'
678 return self.process_ie_result(
679 new_result, download=download, extra_info=extra_info)
680 elif result_type == 'playlist' or result_type == 'multi_video':
681 # We process each entry in the playlist
682 playlist = ie_result.get('title', None) or ie_result.get('id', None)
683 self.to_screen('[download] Downloading playlist: %s' % playlist)
685 playlist_results = []
687 playliststart = self.params.get('playliststart', 1) - 1
688 playlistend = self.params.get('playlistend', None)
689 # For backwards compatibility, interpret -1 as whole list
690 if playlistend == -1:
693 ie_entries = ie_result['entries']
694 if isinstance(ie_entries, list):
695 n_all_entries = len(ie_entries)
696 entries = ie_entries[playliststart:playlistend]
697 n_entries = len(entries)
699 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
700 (ie_result['extractor'], playlist, n_all_entries, n_entries))
701 elif isinstance(ie_entries, PagedList):
702 entries = ie_entries.getslice(
703 playliststart, playlistend)
704 n_entries = len(entries)
706 "[%s] playlist %s: Downloading %d videos" %
707 (ie_result['extractor'], playlist, n_entries))
709 entries = list(itertools.islice(
710 ie_entries, playliststart, playlistend))
711 n_entries = len(entries)
713 "[%s] playlist %s: Downloading %d videos" %
714 (ie_result['extractor'], playlist, n_entries))
716 if self.params.get('playlistreverse', False):
717 entries = entries[::-1]
719 for i, entry in enumerate(entries, 1):
720 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
722 'n_entries': n_entries,
723 'playlist': playlist,
724 'playlist_id': ie_result.get('id'),
725 'playlist_title': ie_result.get('title'),
726 'playlist_index': i + playliststart,
727 'extractor': ie_result['extractor'],
728 'webpage_url': ie_result['webpage_url'],
729 'webpage_url_basename': url_basename(ie_result['webpage_url']),
730 'extractor_key': ie_result['extractor_key'],
733 reason = self._match_entry(entry)
734 if reason is not None:
735 self.to_screen('[download] ' + reason)
738 entry_result = self.process_ie_result(entry,
741 playlist_results.append(entry_result)
742 ie_result['entries'] = playlist_results
744 elif result_type == 'compat_list':
746 'Extractor %s returned a compat_list result. '
747 'It needs to be updated.' % ie_result.get('extractor'))
753 'extractor': ie_result['extractor'],
754 'webpage_url': ie_result['webpage_url'],
755 'webpage_url_basename': url_basename(ie_result['webpage_url']),
756 'extractor_key': ie_result['extractor_key'],
760 ie_result['entries'] = [
761 self.process_ie_result(_fixup(r), download, extra_info)
762 for r in ie_result['entries']
766 raise Exception('Invalid result type: %s' % result_type)
768 def select_format(self, format_spec, available_formats):
769 if format_spec == 'best' or format_spec is None:
770 return available_formats[-1]
771 elif format_spec == 'worst':
772 return available_formats[0]
773 elif format_spec == 'bestaudio':
775 f for f in available_formats
776 if f.get('vcodec') == 'none']
778 return audio_formats[-1]
779 elif format_spec == 'worstaudio':
781 f for f in available_formats
782 if f.get('vcodec') == 'none']
784 return audio_formats[0]
785 elif format_spec == 'bestvideo':
787 f for f in available_formats
788 if f.get('acodec') == 'none']
790 return video_formats[-1]
791 elif format_spec == 'worstvideo':
793 f for f in available_formats
794 if f.get('acodec') == 'none']
796 return video_formats[0]
798 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
799 if format_spec in extensions:
800 filter_f = lambda f: f['ext'] == format_spec
802 filter_f = lambda f: f['format_id'] == format_spec
803 matches = list(filter(filter_f, available_formats))
808 def process_video_result(self, info_dict, download=True):
809 assert info_dict.get('_type', 'video') == 'video'
811 if 'id' not in info_dict:
812 raise ExtractorError('Missing "id" field in extractor result')
813 if 'title' not in info_dict:
814 raise ExtractorError('Missing "title" field in extractor result')
816 if 'playlist' not in info_dict:
817 # It isn't part of a playlist
818 info_dict['playlist'] = None
819 info_dict['playlist_index'] = None
821 thumbnails = info_dict.get('thumbnails')
823 thumbnails.sort(key=lambda t: (
824 t.get('width'), t.get('height'), t.get('url')))
826 if 'width' in t and 'height' in t:
827 t['resolution'] = '%dx%d' % (t['width'], t['height'])
829 if thumbnails and 'thumbnail' not in info_dict:
830 info_dict['thumbnail'] = thumbnails[-1]['url']
832 if 'display_id' not in info_dict and 'id' in info_dict:
833 info_dict['display_id'] = info_dict['id']
835 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
836 # Working around negative timestamps in Windows
837 # (see http://bugs.python.org/issue1646728)
838 if info_dict['timestamp'] < 0 and os.name == 'nt':
839 info_dict['timestamp'] = 0
840 upload_date = datetime.datetime.utcfromtimestamp(
841 info_dict['timestamp'])
842 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
844 # This extractors handle format selection themselves
845 if info_dict['extractor'] in ['Youku']:
847 self.process_info(info_dict)
850 # We now pick which formats have to be downloaded
851 if info_dict.get('formats') is None:
852 # There's only one format available
853 formats = [info_dict]
855 formats = info_dict['formats']
858 raise ExtractorError('No video formats found!')
860 # We check that all the formats have the format and format_id fields
861 for i, format in enumerate(formats):
862 if 'url' not in format:
863 raise ExtractorError('Missing "url" key in result (index %d)' % i)
865 if format.get('format_id') is None:
866 format['format_id'] = compat_str(i)
867 if format.get('format') is None:
868 format['format'] = '{id} - {res}{note}'.format(
869 id=format['format_id'],
870 res=self.format_resolution(format),
871 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
873 # Automatically determine file extension if missing
874 if 'ext' not in format:
875 format['ext'] = determine_ext(format['url']).lower()
877 format_limit = self.params.get('format_limit', None)
879 formats = list(takewhile_inclusive(
880 lambda f: f['format_id'] != format_limit, formats
883 # TODO Central sorting goes here
885 if formats[0] is not info_dict:
886 # only set the 'formats' fields if the original info_dict list them
887 # otherwise we end up with a circular reference, the first (and unique)
888 # element in the 'formats' field in info_dict is info_dict itself,
889 # wich can't be exported to json
890 info_dict['formats'] = formats
891 if self.params.get('listformats', None):
892 self.list_formats(info_dict)
895 req_format = self.params.get('format')
896 if req_format is None:
898 formats_to_download = []
899 # The -1 is for supporting YoutubeIE
900 if req_format in ('-1', 'all'):
901 formats_to_download = formats
903 for rfstr in req_format.split(','):
904 # We can accept formats requested in the format: 34/5/best, we pick
905 # the first that is available, starting from left
906 req_formats = rfstr.split('/')
907 for rf in req_formats:
908 if re.match(r'.+?\+.+?', rf) is not None:
909 # Two formats have been requested like '137+139'
910 format_1, format_2 = rf.split('+')
911 formats_info = (self.select_format(format_1, formats),
912 self.select_format(format_2, formats))
913 if all(formats_info):
914 # The first format must contain the video and the
916 if formats_info[0].get('vcodec') == 'none':
917 self.report_error('The first format must '
918 'contain the video, try using '
919 '"-f %s+%s"' % (format_2, format_1))
922 formats_info[0]['ext']
923 if self.params.get('merge_output_format') is None
924 else self.params['merge_output_format'])
926 'requested_formats': formats_info,
928 'ext': formats_info[0]['ext'],
929 'width': formats_info[0].get('width'),
930 'height': formats_info[0].get('height'),
931 'resolution': formats_info[0].get('resolution'),
932 'fps': formats_info[0].get('fps'),
933 'vcodec': formats_info[0].get('vcodec'),
934 'vbr': formats_info[0].get('vbr'),
935 'stretched_ratio': formats_info[0].get('stretched_ratio'),
936 'acodec': formats_info[1].get('acodec'),
937 'abr': formats_info[1].get('abr'),
941 selected_format = None
943 selected_format = self.select_format(rf, formats)
944 if selected_format is not None:
945 formats_to_download.append(selected_format)
947 if not formats_to_download:
948 raise ExtractorError('requested format not available',
952 if len(formats_to_download) > 1:
953 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
954 for format in formats_to_download:
955 new_info = dict(info_dict)
956 new_info.update(format)
957 self.process_info(new_info)
958 # We update the info dict with the best quality format (backwards compatibility)
959 info_dict.update(formats_to_download[-1])
962 def process_info(self, info_dict):
963 """Process a single resolved IE result."""
965 assert info_dict.get('_type', 'video') == 'video'
967 max_downloads = self.params.get('max_downloads')
968 if max_downloads is not None:
969 if self._num_downloads >= int(max_downloads):
970 raise MaxDownloadsReached()
972 info_dict['fulltitle'] = info_dict['title']
973 if len(info_dict['title']) > 200:
974 info_dict['title'] = info_dict['title'][:197] + '...'
976 # Keep for backwards compatibility
977 info_dict['stitle'] = info_dict['title']
979 if 'format' not in info_dict:
980 info_dict['format'] = info_dict['ext']
982 reason = self._match_entry(info_dict)
983 if reason is not None:
984 self.to_screen('[download] ' + reason)
987 self._num_downloads += 1
989 filename = self.prepare_filename(info_dict)
992 if self.params.get('forcetitle', False):
993 self.to_stdout(info_dict['fulltitle'])
994 if self.params.get('forceid', False):
995 self.to_stdout(info_dict['id'])
996 if self.params.get('forceurl', False):
997 if info_dict.get('requested_formats') is not None:
998 for f in info_dict['requested_formats']:
999 self.to_stdout(f['url'] + f.get('play_path', ''))
1001 # For RTMP URLs, also include the playpath
1002 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1003 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1004 self.to_stdout(info_dict['thumbnail'])
1005 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1006 self.to_stdout(info_dict['description'])
1007 if self.params.get('forcefilename', False) and filename is not None:
1008 self.to_stdout(filename)
1009 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1010 self.to_stdout(formatSeconds(info_dict['duration']))
1011 if self.params.get('forceformat', False):
1012 self.to_stdout(info_dict['format'])
1013 if self.params.get('forcejson', False):
1014 info_dict['_filename'] = filename
1015 self.to_stdout(json.dumps(info_dict))
1016 if self.params.get('dump_single_json', False):
1017 info_dict['_filename'] = filename
1019 # Do nothing else if in simulate mode
1020 if self.params.get('simulate', False):
1023 if filename is None:
1027 dn = os.path.dirname(encodeFilename(filename))
1028 if dn and not os.path.exists(dn):
1030 except (OSError, IOError) as err:
1031 self.report_error('unable to create directory ' + compat_str(err))
1034 if self.params.get('writedescription', False):
1035 descfn = filename + '.description'
1036 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1037 self.to_screen('[info] Video description is already present')
1038 elif info_dict.get('description') is None:
1039 self.report_warning('There\'s no description to write.')
1042 self.to_screen('[info] Writing video description to: ' + descfn)
1043 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1044 descfile.write(info_dict['description'])
1045 except (OSError, IOError):
1046 self.report_error('Cannot write description file ' + descfn)
1049 if self.params.get('writeannotations', False):
1050 annofn = filename + '.annotations.xml'
1051 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1052 self.to_screen('[info] Video annotations are already present')
1055 self.to_screen('[info] Writing video annotations to: ' + annofn)
1056 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1057 annofile.write(info_dict['annotations'])
1058 except (KeyError, TypeError):
1059 self.report_warning('There are no annotations to write.')
1060 except (OSError, IOError):
1061 self.report_error('Cannot write annotations file: ' + annofn)
1064 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1065 self.params.get('writeautomaticsub')])
1067 if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
1068 # subtitles download errors are already managed as troubles in relevant IE
1069 # that way it will silently go on when used with unsupporting IE
1070 subtitles = info_dict['subtitles']
1071 sub_format = self.params.get('subtitlesformat', 'srt')
1072 for sub_lang in subtitles.keys():
1073 sub = subtitles[sub_lang]
1077 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1078 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1079 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1081 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1082 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1084 except (OSError, IOError):
1085 self.report_error('Cannot write subtitles file ' + sub_filename)
1088 if self.params.get('writeinfojson', False):
1089 infofn = os.path.splitext(filename)[0] + '.info.json'
1090 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1091 self.to_screen('[info] Video description metadata is already present')
1093 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1095 write_json_file(info_dict, infofn)
1096 except (OSError, IOError):
1097 self.report_error('Cannot write metadata to JSON file ' + infofn)
1100 if self.params.get('writethumbnail', False):
1101 if info_dict.get('thumbnail') is not None:
1102 thumb_format = determine_ext(info_dict['thumbnail'], 'jpg')
1103 thumb_filename = os.path.splitext(filename)[0] + '.' + thumb_format
1104 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
1105 self.to_screen('[%s] %s: Thumbnail is already present' %
1106 (info_dict['extractor'], info_dict['id']))
1108 self.to_screen('[%s] %s: Downloading thumbnail ...' %
1109 (info_dict['extractor'], info_dict['id']))
1111 uf = self.urlopen(info_dict['thumbnail'])
1112 with open(thumb_filename, 'wb') as thumbf:
1113 shutil.copyfileobj(uf, thumbf)
1114 self.to_screen('[%s] %s: Writing thumbnail to: %s' %
1115 (info_dict['extractor'], info_dict['id'], thumb_filename))
1116 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1117 self.report_warning('Unable to download thumbnail "%s": %s' %
1118 (info_dict['thumbnail'], compat_str(err)))
1120 if not self.params.get('skip_download', False):
1121 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
1126 fd = get_suitable_downloader(info)(self, self.params)
1127 for ph in self._progress_hooks:
1128 fd.add_progress_hook(ph)
1129 if self.params.get('verbose'):
1130 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1131 return fd.download(name, info)
1132 if info_dict.get('requested_formats') is not None:
1135 merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
1136 if not merger._executable:
1138 self.report_warning('You have requested multiple '
1139 'formats but ffmpeg or avconv are not installed.'
1140 ' The formats won\'t be merged')
1142 postprocessors = [merger]
1143 for f in info_dict['requested_formats']:
1144 new_info = dict(info_dict)
1146 fname = self.prepare_filename(new_info)
1147 fname = prepend_extension(fname, 'f%s' % f['format_id'])
1148 downloaded.append(fname)
1149 partial_success = dl(fname, new_info)
1150 success = success and partial_success
1151 info_dict['__postprocessors'] = postprocessors
1152 info_dict['__files_to_merge'] = downloaded
1154 # Just a single file
1155 success = dl(filename, info_dict)
1156 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1157 self.report_error('unable to download video data: %s' % str(err))
1159 except (OSError, IOError) as err:
1160 raise UnavailableVideoError(err)
1161 except (ContentTooShortError, ) as err:
1162 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1167 stretched_ratio = info_dict.get('stretched_ratio')
1168 if stretched_ratio is not None and stretched_ratio != 1:
1169 fixup_policy = self.params.get('fixup')
1170 if fixup_policy is None:
1171 fixup_policy = 'detect_or_warn'
1172 if fixup_policy == 'warn':
1173 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1174 info_dict['id'], stretched_ratio))
1175 elif fixup_policy == 'detect_or_warn':
1176 stretched_pp = FFmpegFixupStretchedPP(self)
1177 if stretched_pp.available:
1178 info_dict.setdefault('__postprocessors', [])
1179 info_dict['__postprocessors'].append(stretched_pp)
1181 self.report_warning(
1182 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1183 info_dict['id'], stretched_ratio))
1185 assert fixup_policy == 'ignore'
1188 self.post_process(filename, info_dict)
1189 except (PostProcessingError) as err:
1190 self.report_error('postprocessing: %s' % str(err))
1192 self.record_download_archive(info_dict)
1194 def download(self, url_list):
1195 """Download a given list of URLs."""
1196 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1197 if (len(url_list) > 1 and
1199 and self.params.get('max_downloads') != 1):
1200 raise SameFileError(outtmpl)
1202 for url in url_list:
1204 # It also downloads the videos
1205 res = self.extract_info(url)
1206 except UnavailableVideoError:
1207 self.report_error('unable to download video')
1208 except MaxDownloadsReached:
1209 self.to_screen('[info] Maximum number of downloaded files reached.')
1212 if self.params.get('dump_single_json', False):
1213 self.to_stdout(json.dumps(res))
1215 return self._download_retcode
1217 def download_with_info_file(self, info_filename):
1218 with io.open(info_filename, 'r', encoding='utf-8') as f:
1221 self.process_ie_result(info, download=True)
1222 except DownloadError:
1223 webpage_url = info.get('webpage_url')
1224 if webpage_url is not None:
1225 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1226 return self.download([webpage_url])
1229 return self._download_retcode
1231 def post_process(self, filename, ie_info):
1232 """Run all the postprocessors on the given file."""
1233 info = dict(ie_info)
1234 info['filepath'] = filename
1237 if ie_info.get('__postprocessors') is not None:
1238 pps_chain.extend(ie_info['__postprocessors'])
1239 pps_chain.extend(self._pps)
1240 for pp in pps_chain:
1242 keep_video_wish, new_info = pp.run(info)
1243 if keep_video_wish is not None:
1245 keep_video = keep_video_wish
1246 elif keep_video is None:
1247 # No clear decision yet, let IE decide
1248 keep_video = keep_video_wish
1249 except PostProcessingError as e:
1250 self.report_error(e.msg)
1251 if keep_video is False and not self.params.get('keepvideo', False):
1253 self.to_screen('Deleting original file %s (pass -k to keep)' % filename)
1254 os.remove(encodeFilename(filename))
1255 except (IOError, OSError):
1256 self.report_warning('Unable to remove downloaded video file')
1258 def _make_archive_id(self, info_dict):
1259 # Future-proof against any change in case
1260 # and backwards compatibility with prior versions
1261 extractor = info_dict.get('extractor_key')
1262 if extractor is None:
1263 if 'id' in info_dict:
1264 extractor = info_dict.get('ie_key') # key in a playlist
1265 if extractor is None:
1266 return None # Incomplete video information
1267 return extractor.lower() + ' ' + info_dict['id']
1269 def in_download_archive(self, info_dict):
1270 fn = self.params.get('download_archive')
1274 vid_id = self._make_archive_id(info_dict)
1276 return False # Incomplete video information
1279 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1280 for line in archive_file:
1281 if line.strip() == vid_id:
1283 except IOError as ioe:
1284 if ioe.errno != errno.ENOENT:
1288 def record_download_archive(self, info_dict):
1289 fn = self.params.get('download_archive')
1292 vid_id = self._make_archive_id(info_dict)
1294 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1295 archive_file.write(vid_id + '\n')
1298 def format_resolution(format, default='unknown'):
1299 if format.get('vcodec') == 'none':
1301 if format.get('resolution') is not None:
1302 return format['resolution']
1303 if format.get('height') is not None:
1304 if format.get('width') is not None:
1305 res = '%sx%s' % (format['width'], format['height'])
1307 res = '%sp' % format['height']
1308 elif format.get('width') is not None:
1309 res = '?x%d' % format['width']
1314 def _format_note(self, fdict):
1316 if fdict.get('ext') in ['f4f', 'f4m']:
1317 res += '(unsupported) '
1318 if fdict.get('format_note') is not None:
1319 res += fdict['format_note'] + ' '
1320 if fdict.get('tbr') is not None:
1321 res += '%4dk ' % fdict['tbr']
1322 if fdict.get('container') is not None:
1325 res += '%s container' % fdict['container']
1326 if (fdict.get('vcodec') is not None and
1327 fdict.get('vcodec') != 'none'):
1330 res += fdict['vcodec']
1331 if fdict.get('vbr') is not None:
1333 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1335 if fdict.get('vbr') is not None:
1336 res += '%4dk' % fdict['vbr']
1337 if fdict.get('fps') is not None:
1338 res += ', %sfps' % fdict['fps']
1339 if fdict.get('acodec') is not None:
1342 if fdict['acodec'] == 'none':
1345 res += '%-5s' % fdict['acodec']
1346 elif fdict.get('abr') is not None:
1350 if fdict.get('abr') is not None:
1351 res += '@%3dk' % fdict['abr']
1352 if fdict.get('asr') is not None:
1353 res += ' (%5dHz)' % fdict['asr']
1354 if fdict.get('filesize') is not None:
1357 res += format_bytes(fdict['filesize'])
1358 elif fdict.get('filesize_approx') is not None:
1361 res += '~' + format_bytes(fdict['filesize_approx'])
1364 def list_formats(self, info_dict):
1365 def line(format, idlen=20):
1366 return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
1367 format['format_id'],
1369 self.format_resolution(format),
1370 self._format_note(format),
1373 formats = info_dict.get('formats', [info_dict])
1374 idlen = max(len('format code'),
1375 max(len(f['format_id']) for f in formats))
1377 line(f, idlen) for f in formats
1378 if f.get('preference') is None or f['preference'] >= -1000]
1379 if len(formats) > 1:
1380 formats_s[0] += (' ' if self._format_note(formats[0]) else '') + '(worst)'
1381 formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
1383 header_line = line({
1384 'format_id': 'format code', 'ext': 'extension',
1385 'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
1386 self.to_screen('[info] Available formats for %s:\n%s\n%s' %
1387 (info_dict['id'], header_line, '\n'.join(formats_s)))
1389 def urlopen(self, req):
1390 """ Start an HTTP download """
1392 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1393 # always respected by websites, some tend to give out URLs with non percent-encoded
1394 # non-ASCII characters (see telemb.py, ard.py [#3412])
1395 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1396 # To work around aforementioned issue we will replace request's original URL with
1397 # percent-encoded one
1398 req_is_string = isinstance(req, basestring if sys.version_info < (3, 0) else compat_str)
1399 url = req if req_is_string else req.get_full_url()
1400 url_escaped = escape_url(url)
1402 # Substitute URL if any change after escaping
1403 if url != url_escaped:
1407 req = compat_urllib_request.Request(
1408 url_escaped, data=req.data, headers=req.headers,
1409 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1411 return self._opener.open(req, timeout=self._socket_timeout)
1413 def print_debug_header(self):
1414 if not self.params.get('verbose'):
1417 if type('') is not compat_str:
1418 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1419 self.report_warning(
1420 'Your Python is broken! Update to a newer and supported version')
1422 stdout_encoding = getattr(
1423 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1425 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1426 locale.getpreferredencoding(),
1427 sys.getfilesystemencoding(),
1429 self.get_encoding()))
1430 write_string(encoding_str, encoding=None)
1432 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1434 sp = subprocess.Popen(
1435 ['git', 'rev-parse', '--short', 'HEAD'],
1436 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1437 cwd=os.path.dirname(os.path.abspath(__file__)))
1438 out, err = sp.communicate()
1439 out = out.decode().strip()
1440 if re.match('[0-9a-f]+', out):
1441 self._write_string('[debug] Git HEAD: ' + out + '\n')
1447 self._write_string('[debug] Python version %s - %s\n' % (
1448 platform.python_version(), platform_name()))
1450 exe_versions = FFmpegPostProcessor.get_versions()
1451 exe_versions['rtmpdump'] = rtmpdump_version()
1452 exe_str = ', '.join(
1454 for exe, v in sorted(exe_versions.items())
1459 self._write_string('[debug] exe versions: %s\n' % exe_str)
1462 for handler in self._opener.handlers:
1463 if hasattr(handler, 'proxies'):
1464 proxy_map.update(handler.proxies)
1465 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1467 def _setup_opener(self):
1468 timeout_val = self.params.get('socket_timeout')
1469 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1471 opts_cookiefile = self.params.get('cookiefile')
1472 opts_proxy = self.params.get('proxy')
1474 if opts_cookiefile is None:
1475 self.cookiejar = compat_cookiejar.CookieJar()
1477 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1479 if os.access(opts_cookiefile, os.R_OK):
1480 self.cookiejar.load()
1482 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1484 if opts_proxy is not None:
1485 if opts_proxy == '':
1488 proxies = {'http': opts_proxy, 'https': opts_proxy}
1490 proxies = compat_urllib_request.getproxies()
1491 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1492 if 'http' in proxies and 'https' not in proxies:
1493 proxies['https'] = proxies['http']
1494 proxy_handler = compat_urllib_request.ProxyHandler(proxies)
1496 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1497 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1498 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1499 opener = compat_urllib_request.build_opener(
1500 https_handler, proxy_handler, cookie_processor, ydlh)
1501 # Delete the default user-agent header, which would otherwise apply in
1502 # cases where our custom HTTP handler doesn't come into play
1503 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1504 opener.addheaders = []
1505 self._opener = opener
1507 def encode(self, s):
1508 if isinstance(s, bytes):
1509 return s # Already encoded
1512 return s.encode(self.get_encoding())
1513 except UnicodeEncodeError as err:
1514 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1517 def get_encoding(self):
1518 encoding = self.params.get('encoding')
1519 if encoding is None:
1520 encoding = preferredencoding()