2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import
28 compat_urllib_request,
46 UnavailableVideoError,
51 from .extractor import get_info_extractor, gen_extractors
52 from .FileDownloader import FileDownloader
53 from .version import __version__
56 class YoutubeDL(object):
59 YoutubeDL objects are the ones responsible of downloading the
60 actual video file and writing it to disk if the user has requested
61 it, among some other tasks. In most cases there should be one per
62 program. As, given a video URL, the downloader doesn't know how to
63 extract all the needed information, task that InfoExtractors do, it
64 has to pass the URL to one of them.
66 For this, YoutubeDL objects have a method that allows
67 InfoExtractors to be registered in a given order. When it is passed
68 a URL, the YoutubeDL object handles it to the first InfoExtractor it
69 finds that reports being able to handle it. The InfoExtractor extracts
70 all the information about the video or videos the URL refers to, and
71 YoutubeDL process the extracted information, possibly using a File
72 Downloader to download the video.
74 YoutubeDL objects accept a lot of parameters. In order not to saturate
75 the object constructor with arguments, it receives a dictionary of
76 options instead. These options are available through the params
77 attribute for the InfoExtractors to use. The YoutubeDL also
78 registers itself as the downloader in charge for the InfoExtractors
79 that are added to it, so this is a "mutual registration".
83 username: Username for authentication purposes.
84 password: Password for authentication purposes.
85 videopassword: Password for acces a video.
86 usenetrc: Use netrc for authentication instead.
87 verbose: Print additional info to stdout.
88 quiet: Do not print messages to stdout.
89 forceurl: Force printing final URL.
90 forcetitle: Force printing title.
91 forceid: Force printing ID.
92 forcethumbnail: Force printing thumbnail URL.
93 forcedescription: Force printing description.
94 forcefilename: Force printing final filename.
95 forcejson: Force printing info_dict as JSON.
96 simulate: Do not download the video files.
97 format: Video format code.
98 format_limit: Highest quality format to try.
99 outtmpl: Template for output names.
100 restrictfilenames: Do not allow "&" and spaces in file names
101 ignoreerrors: Do not stop on download errors.
102 nooverwrites: Prevent overwriting files.
103 playliststart: Playlist item to start at.
104 playlistend: Playlist item to end at.
105 matchtitle: Download only matching titles.
106 rejecttitle: Reject downloads for matching titles.
107 logtostderr: Log messages to stderr instead of stdout.
108 writedescription: Write the video description to a .description file
109 writeinfojson: Write the video description to a .info.json file
110 writeannotations: Write the video annotations to a .annotations.xml file
111 writethumbnail: Write the thumbnail image to a file
112 writesubtitles: Write the video subtitles to a file
113 writeautomaticsub: Write the automatic subtitles to a file
114 allsubtitles: Downloads all the subtitles of the video
115 (requires writesubtitles or writeautomaticsub)
116 listsubtitles: Lists all available subtitles for the video
117 subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
118 subtitleslangs: List of languages of the subtitles to download
119 keepvideo: Keep the video file after post-processing
120 daterange: A DateRange object, download only if the upload_date is in the range.
121 skip_download: Skip the actual download of the video file
122 cachedir: Location of the cache files in the filesystem.
123 None to disable filesystem cache.
124 noplaylist: Download single video instead of a playlist if in doubt.
125 age_limit: An integer representing the user's age in years.
126 Unsuitable videos for the given age are skipped.
127 downloadarchive: File name of a file where all downloads are recorded.
128 Videos already present in the file are not downloaded
130 cookiefile: File name where cookies should be read from and dumped to.
131 nocheckcertificate Do not verify SSL certificates
133 The following parameters are not used by YoutubeDL itself, they are used by
135 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
136 noresizebuffer, retries, continuedl, noprogress, consoletitle
142 _download_retcode = None
143 _num_downloads = None
146 def __init__(self, params):
147 """Create a FileDownloader object with the given options."""
149 self._ies_instances = {}
151 self._progress_hooks = []
152 self._download_retcode = 0
153 self._num_downloads = 0
154 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
156 if (sys.version_info >= (3,) and sys.platform != 'win32' and
157 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
158 and not params['restrictfilenames']):
159 # On Python 3, the Unicode filesystem API will throw errors (#1474)
161 u'Assuming --restrict-filenames since file system encoding '
162 u'cannot encode all charactes. '
163 u'Set the LC_ALL environment variable to fix this.')
164 params['restrictfilenames'] = True
167 self.fd = FileDownloader(self, self.params)
169 if '%(stitle)s' in self.params['outtmpl']:
170 self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
174 def add_info_extractor(self, ie):
175 """Add an InfoExtractor object to the end of the list."""
177 self._ies_instances[ie.ie_key()] = ie
178 ie.set_downloader(self)
180 def get_info_extractor(self, ie_key):
182 Get an instance of an IE with name ie_key, it will try to get one from
183 the _ies list, if there's no instance it will create a new one and add
184 it to the extractor list.
186 ie = self._ies_instances.get(ie_key)
188 ie = get_info_extractor(ie_key)()
189 self.add_info_extractor(ie)
192 def add_default_info_extractors(self):
194 Add the InfoExtractors returned by gen_extractors to the end of the list
196 for ie in gen_extractors():
197 self.add_info_extractor(ie)
199 def add_post_processor(self, pp):
200 """Add a PostProcessor object to the end of the chain."""
202 pp.set_downloader(self)
204 def to_screen(self, message, skip_eol=False):
205 """Print message to stdout if not in quiet mode."""
206 if not self.params.get('quiet', False):
207 terminator = [u'\n', u''][skip_eol]
208 output = message + terminator
209 write_string(output, self._screen_file)
211 def to_stderr(self, message):
212 """Print message to stderr."""
213 assert type(message) == type(u'')
214 output = message + u'\n'
215 if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
216 output = output.encode(preferredencoding())
217 sys.stderr.write(output)
219 def to_console_title(self, message):
220 if not self.params.get('consoletitle', False):
222 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
223 # c_wchar_p() might not be necessary if `message` is
224 # already of type unicode()
225 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
226 elif 'TERM' in os.environ:
227 write_string(u'\033]0;%s\007' % message, self._screen_file)
229 def save_console_title(self):
230 if not self.params.get('consoletitle', False):
232 if 'TERM' in os.environ:
233 # Save the title on stack
234 write_string(u'\033[22;0t', self._screen_file)
236 def restore_console_title(self):
237 if not self.params.get('consoletitle', False):
239 if 'TERM' in os.environ:
240 # Restore the title from stack
241 write_string(u'\033[23;0t', self._screen_file)
244 self.save_console_title()
247 def __exit__(self, *args):
248 self.restore_console_title()
250 if self.params.get('cookiefile') is not None:
251 self.cookiejar.save()
253 def fixed_template(self):
254 """Checks if the output template is fixed."""
255 return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
257 def trouble(self, message=None, tb=None):
258 """Determine action to take when a download problem appears.
260 Depending on if the downloader has been configured to ignore
261 download errors or not, this method may throw an exception or
262 not when errors are found, after printing the message.
264 tb, if given, is additional traceback information.
266 if message is not None:
267 self.to_stderr(message)
268 if self.params.get('verbose'):
270 if sys.exc_info()[0]: # if .trouble has been called from an except block
272 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
273 tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
274 tb += compat_str(traceback.format_exc())
276 tb_data = traceback.format_list(traceback.extract_stack())
277 tb = u''.join(tb_data)
279 if not self.params.get('ignoreerrors', False):
280 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
281 exc_info = sys.exc_info()[1].exc_info
283 exc_info = sys.exc_info()
284 raise DownloadError(message, exc_info)
285 self._download_retcode = 1
287 def report_warning(self, message):
289 Print the message to stderr, it will be prefixed with 'WARNING:'
290 If stderr is a tty file the 'WARNING:' will be colored
292 if sys.stderr.isatty() and os.name != 'nt':
293 _msg_header = u'\033[0;33mWARNING:\033[0m'
295 _msg_header = u'WARNING:'
296 warning_message = u'%s %s' % (_msg_header, message)
297 self.to_stderr(warning_message)
299 def report_error(self, message, tb=None):
301 Do the same as trouble, but prefixes the message with 'ERROR:', colored
302 in red if stderr is a tty file.
304 if sys.stderr.isatty() and os.name != 'nt':
305 _msg_header = u'\033[0;31mERROR:\033[0m'
307 _msg_header = u'ERROR:'
308 error_message = u'%s %s' % (_msg_header, message)
309 self.trouble(error_message, tb)
311 def report_writedescription(self, descfn):
312 """ Report that the description file is being written """
313 self.to_screen(u'[info] Writing video description to: ' + descfn)
315 def report_writesubtitles(self, sub_filename):
316 """ Report that the subtitles file is being written """
317 self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename)
319 def report_writeinfojson(self, infofn):
320 """ Report that the metadata file has been written """
321 self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
323 def report_writeannotations(self, annofn):
324 """ Report that the annotations file has been written. """
325 self.to_screen(u'[info] Writing video annotations to: ' + annofn)
327 def report_file_already_downloaded(self, file_name):
328 """Report file has already been fully downloaded."""
330 self.to_screen(u'[download] %s has already been downloaded' % file_name)
331 except UnicodeEncodeError:
332 self.to_screen(u'[download] The file has already been downloaded')
334 def increment_downloads(self):
335 """Increment the ordinal that assigns a number to each file."""
336 self._num_downloads += 1
338 def prepare_filename(self, info_dict):
339 """Generate the output filename."""
341 template_dict = dict(info_dict)
343 template_dict['epoch'] = int(time.time())
344 autonumber_size = self.params.get('autonumber_size')
345 if autonumber_size is None:
347 autonumber_templ = u'%0' + str(autonumber_size) + u'd'
348 template_dict['autonumber'] = autonumber_templ % self._num_downloads
349 if template_dict.get('playlist_index') is not None:
350 template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index']
352 sanitize = lambda k, v: sanitize_filename(
353 u'NA' if v is None else compat_str(v),
354 restricted=self.params.get('restrictfilenames'),
356 template_dict = dict((k, sanitize(k, v))
357 for k, v in template_dict.items())
359 tmpl = os.path.expanduser(self.params['outtmpl'])
360 filename = tmpl % template_dict
362 except KeyError as err:
363 self.report_error(u'Erroneous output template')
365 except ValueError as err:
366 self.report_error(u'Error in output template: ' + str(err) + u' (encoding: ' + repr(preferredencoding()) + ')')
369 def _match_entry(self, info_dict):
370 """ Returns None iff the file should be downloaded """
372 title = info_dict['title']
373 matchtitle = self.params.get('matchtitle', False)
375 if not re.search(matchtitle, title, re.IGNORECASE):
376 return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
377 rejecttitle = self.params.get('rejecttitle', False)
379 if re.search(rejecttitle, title, re.IGNORECASE):
380 return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
381 date = info_dict.get('upload_date', None)
383 dateRange = self.params.get('daterange', DateRange())
384 if date not in dateRange:
385 return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
386 age_limit = self.params.get('age_limit')
387 if age_limit is not None:
388 if age_limit < info_dict.get('age_limit', 0):
389 return u'Skipping "' + title + '" because it is age restricted'
390 if self.in_download_archive(info_dict):
391 return (u'%(title)s has already been recorded in archive'
396 def add_extra_info(info_dict, extra_info):
397 '''Set the keys from extra_info in info dict if they are missing'''
398 for key, value in extra_info.items():
399 info_dict.setdefault(key, value)
401 def extract_info(self, url, download=True, ie_key=None, extra_info={}):
403 Returns a list with a dictionary for each video we find.
404 If 'download', also downloads the videos.
405 extra_info is a dict containing the extra values to add to each result
409 ies = [self.get_info_extractor(ie_key)]
414 if not ie.suitable(url):
418 self.report_warning(u'The program functionality for this site has been marked as broken, '
419 u'and will probably not work.')
422 ie_result = ie.extract(url)
423 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
425 if isinstance(ie_result, list):
426 # Backwards compatibility: old IE result format
428 '_type': 'compat_list',
429 'entries': ie_result,
431 self.add_extra_info(ie_result,
433 'extractor': ie.IE_NAME,
435 'extractor_key': ie.ie_key(),
437 return self.process_ie_result(ie_result, download, extra_info)
438 except ExtractorError as de: # An error we somewhat expected
439 self.report_error(compat_str(de), de.format_traceback())
441 except Exception as e:
442 if self.params.get('ignoreerrors', False):
443 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
448 self.report_error(u'no suitable InfoExtractor: %s' % url)
450 def process_ie_result(self, ie_result, download=True, extra_info={}):
452 Take the result of the ie(may be modified) and resolve all unresolved
453 references (URLs, playlist items).
455 It will also download the videos if 'download'.
456 Returns the resolved ie_result.
459 result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
460 if result_type == 'video':
461 self.add_extra_info(ie_result, extra_info)
462 return self.process_video_result(ie_result, download=download)
463 elif result_type == 'url':
464 # We have to add extra_info to the results because it may be
465 # contained in a playlist
466 return self.extract_info(ie_result['url'],
468 ie_key=ie_result.get('ie_key'),
469 extra_info=extra_info)
470 elif result_type == 'playlist':
471 self.add_extra_info(ie_result, extra_info)
472 # We process each entry in the playlist
473 playlist = ie_result.get('title', None) or ie_result.get('id', None)
474 self.to_screen(u'[download] Downloading playlist: %s' % playlist)
476 playlist_results = []
478 n_all_entries = len(ie_result['entries'])
479 playliststart = self.params.get('playliststart', 1) - 1
480 playlistend = self.params.get('playlistend', -1)
482 if playlistend == -1:
483 entries = ie_result['entries'][playliststart:]
485 entries = ie_result['entries'][playliststart:playlistend]
487 n_entries = len(entries)
489 self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
490 (ie_result['extractor'], playlist, n_all_entries, n_entries))
492 for i, entry in enumerate(entries, 1):
493 self.to_screen(u'[download] Downloading video #%s of %s' % (i, n_entries))
495 'playlist': playlist,
496 'playlist_index': i + playliststart,
497 'extractor': ie_result['extractor'],
498 'webpage_url': ie_result['webpage_url'],
499 'extractor_key': ie_result['extractor_key'],
501 entry_result = self.process_ie_result(entry,
504 playlist_results.append(entry_result)
505 ie_result['entries'] = playlist_results
507 elif result_type == 'compat_list':
509 self.add_extra_info(r,
511 'extractor': ie_result['extractor'],
512 'webpage_url': ie_result['webpage_url'],
513 'extractor_key': ie_result['extractor_key'],
516 ie_result['entries'] = [
517 self.process_ie_result(_fixup(r), download, extra_info)
518 for r in ie_result['entries']
522 raise Exception('Invalid result type: %s' % result_type)
524 def select_format(self, format_spec, available_formats):
525 if format_spec == 'best' or format_spec is None:
526 return available_formats[-1]
527 elif format_spec == 'worst':
528 return available_formats[0]
530 extensions = [u'mp4', u'flv', u'webm', u'3gp']
531 if format_spec in extensions:
532 filter_f = lambda f: f['ext'] == format_spec
534 filter_f = lambda f: f['format_id'] == format_spec
535 matches = list(filter(filter_f, available_formats))
540 def process_video_result(self, info_dict, download=True):
541 assert info_dict.get('_type', 'video') == 'video'
543 if 'playlist' not in info_dict:
544 # It isn't part of a playlist
545 info_dict['playlist'] = None
546 info_dict['playlist_index'] = None
548 # This extractors handle format selection themselves
549 if info_dict['extractor'] in [u'youtube', u'Youku']:
551 self.process_info(info_dict)
554 # We now pick which formats have to be downloaded
555 if info_dict.get('formats') is None:
556 # There's only one format available
557 formats = [info_dict]
559 formats = info_dict['formats']
561 # We check that all the formats have the format and format_id fields
562 for (i, format) in enumerate(formats):
563 if format.get('format_id') is None:
564 format['format_id'] = compat_str(i)
565 if format.get('format') is None:
566 format['format'] = u'{id} - {res}{note}'.format(
567 id=format['format_id'],
568 res=self.format_resolution(format),
569 note=u' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
571 # Automatically determine file extension if missing
572 if 'ext' not in format:
573 format['ext'] = determine_ext(format['url'])
575 if self.params.get('listformats', None):
576 self.list_formats(info_dict)
579 format_limit = self.params.get('format_limit', None)
581 formats = list(takewhile_inclusive(
582 lambda f: f['format_id'] != format_limit, formats
584 if self.params.get('prefer_free_formats'):
585 def _free_formats_key(f):
587 ext_ord = [u'flv', u'mp4', u'webm'].index(f['ext'])
590 # We only compare the extension if they have the same height and width
591 return (f.get('height'), f.get('width'), ext_ord)
592 formats = sorted(formats, key=_free_formats_key)
594 req_format = self.params.get('format', 'best')
595 if req_format is None:
597 formats_to_download = []
598 # The -1 is for supporting YoutubeIE
599 if req_format in ('-1', 'all'):
600 formats_to_download = formats
602 # We can accept formats requestd in the format: 34/5/best, we pick
603 # the first that is available, starting from left
604 req_formats = req_format.split('/')
605 for rf in req_formats:
606 selected_format = self.select_format(rf, formats)
607 if selected_format is not None:
608 formats_to_download = [selected_format]
610 if not formats_to_download:
611 raise ExtractorError(u'requested format not available',
615 if len(formats_to_download) > 1:
616 self.to_screen(u'[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
617 for format in formats_to_download:
618 new_info = dict(info_dict)
619 new_info.update(format)
620 self.process_info(new_info)
621 # We update the info dict with the best quality format (backwards compatibility)
622 info_dict.update(formats_to_download[-1])
625 def process_info(self, info_dict):
626 """Process a single resolved IE result."""
628 assert info_dict.get('_type', 'video') == 'video'
629 #We increment the download the download count here to match the previous behaviour.
630 self.increment_downloads()
632 info_dict['fulltitle'] = info_dict['title']
633 if len(info_dict['title']) > 200:
634 info_dict['title'] = info_dict['title'][:197] + u'...'
636 # Keep for backwards compatibility
637 info_dict['stitle'] = info_dict['title']
639 if not 'format' in info_dict:
640 info_dict['format'] = info_dict['ext']
642 reason = self._match_entry(info_dict)
643 if reason is not None:
644 self.to_screen(u'[download] ' + reason)
647 max_downloads = self.params.get('max_downloads')
648 if max_downloads is not None:
649 if self._num_downloads > int(max_downloads):
650 raise MaxDownloadsReached()
652 filename = self.prepare_filename(info_dict)
655 if self.params.get('forcetitle', False):
656 compat_print(info_dict['title'])
657 if self.params.get('forceid', False):
658 compat_print(info_dict['id'])
659 if self.params.get('forceurl', False):
660 # For RTMP URLs, also include the playpath
661 compat_print(info_dict['url'] + info_dict.get('play_path', u''))
662 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
663 compat_print(info_dict['thumbnail'])
664 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
665 compat_print(info_dict['description'])
666 if self.params.get('forcefilename', False) and filename is not None:
667 compat_print(filename)
668 if self.params.get('forceformat', False):
669 compat_print(info_dict['format'])
670 if self.params.get('forcejson', False):
671 compat_print(json.dumps(info_dict))
673 # Do nothing else if in simulate mode
674 if self.params.get('simulate', False):
681 dn = os.path.dirname(encodeFilename(filename))
682 if dn != '' and not os.path.exists(dn):
684 except (OSError, IOError) as err:
685 self.report_error(u'unable to create directory ' + compat_str(err))
688 if self.params.get('writedescription', False):
690 descfn = filename + u'.description'
691 self.report_writedescription(descfn)
692 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
693 descfile.write(info_dict['description'])
694 except (KeyError, TypeError):
695 self.report_warning(u'There\'s no description to write.')
696 except (OSError, IOError):
697 self.report_error(u'Cannot write description file ' + descfn)
700 if self.params.get('writeannotations', False):
702 annofn = filename + u'.annotations.xml'
703 self.report_writeannotations(annofn)
704 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
705 annofile.write(info_dict['annotations'])
706 except (KeyError, TypeError):
707 self.report_warning(u'There are no annotations to write.')
708 except (OSError, IOError):
709 self.report_error(u'Cannot write annotations file: ' + annofn)
712 subtitles_are_requested = any([self.params.get('writesubtitles', False),
713 self.params.get('writeautomaticsub')])
715 if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
716 # subtitles download errors are already managed as troubles in relevant IE
717 # that way it will silently go on when used with unsupporting IE
718 subtitles = info_dict['subtitles']
719 sub_format = self.params.get('subtitlesformat', 'srt')
720 for sub_lang in subtitles.keys():
721 sub = subtitles[sub_lang]
725 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
726 self.report_writesubtitles(sub_filename)
727 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
729 except (OSError, IOError):
730 self.report_error(u'Cannot write subtitles file ' + descfn)
733 if self.params.get('writeinfojson', False):
734 infofn = os.path.splitext(filename)[0] + u'.info.json'
735 self.report_writeinfojson(infofn)
737 json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle'])
738 write_json_file(json_info_dict, encodeFilename(infofn))
739 except (OSError, IOError):
740 self.report_error(u'Cannot write metadata to JSON file ' + infofn)
743 if self.params.get('writethumbnail', False):
744 if info_dict.get('thumbnail') is not None:
745 thumb_format = determine_ext(info_dict['thumbnail'], u'jpg')
746 thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format
747 self.to_screen(u'[%s] %s: Downloading thumbnail ...' %
748 (info_dict['extractor'], info_dict['id']))
750 uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
751 with open(thumb_filename, 'wb') as thumbf:
752 shutil.copyfileobj(uf, thumbf)
753 self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
754 (info_dict['extractor'], info_dict['id'], thumb_filename))
755 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
756 self.report_warning(u'Unable to download thumbnail "%s": %s' %
757 (info_dict['thumbnail'], compat_str(err)))
759 if not self.params.get('skip_download', False):
760 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
764 success = self.fd._do_download(filename, info_dict)
765 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
766 self.report_error(u'unable to download video data: %s' % str(err))
768 except (OSError, IOError) as err:
769 raise UnavailableVideoError(err)
770 except (ContentTooShortError, ) as err:
771 self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
776 self.post_process(filename, info_dict)
777 except (PostProcessingError) as err:
778 self.report_error(u'postprocessing: %s' % str(err))
781 self.record_download_archive(info_dict)
783 def download(self, url_list):
784 """Download a given list of URLs."""
785 if len(url_list) > 1 and self.fixed_template():
786 raise SameFileError(self.params['outtmpl'])
790 #It also downloads the videos
791 self.extract_info(url)
792 except UnavailableVideoError:
793 self.report_error(u'unable to download video')
794 except MaxDownloadsReached:
795 self.to_screen(u'[info] Maximum number of downloaded files reached.')
798 return self._download_retcode
800 def post_process(self, filename, ie_info):
801 """Run all the postprocessors on the given file."""
803 info['filepath'] = filename
807 keep_video_wish, new_info = pp.run(info)
808 if keep_video_wish is not None:
810 keep_video = keep_video_wish
811 elif keep_video is None:
812 # No clear decision yet, let IE decide
813 keep_video = keep_video_wish
814 except PostProcessingError as e:
815 self.report_error(e.msg)
816 if keep_video is False and not self.params.get('keepvideo', False):
818 self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename)
819 os.remove(encodeFilename(filename))
820 except (IOError, OSError):
821 self.report_warning(u'Unable to remove downloaded video file')
823 def in_download_archive(self, info_dict):
824 fn = self.params.get('download_archive')
827 vid_id = info_dict['extractor'] + u' ' + info_dict['id']
829 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
830 for line in archive_file:
831 if line.strip() == vid_id:
833 except IOError as ioe:
834 if ioe.errno != errno.ENOENT:
838 def record_download_archive(self, info_dict):
839 fn = self.params.get('download_archive')
842 vid_id = info_dict['extractor'] + u' ' + info_dict['id']
843 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
844 archive_file.write(vid_id + u'\n')
847 def format_resolution(format, default='unknown'):
848 if format.get('_resolution') is not None:
849 return format['_resolution']
850 if format.get('height') is not None:
851 if format.get('width') is not None:
852 res = u'%sx%s' % (format['width'], format['height'])
854 res = u'%sp' % format['height']
859 def list_formats(self, info_dict):
860 def format_note(fdict):
861 if fdict.get('format_note') is not None:
862 return fdict['format_note']
864 if fdict.get('vcodec') is not None:
865 res += u'%-5s' % fdict['vcodec']
866 elif fdict.get('vbr') is not None:
868 if fdict.get('vbr') is not None:
869 res += u'@%4dk' % fdict['vbr']
870 if fdict.get('acodec') is not None:
873 res += u'%-5s' % fdict['acodec']
874 elif fdict.get('abr') is not None:
878 if fdict.get('abr') is not None:
879 res += u'@%3dk' % fdict['abr']
883 return (u'%-20s%-10s%-12s%s' % (
886 self.format_resolution(format),
891 formats = info_dict.get('formats', [info_dict])
892 formats_s = list(map(line, formats))
894 formats_s[0] += (' ' if format_note(formats[0]) else '') + '(worst)'
895 formats_s[-1] += (' ' if format_note(formats[-1]) else '') + '(best)'
898 'format_id': u'format code', 'ext': u'extension',
899 '_resolution': u'resolution', 'format_note': u'note'})
900 self.to_screen(u'[info] Available formats for %s:\n%s\n%s' %
901 (info_dict['id'], header_line, u"\n".join(formats_s)))
903 def urlopen(self, req):
904 """ Start an HTTP download """
905 return self._opener.open(req)
907 def print_debug_header(self):
908 if not self.params.get('verbose'):
910 write_string(u'[debug] youtube-dl version ' + __version__ + u'\n')
912 sp = subprocess.Popen(
913 ['git', 'rev-parse', '--short', 'HEAD'],
914 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
915 cwd=os.path.dirname(os.path.abspath(__file__)))
916 out, err = sp.communicate()
917 out = out.decode().strip()
918 if re.match('[0-9a-f]+', out):
919 write_string(u'[debug] Git HEAD: ' + out + u'\n')
925 write_string(u'[debug] Python version %s - %s' %
926 (platform.python_version(), platform_name()) + u'\n')
929 for handler in self._opener.handlers:
930 if hasattr(handler, 'proxies'):
931 proxy_map.update(handler.proxies)
932 write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n')
934 def _setup_opener(self, timeout=300):
935 opts_cookiefile = self.params.get('cookiefile')
936 opts_proxy = self.params.get('proxy')
938 if opts_cookiefile is None:
939 self.cookiejar = compat_cookiejar.CookieJar()
941 self.cookiejar = compat_cookiejar.MozillaCookieJar(
943 if os.access(opts_cookiefile, os.R_OK):
944 self.cookiejar.load()
946 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
948 if opts_proxy is not None:
952 proxies = {'http': opts_proxy, 'https': opts_proxy}
954 proxies = compat_urllib_request.getproxies()
955 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
956 if 'http' in proxies and 'https' not in proxies:
957 proxies['https'] = proxies['http']
958 proxy_handler = compat_urllib_request.ProxyHandler(proxies)
959 https_handler = make_HTTPS_handler(
960 self.params.get('nocheckcertificate', False))
961 opener = compat_urllib_request.build_opener(
962 https_handler, proxy_handler, cookie_processor, YoutubeDLHandler())
963 # Delete the default user-agent header, which would otherwise apply in
964 # cases where our custom HTTP handler doesn't come into play
965 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
966 opener.addheaders = []
967 self._opener = opener
969 # TODO remove this global modification
970 compat_urllib_request.install_opener(opener)
971 socket.setdefaulttimeout(timeout)