8 compat_subprocess_get_DEVNULL,
17 class PostProcessor(object):
18 """Post Processor class.
20 PostProcessor objects can be added to downloaders with their
21 add_post_processor() method. When the downloader has finished a
22 successful download, it will take its internal chain of PostProcessors
23 and start calling the run() method on each one of them, first with
24 an initial argument and then with the returned value of the previous
27 The chain will be stopped if one of them ever returns None or the end
28 of the chain is reached.
30 PostProcessor objects follow a "mutual registration" process similar
31 to InfoExtractor objects.
36 def __init__(self, downloader=None):
37 self._downloader = downloader
39 def set_downloader(self, downloader):
40 """Sets the downloader for this PP."""
41 self._downloader = downloader
43 def run(self, information):
44 """Run the PostProcessor.
46 The "information" argument is a dictionary like the ones
47 composed by InfoExtractors. The only difference is that this
48 one has an extra field called "filepath" that points to the
51 This method returns a tuple, the first element of which describes
52 whether the original file should be kept (i.e. not deleted - None for
53 no preference), and the second of which is the updated information.
55 In addition, this method may raise a PostProcessingError
56 exception if post processing fails.
58 return None, information # by default, keep file and do nothing
60 class FFmpegPostProcessorError(PostProcessingError):
63 class AudioConversionError(PostProcessingError):
67 class FFmpegPostProcessor(PostProcessor):
68 def __init__(self,downloader=None):
69 PostProcessor.__init__(self, downloader)
70 self._exes = self.detect_executables()
73 def detect_executables():
76 subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
80 programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
81 return dict((program, executable(program)) for program in programs)
83 def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
84 if not self._exes['ffmpeg'] and not self._exes['avconv']:
85 raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.')
88 for path in input_paths:
89 files_cmd.extend(['-i', encodeFilename(path, True)])
90 cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y'] + files_cmd
92 [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
94 if self._downloader.params.get('verbose', False):
95 self._downloader.to_screen(u'[debug] ffmpeg command line: %s' % shell_quote(cmd))
96 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
97 stdout,stderr = p.communicate()
99 stderr = stderr.decode('utf-8', 'replace')
100 msg = stderr.strip().split('\n')[-1]
101 raise FFmpegPostProcessorError(msg)
103 def run_ffmpeg(self, path, out_path, opts):
104 self.run_ffmpeg_multiple_files([path], out_path, opts)
106 def _ffmpeg_filename_argument(self, fn):
107 # ffmpeg broke --, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details
108 if fn.startswith(u'-'):
113 class FFmpegExtractAudioPP(FFmpegPostProcessor):
114 def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, nopostoverwrites=False):
115 FFmpegPostProcessor.__init__(self, downloader)
116 if preferredcodec is None:
117 preferredcodec = 'best'
118 self._preferredcodec = preferredcodec
119 self._preferredquality = preferredquality
120 self._nopostoverwrites = nopostoverwrites
122 def get_audio_codec(self, path):
123 if not self._exes['ffprobe'] and not self._exes['avprobe']:
124 raise PostProcessingError(u'ffprobe or avprobe not found. Please install one.')
127 self._exes['avprobe'] or self._exes['ffprobe'],
129 encodeFilename(self._ffmpeg_filename_argument(path), True)]
130 handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
131 output = handle.communicate()[0]
132 if handle.wait() != 0:
134 except (IOError, OSError):
137 for line in output.decode('ascii', 'ignore').split('\n'):
138 if line.startswith('codec_name='):
139 audio_codec = line.split('=')[1].strip()
140 elif line.strip() == 'codec_type=audio' and audio_codec is not None:
144 def run_ffmpeg(self, path, out_path, codec, more_opts):
145 if not self._exes['ffmpeg'] and not self._exes['avconv']:
146 raise AudioConversionError('ffmpeg or avconv not found. Please install one.')
150 acodec_opts = ['-acodec', codec]
151 opts = ['-vn'] + acodec_opts + more_opts
153 FFmpegPostProcessor.run_ffmpeg(self, path, out_path, opts)
154 except FFmpegPostProcessorError as err:
155 raise AudioConversionError(err.msg)
157 def run(self, information):
158 path = information['filepath']
160 filecodec = self.get_audio_codec(path)
161 if filecodec is None:
162 raise PostProcessingError(u'WARNING: unable to obtain file audio codec with ffprobe')
165 if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
166 if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']:
167 # Lossless, but in another container
170 more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
171 elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']:
172 # Lossless if possible
174 extension = filecodec
175 if filecodec == 'aac':
176 more_opts = ['-f', 'adts']
177 if filecodec == 'vorbis':
181 acodec = 'libmp3lame'
184 if self._preferredquality is not None:
185 if int(self._preferredquality) < 10:
186 more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
188 more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
190 # We convert the audio (lossy)
191 acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'opus': 'opus', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
192 extension = self._preferredcodec
194 if self._preferredquality is not None:
195 # The opus codec doesn't support the -aq option
196 if int(self._preferredquality) < 10 and extension != 'opus':
197 more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality]
199 more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k']
200 if self._preferredcodec == 'aac':
201 more_opts += ['-f', 'adts']
202 if self._preferredcodec == 'm4a':
203 more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc']
204 if self._preferredcodec == 'vorbis':
206 if self._preferredcodec == 'wav':
208 more_opts += ['-f', 'wav']
210 prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
211 new_path = prefix + sep + extension
213 # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly.
215 self._nopostoverwrites = True
218 if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
219 self._downloader.to_screen(u'[youtube] Post-process file %s exists, skipping' % new_path)
221 self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path)
222 self.run_ffmpeg(path, new_path, acodec, more_opts)
224 etype,e,tb = sys.exc_info()
225 if isinstance(e, AudioConversionError):
226 msg = u'audio conversion failed: ' + e.msg
228 msg = u'error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg')
229 raise PostProcessingError(msg)
231 # Try to update the date time for extracted audio file.
232 if information.get('filetime') is not None:
234 os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
236 self._downloader.report_warning(u'Cannot update utime of audio file')
238 information['filepath'] = new_path
239 return self._nopostoverwrites,information
242 class FFmpegVideoConvertor(FFmpegPostProcessor):
243 def __init__(self, downloader=None,preferedformat=None):
244 super(FFmpegVideoConvertor, self).__init__(downloader)
245 self._preferedformat=preferedformat
247 def run(self, information):
248 path = information['filepath']
249 prefix, sep, ext = path.rpartition(u'.')
250 outpath = prefix + sep + self._preferedformat
251 if information['ext'] == self._preferedformat:
252 self._downloader.to_screen(u'[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
253 return True,information
254 self._downloader.to_screen(u'['+'ffmpeg'+'] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) +outpath)
255 self.run_ffmpeg(path, outpath, [])
256 information['filepath'] = outpath
257 information['format'] = self._preferedformat
258 information['ext'] = self._preferedformat
259 return False,information
262 class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
263 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
451 def __init__(self, downloader=None, subtitlesformat='srt'):
452 super(FFmpegEmbedSubtitlePP, self).__init__(downloader)
453 self._subformat = subtitlesformat
456 def _conver_lang_code(cls, code):
457 """Convert language code from ISO 639-1 to ISO 639-2/T"""
458 return cls._lang_map.get(code[:2])
460 def run(self, information):
461 if information['ext'] != u'mp4':
462 self._downloader.to_screen(u'[ffmpeg] Subtitles can only be embedded in mp4 files')
463 return True, information
464 if not information.get('subtitles'):
465 self._downloader.to_screen(u'[ffmpeg] There aren\'t any subtitles to embed')
466 return True, information
468 sub_langs = [key for key in information['subtitles']]
469 filename = information['filepath']
470 input_files = [filename] + [subtitles_filename(filename, lang, self._subformat) for lang in sub_langs]
472 opts = ['-map', '0:0', '-map', '0:1', '-c:v', 'copy', '-c:a', 'copy']
473 for (i, lang) in enumerate(sub_langs):
474 opts.extend(['-map', '%d:0' % (i+1), '-c:s:%d' % i, 'mov_text'])
475 lang_code = self._conver_lang_code(lang)
476 if lang_code is not None:
477 opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])
478 opts.extend(['-f', 'mp4'])
480 temp_filename = filename + u'.temp'
481 self._downloader.to_screen(u'[ffmpeg] Embedding subtitles in \'%s\'' % filename)
482 self.run_ffmpeg_multiple_files(input_files, temp_filename, opts)
483 os.remove(encodeFilename(filename))
484 os.rename(encodeFilename(temp_filename), encodeFilename(filename))
486 return True, information
489 class FFmpegMetadataPP(FFmpegPostProcessor):
492 if info.get('title') is not None:
493 metadata['title'] = info['title']
494 if info.get('upload_date') is not None:
495 metadata['date'] = info['upload_date']
496 if info.get('uploader') is not None:
497 metadata['artist'] = info['uploader']
498 elif info.get('uploader_id') is not None:
499 metadata['artist'] = info['uploader_id']
502 self._downloader.to_screen(u'[ffmpeg] There isn\'t any metadata to add')
505 filename = info['filepath']
506 temp_filename = prepend_extension(filename, 'temp')
508 options = ['-c', 'copy']
509 for (name, value) in metadata.items():
510 options.extend(['-metadata', '%s=%s' % (name, value)])
512 self._downloader.to_screen(u'[ffmpeg] Adding metadata to \'%s\'' % filename)
513 self.run_ffmpeg(filename, temp_filename, options)
514 os.remove(encodeFilename(filename))
515 os.rename(encodeFilename(temp_filename), encodeFilename(filename))
519 class FFmpegMergerPP(FFmpegPostProcessor):
521 filename = info['filepath']
522 args = ['-c', 'copy']
523 self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
527 class XAttrMetadataPP(PostProcessor):
530 # More info about extended attributes for media:
531 # http://freedesktop.org/wiki/CommonExtendedAttributes/
532 # http://www.freedesktop.org/wiki/PhreedomDraft/
533 # http://dublincore.org/documents/usageguide/elements.shtml
536 # * capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated)
537 # * figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution'
541 """ Set extended attributes on downloaded file (if xattr support is found). """
543 from .utils import hyphenate_date
545 # This mess below finds the best xattr tool for the job and creates a
546 # "write_xattr" function.
548 # try the pyxattr module...
550 def write_xattr(path, key, value):
551 return xattr.setxattr(path, key, value)
555 if os.name == 'posix':
557 for dir in os.environ["PATH"].split(":"):
558 path = os.path.join(dir, bin)
559 if os.path.exists(path):
562 user_has_setfattr = which("setfattr")
563 user_has_xattr = which("xattr")
565 if user_has_setfattr or user_has_xattr:
567 def write_xattr(path, key, value):
570 # setfattr: /tmp/blah: Operation not supported
571 "Operation not supported": errno.EOPNOTSUPP,
572 # setfattr: ~/blah: No such file or directory
573 # xattr: No such file: ~/blah
574 "No such file": errno.ENOENT,
577 if user_has_setfattr:
578 cmd = ['setfattr', '-n', key, '-v', value, path]
580 cmd = ['xattr', '-w', key, value, path]
583 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
584 except subprocess.CalledProcessError as e:
585 errorstr = e.output.strip().decode()
586 for potential_errorstr, potential_errno in potential_errors.items():
587 if errorstr.find(potential_errorstr) > -1:
588 e = OSError(potential_errno, potential_errorstr)
591 raise # Reraise unhandled error
594 # On Unix, and can't find pyxattr, setfattr, or xattr.
595 if sys.platform.startswith('linux'):
596 self._downloader.report_error("Couldn't find a tool to set the xattrs. Install either the python 'pyxattr' or 'xattr' modules, or the GNU 'attr' package (which contains the 'setfattr' tool).")
597 elif sys.platform == 'darwin':
598 self._downloader.report_error("Couldn't find a tool to set the xattrs. Install either the python 'xattr' module, or the 'xattr' binary.")
600 # Write xattrs to NTFS Alternate Data Streams: http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
601 def write_xattr(path, key, value):
602 assert(key.find(":") < 0)
603 assert(path.find(":") < 0)
604 assert(os.path.exists(path))
606 ads_fn = path + ":" + key
607 with open(ads_fn, "w") as f:
610 # Write the metadata to the file's xattrs
611 self._downloader.to_screen('[metadata] Writing metadata to file\'s xattrs...')
613 filename = info['filepath']
617 'user.xdg.referrer.url': 'webpage_url',
618 # 'user.xdg.comment': 'description',
619 'user.dublincore.title': 'title',
620 'user.dublincore.date': 'upload_date',
621 'user.dublincore.description': 'description',
622 'user.dublincore.contributor': 'uploader',
623 'user.dublincore.format': 'format',
626 for xattrname, infoname in xattr_mapping.items():
628 value = info.get(infoname)
631 if infoname == "upload_date":
632 value = hyphenate_date(value)
634 write_xattr(filename, xattrname, value)
639 self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")