subtitles_filename,
takewhile_inclusive,
UnavailableVideoError,
+ url_basename,
write_json_file,
write_string,
YoutubeDLHandler,
+ prepend_extension,
)
from .extractor import get_info_extractor, gen_extractors
-from .FileDownloader import FileDownloader
+from .downloader import get_suitable_downloader
+from .PostProcessor import FFmpegMergerPP
from .version import __version__
socket_timeout: Time to wait for unresponsive hosts, in seconds
bidi_workaround: Work around buggy terminals without bidirectional text
support, using fridibi
+ debug_printtraffic:Print out sent and received HTTP traffic
The following parameters are not used by YoutubeDL itself, they are used by
the FileDownloader:
def __init__(self, params=None):
"""Create a FileDownloader object with the given options."""
+ if params is None:
+ params = {}
self._ies = []
self._ies_instances = {}
self._pps = []
self._num_downloads = 0
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
self._err_file = sys.stderr
- self.params = {} if params is None else params
+ self.params = params
if params.get('bidi_workaround', False):
try:
width_args = []
else:
width_args = ['-w', str(width)]
- self._fribidi = subprocess.Popen(
- ['fribidi', '-c', 'UTF-8'] + width_args,
+ sp_kwargs = dict(
stdin=subprocess.PIPE,
stdout=slave,
stderr=self._err_file)
- self._fribidi_channel = os.fdopen(master, 'rb')
+ try:
+ self._output_process = subprocess.Popen(
+ ['bidiv'] + width_args, **sp_kwargs
+ )
+ except OSError:
+ self._output_process = subprocess.Popen(
+ ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
+ self._output_channel = os.fdopen(master, 'rb')
except OSError as ose:
if ose.errno == 2:
self.report_warning(u'Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
u'Set the LC_ALL environment variable to fix this.')
self.params['restrictfilenames'] = True
- self.fd = FileDownloader(self, self.params)
-
if '%(stitle)s' in self.params.get('outtmpl', ''):
self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
self._pps.append(pp)
pp.set_downloader(self)
+ def add_progress_hook(self, ph):
+ """Add the progress hook (currently only for the file downloader)"""
+ self._progress_hooks.append(ph)
+
def _bidi_workaround(self, message):
- if not hasattr(self, '_fribidi_channel'):
+ if not hasattr(self, '_output_channel'):
return message
+ assert hasattr(self, '_output_process')
assert type(message) == type(u'')
line_count = message.count(u'\n') + 1
- self._fribidi.stdin.write((message + u'\n').encode('utf-8'))
- self._fribidi.stdin.flush()
- res = u''.join(self._fribidi_channel.readline().decode('utf-8')
+ self._output_process.stdin.write((message + u'\n').encode('utf-8'))
+ self._output_process.stdin.flush()
+ res = u''.join(self._output_channel.readline().decode('utf-8')
for _ in range(line_count))
return res[:-len(u'\n')]
error_message = u'%s %s' % (_msg_header, message)
self.trouble(error_message, tb)
- def report_writedescription(self, descfn):
- """ Report that the description file is being written """
- self.to_screen(u'[info] Writing video description to: ' + descfn)
-
- def report_writesubtitles(self, sub_filename):
- """ Report that the subtitles file is being written """
- self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename)
-
- def report_writeinfojson(self, infofn):
- """ Report that the metadata file has been written """
- self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
-
- def report_writeannotations(self, annofn):
- """ Report that the annotations file has been written. """
- self.to_screen(u'[info] Writing video annotations to: ' + annofn)
-
def report_file_already_downloaded(self, file_name):
"""Report file has already been fully downloaded."""
try:
{
'extractor': ie.IE_NAME,
'webpage_url': url,
+ 'webpage_url_basename': url_basename(url),
'extractor_key': ie.ie_key(),
})
if process:
def make_result(embedded_info):
new_result = ie_result.copy()
for f in ('_type', 'url', 'ext', 'player_url', 'formats',
- 'entries', 'urlhandle', 'ie_key', 'duration',
+ 'entries', 'ie_key', 'duration',
'subtitles', 'annotations', 'format',
'thumbnail', 'thumbnails'):
if f in new_result:
n_all_entries = len(ie_result['entries'])
playliststart = self.params.get('playliststart', 1) - 1
- playlistend = self.params.get('playlistend', -1)
-
+ playlistend = self.params.get('playlistend', None)
+ # For backwards compatibility, interpret -1 as whole list
if playlistend == -1:
- entries = ie_result['entries'][playliststart:]
- else:
- entries = ie_result['entries'][playliststart:playlistend]
+ playlistend = None
+ entries = ie_result['entries'][playliststart:playlistend]
n_entries = len(entries)
- self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
+ self.to_screen(
+ u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" %
(ie_result['extractor'], playlist, n_all_entries, n_entries))
for i, entry in enumerate(entries, 1):
'playlist_index': i + playliststart,
'extractor': ie_result['extractor'],
'webpage_url': ie_result['webpage_url'],
+ 'webpage_url_basename': url_basename(ie_result['webpage_url']),
'extractor_key': ie_result['extractor_key'],
}
{
'extractor': ie_result['extractor'],
'webpage_url': ie_result['webpage_url'],
+ 'webpage_url_basename': url_basename(ie_result['webpage_url']),
'extractor_key': ie_result['extractor_key'],
})
return r
info_dict['playlist_index'] = None
# This extractors handle format selection themselves
- if info_dict['extractor'] in [u'youtube', u'Youku']:
+ if info_dict['extractor'] in [u'Youku']:
if download:
self.process_info(info_dict)
return info_dict
if 'ext' not in format:
format['ext'] = determine_ext(format['url'])
- if self.params.get('listformats', None):
- self.list_formats(info_dict)
- return
-
format_limit = self.params.get('format_limit', None)
if format_limit:
formats = list(takewhile_inclusive(
lambda f: f['format_id'] != format_limit, formats
))
- if self.params.get('prefer_free_formats'):
- def _free_formats_key(f):
- try:
- ext_ord = [u'flv', u'mp4', u'webm'].index(f['ext'])
- except ValueError:
- ext_ord = -1
- # We only compare the extension if they have the same height and width
- return (f.get('height'), f.get('width'), ext_ord)
- formats = sorted(formats, key=_free_formats_key)
+
+ # TODO Central sorting goes here
+
+ if formats[0] is not info_dict:
+ # only set the 'formats' fields if the original info_dict list them
+ # otherwise we end up with a circular reference, the first (and unique)
+ # element in the 'formats' field in info_dict is info_dict itself,
+ # wich can't be exported to json
+ info_dict['formats'] = formats
+ if self.params.get('listformats', None):
+ self.list_formats(info_dict)
+ return
req_format = self.params.get('format', 'best')
if req_format is None:
# the first that is available, starting from left
req_formats = req_format.split('/')
for rf in req_formats:
- selected_format = self.select_format(rf, formats)
+ if re.match(r'.+?\+.+?', rf) is not None:
+ # Two formats have been requested like '137+139'
+ format_1, format_2 = rf.split('+')
+ formats_info = (self.select_format(format_1, formats),
+ self.select_format(format_2, formats))
+ if all(formats_info):
+ selected_format = {'requested_formats': formats_info}
+ else:
+ selected_format = None
+ else:
+ selected_format = self.select_format(rf, formats)
if selected_format is not None:
formats_to_download = [selected_format]
break
return
if self.params.get('writedescription', False):
- try:
- descfn = filename + u'.description'
- self.report_writedescription(descfn)
- with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
- descfile.write(info_dict['description'])
- except (KeyError, TypeError):
- self.report_warning(u'There\'s no description to write.')
- except (OSError, IOError):
- self.report_error(u'Cannot write description file ' + descfn)
- return
+ descfn = filename + u'.description'
+ if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
+ self.to_screen(u'[info] Video description is already present')
+ else:
+ try:
+ self.to_screen(u'[info] Writing video description to: ' + descfn)
+ with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
+ descfile.write(info_dict['description'])
+ except (KeyError, TypeError):
+ self.report_warning(u'There\'s no description to write.')
+ except (OSError, IOError):
+ self.report_error(u'Cannot write description file ' + descfn)
+ return
if self.params.get('writeannotations', False):
- try:
- annofn = filename + u'.annotations.xml'
- self.report_writeannotations(annofn)
- with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
- annofile.write(info_dict['annotations'])
- except (KeyError, TypeError):
- self.report_warning(u'There are no annotations to write.')
- except (OSError, IOError):
- self.report_error(u'Cannot write annotations file: ' + annofn)
- return
+ annofn = filename + u'.annotations.xml'
+ if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
+ self.to_screen(u'[info] Video annotations are already present')
+ else:
+ try:
+ self.to_screen(u'[info] Writing video annotations to: ' + annofn)
+ with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
+ annofile.write(info_dict['annotations'])
+ except (KeyError, TypeError):
+ self.report_warning(u'There are no annotations to write.')
+ except (OSError, IOError):
+ self.report_error(u'Cannot write annotations file: ' + annofn)
+ return
subtitles_are_requested = any([self.params.get('writesubtitles', False),
self.params.get('writeautomaticsub')])
continue
try:
sub_filename = subtitles_filename(filename, sub_lang, sub_format)
- self.report_writesubtitles(sub_filename)
- with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
- subfile.write(sub)
+ if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
+ self.to_screen(u'[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
+ else:
+ self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename)
+ with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
+ subfile.write(sub)
except (OSError, IOError):
self.report_error(u'Cannot write subtitles file ' + descfn)
return
if self.params.get('writeinfojson', False):
infofn = os.path.splitext(filename)[0] + u'.info.json'
- self.report_writeinfojson(infofn)
- try:
- json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle'])
- write_json_file(json_info_dict, encodeFilename(infofn))
- except (OSError, IOError):
- self.report_error(u'Cannot write metadata to JSON file ' + infofn)
- return
+ if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
+ self.to_screen(u'[info] Video description metadata is already present')
+ else:
+ self.to_screen(u'[info] Writing video description metadata as JSON to: ' + infofn)
+ try:
+ write_json_file(info_dict, encodeFilename(infofn))
+ except (OSError, IOError):
+ self.report_error(u'Cannot write metadata to JSON file ' + infofn)
+ return
if self.params.get('writethumbnail', False):
if info_dict.get('thumbnail') is not None:
thumb_format = determine_ext(info_dict['thumbnail'], u'jpg')
thumb_filename = os.path.splitext(filename)[0] + u'.' + thumb_format
- self.to_screen(u'[%s] %s: Downloading thumbnail ...' %
- (info_dict['extractor'], info_dict['id']))
- try:
- uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
- with open(thumb_filename, 'wb') as thumbf:
- shutil.copyfileobj(uf, thumbf)
- self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
- (info_dict['extractor'], info_dict['id'], thumb_filename))
- except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
- self.report_warning(u'Unable to download thumbnail "%s": %s' %
- (info_dict['thumbnail'], compat_str(err)))
+ if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
+ self.to_screen(u'[%s] %s: Thumbnail is already present' %
+ (info_dict['extractor'], info_dict['id']))
+ else:
+ self.to_screen(u'[%s] %s: Downloading thumbnail ...' %
+ (info_dict['extractor'], info_dict['id']))
+ try:
+ uf = compat_urllib_request.urlopen(info_dict['thumbnail'])
+ with open(thumb_filename, 'wb') as thumbf:
+ shutil.copyfileobj(uf, thumbf)
+ self.to_screen(u'[%s] %s: Writing thumbnail to: %s' %
+ (info_dict['extractor'], info_dict['id'], thumb_filename))
+ except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+ self.report_warning(u'Unable to download thumbnail "%s": %s' %
+ (info_dict['thumbnail'], compat_str(err)))
if not self.params.get('skip_download', False):
if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
success = True
else:
try:
- success = self.fd._do_download(filename, info_dict)
+ def dl(name, info):
+ fd = get_suitable_downloader(info)(self, self.params)
+ for ph in self._progress_hooks:
+ fd.add_progress_hook(ph)
+ return fd.download(name, info)
+ if info_dict.get('requested_formats') is not None:
+ downloaded = []
+ success = True
+ for f in info_dict['requested_formats']:
+ new_info = dict(info_dict)
+ new_info.update(f)
+ fname = self.prepare_filename(new_info)
+ fname = prepend_extension(fname, 'f%s' % f['format_id'])
+ downloaded.append(fname)
+ partial_success = dl(fname, new_info)
+ success = success and partial_success
+ info_dict['__postprocessors'] = [FFmpegMergerPP(self)]
+ info_dict['__files_to_merge'] = downloaded
+ else:
+ # Just a single file
+ success = dl(filename, info_dict)
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self.report_error(u'unable to download video data: %s' % str(err))
return
info = dict(ie_info)
info['filepath'] = filename
keep_video = None
- for pp in self._pps:
+ pps_chain = []
+ if ie_info.get('__postprocessors') is not None:
+ pps_chain.extend(ie_info['__postprocessors'])
+ pps_chain.extend(self._pps)
+ for pp in pps_chain:
try:
keep_video_wish, new_info = pp.run(info)
if keep_video_wish is not None:
def format_resolution(format, default='unknown'):
if format.get('vcodec') == 'none':
return 'audio only'
- if format.get('_resolution') is not None:
- return format['_resolution']
+ if format.get('resolution') is not None:
+ return format['resolution']
if format.get('height') is not None:
if format.get('width') is not None:
res = u'%sx%s' % (format['width'], format['height'])
else:
res = u'%sp' % format['height']
+ elif format.get('width') is not None:
+ res = u'?x%d' % format['width']
else:
res = default
return res
def list_formats(self, info_dict):
def format_note(fdict):
res = u''
+ if fdict.get('ext') in ['f4f', 'f4m']:
+ res += u'(unsupported) '
if fdict.get('format_note') is not None:
res += fdict['format_note'] + u' '
+ if fdict.get('tbr') is not None:
+ res += u'%4dk ' % fdict['tbr']
if (fdict.get('vcodec') is not None and
fdict.get('vcodec') != 'none'):
- res += u'%-5s' % fdict['vcodec']
- elif fdict.get('vbr') is not None:
- res += u'video'
+ res += u'%-5s@' % fdict['vcodec']
+ elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
+ res += u'video@'
if fdict.get('vbr') is not None:
- res += u'@%4dk' % fdict['vbr']
+ res += u'%4dk' % fdict['vbr']
if fdict.get('acodec') is not None:
if res:
res += u', '
header_line = line({
'format_id': u'format code', 'ext': u'extension',
- '_resolution': u'resolution', 'format_note': u'note'}, idlen=idlen)
+ 'resolution': u'resolution', 'format_note': u'note'}, idlen=idlen)
self.to_screen(u'[info] Available formats for %s:\n%s\n%s' %
(info_dict['id'], header_line, u"\n".join(formats_s)))
if 'http' in proxies and 'https' not in proxies:
proxies['https'] = proxies['http']
proxy_handler = compat_urllib_request.ProxyHandler(proxies)
+
+ debuglevel = 1 if self.params.get('debug_printtraffic') else 0
https_handler = make_HTTPS_handler(
- self.params.get('nocheckcertificate', False))
+ self.params.get('nocheckcertificate', False), debuglevel=debuglevel)
+ ydlh = YoutubeDLHandler(debuglevel=debuglevel)
opener = compat_urllib_request.build_opener(
- https_handler, proxy_handler, cookie_processor, YoutubeDLHandler())
+ https_handler, proxy_handler, cookie_processor, ydlh)
# Delete the default user-agent header, which would otherwise apply in
# cases where our custom HTTP handler doesn't come into play
# (See https://github.com/rg3/youtube-dl/issues/1309 for details)
projects / youtube-dl / blobdiff