projects
/
youtube-dl
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
[test/test_socks] Skip SOCKS tests
[youtube-dl]
/
youtube_dl
/
YoutubeDL.py
diff --git
a/youtube_dl/YoutubeDL.py
b/youtube_dl/YoutubeDL.py
index 97cf31eb2cfc4c5f59bb16a0a7b90ec04f0806a8..03a6a189093240c2c4fb96e6c1e383392e425eda 100755
(executable)
--- a/
youtube_dl/YoutubeDL.py
+++ b/
youtube_dl/YoutubeDL.py
@@
-24,9
+24,6
@@
import time
import tokenize
import traceback
import tokenize
import traceback
-if os.name == 'nt':
- import ctypes
-
from .compat import (
compat_basestring,
compat_cookiejar,
from .compat import (
compat_basestring,
compat_cookiejar,
@@
-34,6
+31,7
@@
from .compat import (
compat_get_terminal_size,
compat_http_client,
compat_kwargs,
compat_get_terminal_size,
compat_http_client,
compat_kwargs,
+ compat_os_name,
compat_str,
compat_tokenize_tokenize,
compat_urllib_error,
compat_str,
compat_tokenize_tokenize,
compat_urllib_error,
@@
-41,6
+39,8
@@
from .compat import (
compat_urllib_request_DataHandler,
)
from .utils import (
compat_urllib_request_DataHandler,
)
from .utils import (
+ age_restricted,
+ args_to_str,
ContentTooShortError,
date_from_str,
DateRange,
ContentTooShortError,
date_from_str,
DateRange,
@@
-60,13
+60,17
@@
from .utils import (
PagedList,
parse_filesize,
PerRequestProxyHandler,
PagedList,
parse_filesize,
PerRequestProxyHandler,
- PostProcessingError,
platform_name,
platform_name,
+ PostProcessingError,
preferredencoding,
preferredencoding,
+ prepend_extension,
+ register_socks_protocols,
render_table,
render_table,
+ replace_extension,
SameFileError,
sanitize_filename,
sanitize_path,
SameFileError,
sanitize_filename,
sanitize_path,
+ sanitize_url,
sanitized_Request,
std_headers,
subtitles_filename,
sanitized_Request,
std_headers,
subtitles_filename,
@@
-77,13
+81,9
@@
from .utils import (
write_string,
YoutubeDLCookieProcessor,
YoutubeDLHandler,
write_string,
YoutubeDLCookieProcessor,
YoutubeDLHandler,
- prepend_extension,
- replace_extension,
- args_to_str,
- age_restricted,
)
from .cache import Cache
)
from .cache import Cache
-from .extractor import get_info_extractor, gen_extractor
s
+from .extractor import get_info_extractor, gen_extractor
_classes, _LAZY_LOADER
from .downloader import get_suitable_downloader
from .downloader.rtmp import rtmpdump_version
from .postprocessor import (
from .downloader import get_suitable_downloader
from .downloader.rtmp import rtmpdump_version
from .postprocessor import (
@@
-96,6
+96,9
@@
from .postprocessor import (
)
from .version import __version__
)
from .version import __version__
+if compat_os_name == 'nt':
+ import ctypes
+
class YoutubeDL(object):
"""YoutubeDL class.
class YoutubeDL(object):
"""YoutubeDL class.
@@
-258,7
+261,9
@@
class YoutubeDL(object):
The following options determine which downloader is picked:
external_downloader: Executable of the external downloader to call.
None or unset for standard (built-in) downloader.
The following options determine which downloader is picked:
external_downloader: Executable of the external downloader to call.
None or unset for standard (built-in) downloader.
- hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
+ hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv
+ if True, otherwise use ffmpeg/avconv if False, otherwise
+ use downloader suggested by extractor if None.
The following parameters are not used by YoutubeDL itself, they are used by
the downloader (see youtube_dl/downloader/common.py):
The following parameters are not used by YoutubeDL itself, they are used by
the downloader (see youtube_dl/downloader/common.py):
@@
-357,6
+362,8
@@
class YoutubeDL(object):
for ph in self.params.get('progress_hooks', []):
self.add_progress_hook(ph)
for ph in self.params.get('progress_hooks', []):
self.add_progress_hook(ph)
+ register_socks_protocols()
+
def warn_if_short_id(self, argv):
# short YouTube ID starting with dash?
idxs = [
def warn_if_short_id(self, argv):
# short YouTube ID starting with dash?
idxs = [
@@
-376,8
+383,9
@@
class YoutubeDL(object):
def add_info_extractor(self, ie):
"""Add an InfoExtractor object to the end of the list."""
self._ies.append(ie)
def add_info_extractor(self, ie):
"""Add an InfoExtractor object to the end of the list."""
self._ies.append(ie)
- self._ies_instances[ie.ie_key()] = ie
- ie.set_downloader(self)
+ if not isinstance(ie, type):
+ self._ies_instances[ie.ie_key()] = ie
+ ie.set_downloader(self)
def get_info_extractor(self, ie_key):
"""
def get_info_extractor(self, ie_key):
"""
@@
-395,7
+403,7
@@
class YoutubeDL(object):
"""
Add the InfoExtractors returned by gen_extractors to the end of the list
"""
"""
Add the InfoExtractors returned by gen_extractors to the end of the list
"""
- for ie in gen_extractors():
+ for ie in gen_extractor
_classe
s():
self.add_info_extractor(ie)
def add_post_processor(self, pp):
self.add_info_extractor(ie)
def add_post_processor(self, pp):
@@
-451,7
+459,7
@@
class YoutubeDL(object):
def to_console_title(self, message):
if not self.params.get('consoletitle', False):
return
def to_console_title(self, message):
if not self.params.get('consoletitle', False):
return
- if
os.
name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
+ if
compat_os_
name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
# c_wchar_p() might not be necessary if `message` is
# already of type unicode()
ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
# c_wchar_p() might not be necessary if `message` is
# already of type unicode()
ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
@@
-522,7
+530,7
@@
class YoutubeDL(object):
else:
if self.params.get('no_warnings'):
return
else:
if self.params.get('no_warnings'):
return
- if not self.params.get('no_color') and self._err_file.isatty() and
os.
name != 'nt':
+ if not self.params.get('no_color') and self._err_file.isatty() and
compat_os_
name != 'nt':
_msg_header = '\033[0;33mWARNING:\033[0m'
else:
_msg_header = 'WARNING:'
_msg_header = '\033[0;33mWARNING:\033[0m'
else:
_msg_header = 'WARNING:'
@@
-534,7
+542,7
@@
class YoutubeDL(object):
Do the same as trouble, but prefixes the message with 'ERROR:', colored
in red if stderr is a tty file.
'''
Do the same as trouble, but prefixes the message with 'ERROR:', colored
in red if stderr is a tty file.
'''
- if not self.params.get('no_color') and self._err_file.isatty() and
os.
name != 'nt':
+ if not self.params.get('no_color') and self._err_file.isatty() and
compat_os_
name != 'nt':
_msg_header = '\033[0;31mERROR:\033[0m'
else:
_msg_header = 'ERROR:'
_msg_header = '\033[0;31mERROR:\033[0m'
else:
_msg_header = 'ERROR:'
@@
-567,7
+575,7
@@
class YoutubeDL(object):
elif template_dict.get('height'):
template_dict['resolution'] = '%sp' % template_dict['height']
elif template_dict.get('width'):
elif template_dict.get('height'):
template_dict['resolution'] = '%sp' % template_dict['height']
elif template_dict.get('width'):
- template_dict['resolution'] = '
?x%d
' % template_dict['width']
+ template_dict['resolution'] = '
%dx?
' % template_dict['width']
sanitize = lambda k, v: sanitize_filename(
compat_str(v),
sanitize = lambda k, v: sanitize_filename(
compat_str(v),
@@
-575,7
+583,7
@@
class YoutubeDL(object):
is_id=(k == 'id'))
template_dict = dict((k, sanitize(k, v))
for k, v in template_dict.items()
is_id=(k == 'id'))
template_dict = dict((k, sanitize(k, v))
for k, v in template_dict.items()
- if v is not None)
+ if v is not None
and not isinstance(v, (list, tuple, dict))
)
template_dict = collections.defaultdict(lambda: 'NA', template_dict)
outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
template_dict = collections.defaultdict(lambda: 'NA', template_dict)
outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
@@
-659,6
+667,7
@@
class YoutubeDL(object):
if not ie.suitable(url):
continue
if not ie.suitable(url):
continue
+ ie = self.get_info_extractor(ie.ie_key())
if not ie.working():
self.report_warning('The program functionality for this site has been marked as broken, '
'and will probably not work.')
if not ie.working():
self.report_warning('The program functionality for this site has been marked as broken, '
'and will probably not work.')
@@
-711,6
+720,7
@@
class YoutubeDL(object):
result_type = ie_result.get('_type', 'video')
if result_type in ('url', 'url_transparent'):
result_type = ie_result.get('_type', 'video')
if result_type in ('url', 'url_transparent'):
+ ie_result['url'] = sanitize_url(ie_result['url'])
extract_flat = self.params.get('extract_flat', False)
if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
extract_flat is True):
extract_flat = self.params.get('extract_flat', False)
if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
extract_flat is True):
@@
-904,7
+914,7
@@
class YoutubeDL(object):
'*=': lambda attr, value: value in attr,
}
str_operator_rex = re.compile(r'''(?x)
'*=': lambda attr, value: value in attr,
}
str_operator_rex = re.compile(r'''(?x)
- \s*(?P<key>ext|acodec|vcodec|container|protocol)
+ \s*(?P<key>ext|acodec|vcodec|container|protocol
|format_id
)
\s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
\s*(?P<value>[a-zA-Z0-9._-]+)
\s*$
\s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
\s*(?P<value>[a-zA-Z0-9._-]+)
\s*$
@@
-1228,12
+1238,20
@@
class YoutubeDL(object):
t.get('preference'), t.get('width'), t.get('height'),
t.get('id'), t.get('url')))
for i, t in enumerate(thumbnails):
t.get('preference'), t.get('width'), t.get('height'),
t.get('id'), t.get('url')))
for i, t in enumerate(thumbnails):
+ t['url'] = sanitize_url(t['url'])
if t.get('width') and t.get('height'):
t['resolution'] = '%dx%d' % (t['width'], t['height'])
if t.get('id') is None:
t['id'] = '%d' % i
if t.get('width') and t.get('height'):
t['resolution'] = '%dx%d' % (t['width'], t['height'])
if t.get('id') is None:
t['id'] = '%d' % i
- if thumbnails and 'thumbnail' not in info_dict:
+ if self.params.get('list_thumbnails'):
+ self.list_thumbnails(info_dict)
+ return
+
+ thumbnail = info_dict.get('thumbnail')
+ if thumbnail:
+ info_dict['thumbnail'] = sanitize_url(thumbnail)
+ elif thumbnails:
info_dict['thumbnail'] = thumbnails[-1]['url']
if 'display_id' not in info_dict and 'id' in info_dict:
info_dict['thumbnail'] = thumbnails[-1]['url']
if 'display_id' not in info_dict and 'id' in info_dict:
@@
-1258,6
+1276,8
@@
class YoutubeDL(object):
if subtitles:
for _, subtitle in subtitles.items():
for subtitle_format in subtitle:
if subtitles:
for _, subtitle in subtitles.items():
for subtitle_format in subtitle:
+ if subtitle_format.get('url'):
+ subtitle_format['url'] = sanitize_url(subtitle_format['url'])
if 'ext' not in subtitle_format:
subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
if 'ext' not in subtitle_format:
subtitle_format['ext'] = determine_ext(subtitle_format['url']).lower()
@@
-1287,6
+1307,8
@@
class YoutubeDL(object):
if 'url' not in format:
raise ExtractorError('Missing "url" key in result (index %d)' % i)
if 'url' not in format:
raise ExtractorError('Missing "url" key in result (index %d)' % i)
+ format['url'] = sanitize_url(format['url'])
+
if format.get('format_id') is None:
format['format_id'] = compat_str(i)
else:
if format.get('format_id') is None:
format['format_id'] = compat_str(i)
else:
@@
-1334,9
+1356,6
@@
class YoutubeDL(object):
if self.params.get('listformats'):
self.list_formats(info_dict)
return
if self.params.get('listformats'):
self.list_formats(info_dict)
return
- if self.params.get('list_thumbnails'):
- self.list_thumbnails(info_dict)
- return
req_format = self.params.get('format')
if req_format is None:
req_format = self.params.get('format')
if req_format is None:
@@
-1624,7
+1643,7
@@
class YoutubeDL(object):
# Just a single file
success = dl(filename, info_dict)
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
# Just a single file
success = dl(filename, info_dict)
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
- self.report_error('unable to download video data: %s' % str(err))
+ self.report_error('unable to download video data: %s' %
error_to_compat_
str(err))
return
except (OSError, IOError) as err:
raise UnavailableVideoError(err)
return
except (OSError, IOError) as err:
raise UnavailableVideoError(err)
@@
-1632,12
+1651,14
@@
class YoutubeDL(object):
self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
return
self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
return
- if success:
+ if success
and filename != '-'
:
# Fixup content
fixup_policy = self.params.get('fixup')
if fixup_policy is None:
fixup_policy = 'detect_or_warn'
# Fixup content
fixup_policy = self.params.get('fixup')
if fixup_policy is None:
fixup_policy = 'detect_or_warn'
+ INSTALL_FFMPEG_MESSAGE = 'Install ffmpeg or avconv to fix this automatically.'
+
stretched_ratio = info_dict.get('stretched_ratio')
if stretched_ratio is not None and stretched_ratio != 1:
if fixup_policy == 'warn':
stretched_ratio = info_dict.get('stretched_ratio')
if stretched_ratio is not None and stretched_ratio != 1:
if fixup_policy == 'warn':
@@
-1650,15
+1671,18
@@
class YoutubeDL(object):
info_dict['__postprocessors'].append(stretched_pp)
else:
self.report_warning(
info_dict['__postprocessors'].append(stretched_pp)
else:
self.report_warning(
- '%s: Non-uniform pixel ratio (%s).
Install ffmpeg or avconv to fix this automatically.' % (
-
info_dict['id'], stretched_ratio
))
+ '%s: Non-uniform pixel ratio (%s).
%s'
+
% (info_dict['id'], stretched_ratio, INSTALL_FFMPEG_MESSAGE
))
else:
assert fixup_policy in ('ignore', 'never')
else:
assert fixup_policy in ('ignore', 'never')
- if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
+ if (info_dict.get('requested_formats') is None and
+ info_dict.get('container') == 'm4a_dash'):
if fixup_policy == 'warn':
if fixup_policy == 'warn':
- self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
- info_dict['id']))
+ self.report_warning(
+ '%s: writing DASH m4a. '
+ 'Only some players support this container.'
+ % info_dict['id'])
elif fixup_policy == 'detect_or_warn':
fixup_pp = FFmpegFixupM4aPP(self)
if fixup_pp.available:
elif fixup_policy == 'detect_or_warn':
fixup_pp = FFmpegFixupM4aPP(self)
if fixup_pp.available:
@@
-1666,12
+1690,15
@@
class YoutubeDL(object):
info_dict['__postprocessors'].append(fixup_pp)
else:
self.report_warning(
info_dict['__postprocessors'].append(fixup_pp)
else:
self.report_warning(
- '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
- info_dict['id']))
+ '%s: writing DASH m4a. '
+ 'Only some players support this container. %s'
+ % (info_dict['id'], INSTALL_FFMPEG_MESSAGE))
else:
assert fixup_policy in ('ignore', 'never')
else:
assert fixup_policy in ('ignore', 'never')
- if info_dict.get('protocol') == 'm3u8_native' or info_dict.get('protocol') == 'm3u8' and self._downloader.params.get('hls_prefer_native', False):
+ if (info_dict.get('protocol') == 'm3u8_native' or
+ info_dict.get('protocol') == 'm3u8' and
+ self.params.get('hls_prefer_native')):
if fixup_policy == 'warn':
self.report_warning('%s: malformated aac bitstream.' % (
info_dict['id']))
if fixup_policy == 'warn':
self.report_warning('%s: malformated aac bitstream.' % (
info_dict['id']))
@@
-1682,8
+1709,8
@@
class YoutubeDL(object):
info_dict['__postprocessors'].append(fixup_pp)
else:
self.report_warning(
info_dict['__postprocessors'].append(fixup_pp)
else:
self.report_warning(
- '%s: malformated aac bitstream.
Install ffmpeg or avconv to fix this automatically.' % (
-
info_dict['id']
))
+ '%s: malformated aac bitstream.
%s'
+
% (info_dict['id'], INSTALL_FFMPEG_MESSAGE
))
else:
assert fixup_policy in ('ignore', 'never')
else:
assert fixup_policy in ('ignore', 'never')
@@
-1826,7
+1853,7
@@
class YoutubeDL(object):
if fdict.get('language'):
if res:
res += ' '
if fdict.get('language'):
if res:
res += ' '
- res += '[%s]' % fdict['language']
+ res += '[%s]
' % fdict['language']
if fdict.get('format_note') is not None:
res += fdict['format_note'] + ' '
if fdict.get('tbr') is not None:
if fdict.get('format_note') is not None:
res += fdict['format_note'] + ' '
if fdict.get('tbr') is not None:
@@
-1847,7
+1874,9
@@
class YoutubeDL(object):
if fdict.get('vbr') is not None:
res += '%4dk' % fdict['vbr']
if fdict.get('fps') is not None:
if fdict.get('vbr') is not None:
res += '%4dk' % fdict['vbr']
if fdict.get('fps') is not None:
- res += ', %sfps' % fdict['fps']
+ if res:
+ res += ', '
+ res += '%sfps' % fdict['fps']
if fdict.get('acodec') is not None:
if res:
res += ', '
if fdict.get('acodec') is not None:
if res:
res += ', '
@@
-1890,13
+1919,8
@@
class YoutubeDL(object):
def list_thumbnails(self, info_dict):
thumbnails = info_dict.get('thumbnails')
if not thumbnails:
def list_thumbnails(self, info_dict):
thumbnails = info_dict.get('thumbnails')
if not thumbnails:
- tn_url = info_dict.get('thumbnail')
- if tn_url:
- thumbnails = [{'id': '0', 'url': tn_url}]
- else:
- self.to_screen(
- '[info] No thumbnails present for %s' % info_dict['id'])
- return
+ self.to_screen('[info] No thumbnails present for %s' % info_dict['id'])
+ return
self.to_screen(
'[info] Thumbnails for %s:' % info_dict['id'])
self.to_screen(
'[info] Thumbnails for %s:' % info_dict['id'])
@@
-1941,6
+1965,8
@@
class YoutubeDL(object):
write_string(encoding_str, encoding=None)
self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
write_string(encoding_str, encoding=None)
self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
+ if _LAZY_LOADER:
+ self._write_string('[debug] Lazy loading extractors enabled' + '\n')
try:
sp = subprocess.Popen(
['git', 'rev-parse', '--short', 'HEAD'],
try:
sp = subprocess.Popen(
['git', 'rev-parse', '--short', 'HEAD'],
@@
-1996,6
+2022,7
@@
class YoutubeDL(object):
if opts_cookiefile is None:
self.cookiejar = compat_cookiejar.CookieJar()
else:
if opts_cookiefile is None:
self.cookiejar = compat_cookiejar.CookieJar()
else:
+ opts_cookiefile = compat_expanduser(opts_cookiefile)
self.cookiejar = compat_cookiejar.MozillaCookieJar(
opts_cookiefile)
if os.access(opts_cookiefile, os.R_OK):
self.cookiejar = compat_cookiejar.MozillaCookieJar(
opts_cookiefile)
if os.access(opts_cookiefile, os.R_OK):