projects
/
youtube-dl
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
correct to_stderr
[youtube-dl]
/
youtube_dl
/
FileDownloader.py
diff --git
a/youtube_dl/FileDownloader.py
b/youtube_dl/FileDownloader.py
index d7d5b1521ec72ea46782064f3c6fadd3d5032cb7..2b76a3b09b16b9bf5bcd4c37e22fb32ccc1aae71 100644
(file)
--- a/
youtube_dl/FileDownloader.py
+++ b/
youtube_dl/FileDownloader.py
@@
-1,7
+1,6
@@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import httplib
import math
import os
import re
import math
import os
import re
@@
-9,7
+8,6
@@
import socket
import subprocess
import sys
import time
import subprocess
import sys
import time
-import urllib2
if os.name == 'nt':
import ctypes
if os.name == 'nt':
import ctypes
@@
-108,7
+106,7
@@
class FileDownloader(object):
if bytes == 0.0:
exponent = 0
else:
if bytes == 0.0:
exponent = 0
else:
- exponent =
long
(math.log(bytes, 1024.0))
+ exponent =
int
(math.log(bytes, 1024.0))
suffix = 'bkMGTPEZY'[exponent]
converted = float(bytes) / float(1024 ** exponent)
return '%.2f%s' % (converted, suffix)
suffix = 'bkMGTPEZY'[exponent]
converted = float(bytes) / float(1024 ** exponent)
return '%.2f%s' % (converted, suffix)
@@
-127,7
+125,7
@@
class FileDownloader(object):
if current == 0 or dif < 0.001: # One millisecond
return '--:--'
rate = float(current) / dif
if current == 0 or dif < 0.001: # One millisecond
return '--:--'
rate = float(current) / dif
- eta =
long
((float(total) - float(current)) / rate)
+ eta =
int
((float(total) - float(current)) / rate)
(eta_mins, eta_secs) = divmod(eta, 60)
if eta_mins > 99:
return '--:--'
(eta_mins, eta_secs) = divmod(eta, 60)
if eta_mins > 99:
return '--:--'
@@
-179,7
+177,7
@@
class FileDownloader(object):
if not self.params.get('quiet', False):
terminator = [u'\n', u''][skip_eol]
output = message + terminator
if not self.params.get('quiet', False):
terminator = [u'\n', u''][skip_eol]
output = message + terminator
- if 'b'
not in self._screen_file.mode
or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
+ if 'b'
in getattr(self._screen_file, 'mode', '')
or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
output = output.encode(preferredencoding(), 'ignore')
self._screen_file.write(output)
self._screen_file.flush()
output = output.encode(preferredencoding(), 'ignore')
self._screen_file.write(output)
self._screen_file.flush()
@@
-187,7
+185,10
@@
class FileDownloader(object):
def to_stderr(self, message):
"""Print message to stderr."""
assert type(message) == type(u'')
def to_stderr(self, message):
"""Print message to stderr."""
assert type(message) == type(u'')
- sys.stderr.write((message + u'\n').encode(preferredencoding()))
+ output = message + u'\n'
+ if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
+ output = output.encode(preferredencoding())
+ sys.stderr.write(output)
def to_cons_title(self, message):
"""Set console/terminal window title to message."""
def to_cons_title(self, message):
"""Set console/terminal window title to message."""
@@
-202,7
+203,7
@@
class FileDownloader(object):
def fixed_template(self):
"""Checks if the output template is fixed."""
def fixed_template(self):
"""Checks if the output template is fixed."""
- return (re.search(u
r'(?u)%\(.+?
\)s', self.params['outtmpl']) is None)
+ return (re.search(u
'(?u)%\\(.+?\
\)s', self.params['outtmpl']) is None)
def trouble(self, message=None):
"""Determine action to take when a download problem appears.
def trouble(self, message=None):
"""Determine action to take when a download problem appears.
@@
-247,7
+248,7
@@
class FileDownloader(object):
if old_filename == new_filename:
return
os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
if old_filename == new_filename:
return
os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
- except (IOError, OSError)
,
err:
+ except (IOError, OSError)
as
err:
self.trouble(u'ERROR: unable to rename file')
def try_utime(self, filename, last_modified_hdr):
self.trouble(u'ERROR: unable to rename file')
def try_utime(self, filename, last_modified_hdr):
@@
-305,7
+306,7
@@
class FileDownloader(object):
"""Report file has already been fully downloaded."""
try:
self.to_screen(u'[download] %s has already been downloaded' % file_name)
"""Report file has already been fully downloaded."""
try:
self.to_screen(u'[download] %s has already been downloaded' % file_name)
- except (UnicodeEncodeError)
,
err:
+ except (UnicodeEncodeError)
as
err:
self.to_screen(u'[download] The file has already been downloaded')
def report_unable_to_resume(self):
self.to_screen(u'[download] The file has already been downloaded')
def report_unable_to_resume(self):
@@
-327,13
+328,16
@@
class FileDownloader(object):
"""Generate the output filename."""
try:
template_dict = dict(info_dict)
"""Generate the output filename."""
try:
template_dict = dict(info_dict)
+
template_dict['epoch'] = int(time.time())
template_dict['autonumber'] = u'%05d' % self._num_downloads
template_dict['epoch'] = int(time.time())
template_dict['autonumber'] = u'%05d' % self._num_downloads
+ template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items())
template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
+
filename = self.params['outtmpl'] % template_dict
return filename
filename = self.params['outtmpl'] % template_dict
return filename
- except (ValueError, KeyError)
,
err:
+ except (ValueError, KeyError)
as
err:
self.trouble(u'ERROR: invalid system charset or erroneous output template')
return None
self.trouble(u'ERROR: invalid system charset or erroneous output template')
return None
@@
-359,6
+363,9
@@
class FileDownloader(object):
# Keep for backwards compatibility
info_dict['stitle'] = info_dict['title']
# Keep for backwards compatibility
info_dict['stitle'] = info_dict['title']
+ if not 'format' in info_dict:
+ info_dict['format'] = info_dict['ext']
+
reason = self._match_entry(info_dict)
if reason is not None:
self.to_screen(u'[download] ' + reason)
reason = self._match_entry(info_dict)
if reason is not None:
self.to_screen(u'[download] ' + reason)
@@
-396,8
+403,8
@@
class FileDownloader(object):
dn = os.path.dirname(encodeFilename(filename))
if dn != '' and not os.path.exists(dn): # dn is already encoded
os.makedirs(dn)
dn = os.path.dirname(encodeFilename(filename))
if dn != '' and not os.path.exists(dn): # dn is already encoded
os.makedirs(dn)
- except (OSError, IOError)
,
err:
- self.trouble(u'ERROR: unable to create directory ' +
unicode
(err))
+ except (OSError, IOError)
as
err:
+ self.trouble(u'ERROR: unable to create directory ' +
compat_str
(err))
return
if self.params.get('writedescription', False):
return
if self.params.get('writedescription', False):
@@
-453,19
+460,19
@@
class FileDownloader(object):
else:
try:
success = self._do_download(filename, info_dict)
else:
try:
success = self._do_download(filename, info_dict)
- except (OSError, IOError)
,
err:
+ except (OSError, IOError)
as
err:
raise UnavailableVideoError
raise UnavailableVideoError
- except (
urllib2.URLError, httplib.HTTPException, socket.error),
err:
+ except (
compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as
err:
self.trouble(u'ERROR: unable to download video data: %s' % str(err))
return
self.trouble(u'ERROR: unable to download video data: %s' % str(err))
return
- except (ContentTooShortError, )
,
err:
+ except (ContentTooShortError, )
as
err:
self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
return
if success:
try:
self.post_process(filename, info_dict)
self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
return
if success:
try:
self.post_process(filename, info_dict)
- except (PostProcessingError)
,
err:
+ except (PostProcessingError)
as
err:
self.trouble(u'ERROR: postprocessing: %s' % str(err))
return
self.trouble(u'ERROR: postprocessing: %s' % str(err))
return
@@
-481,6
+488,11
@@
class FileDownloader(object):
if not ie.suitable(url):
continue
if not ie.suitable(url):
continue
+ # Warn if the _WORKING attribute is False
+ if not ie.working():
+ self.trouble(u'WARNING: the program functionality for this site has been marked as broken, '
+ u'and will probably not work. If you want to go on, use the -i option.')
+
# Suitable InfoExtractor found
suitable_found = True
# Suitable InfoExtractor found
suitable_found = True
@@
-574,8
+586,8
@@
class FileDownloader(object):
# Do not include the Accept-Encoding header
headers = {'Youtubedl-no-compression': 'True'}
# Do not include the Accept-Encoding header
headers = {'Youtubedl-no-compression': 'True'}
- basic_request =
urllib2
.Request(url, None, headers)
- request =
urllib2
.Request(url, None, headers)
+ basic_request =
compat_urllib_request
.Request(url, None, headers)
+ request =
compat_urllib_request
.Request(url, None, headers)
# Establish possible resume length
if os.path.isfile(encodeFilename(tmpfilename)):
# Establish possible resume length
if os.path.isfile(encodeFilename(tmpfilename)):
@@
-599,9
+611,9
@@
class FileDownloader(object):
try:
if count == 0 and 'urlhandle' in info_dict:
data = info_dict['urlhandle']
try:
if count == 0 and 'urlhandle' in info_dict:
data = info_dict['urlhandle']
- data =
urllib2
.urlopen(request)
+ data =
compat_urllib_request
.urlopen(request)
break
break
- except (
urllib2.HTTPError, ),
err:
+ except (
compat_urllib_error.HTTPError, ) as
err:
if (err.code < 500 or err.code >= 600) and err.code != 416:
# Unexpected HTTP error
raise
if (err.code < 500 or err.code >= 600) and err.code != 416:
# Unexpected HTTP error
raise
@@
-609,15
+621,15
@@
class FileDownloader(object):
# Unable to resume (requested range not satisfiable)
try:
# Open the connection again without the range header
# Unable to resume (requested range not satisfiable)
try:
# Open the connection again without the range header
- data =
urllib2
.urlopen(basic_request)
+ data =
compat_urllib_request
.urlopen(basic_request)
content_length = data.info()['Content-Length']
content_length = data.info()['Content-Length']
- except (
urllib2.HTTPError, ),
err:
+ except (
compat_urllib_error.HTTPError, ) as
err:
if err.code < 500 or err.code >= 600:
raise
else:
# Examine the reported length
if (content_length is not None and
if err.code < 500 or err.code >= 600:
raise
else:
# Examine the reported length
if (content_length is not None and
- (resume_len - 100 <
long
(content_length) < resume_len + 100)):
+ (resume_len - 100 <
int
(content_length) < resume_len + 100)):
# The file had already been fully downloaded.
# Explanation to the above condition: in issue #175 it was revealed that
# YouTube sometimes adds or removes a few bytes from the end of the file,
# The file had already been fully downloaded.
# Explanation to the above condition: in issue #175 it was revealed that
# YouTube sometimes adds or removes a few bytes from the end of the file,
@@
-644,7
+656,7
@@
class FileDownloader(object):
data_len = data.info().get('Content-length', None)
if data_len is not None:
data_len = data.info().get('Content-length', None)
if data_len is not None:
- data_len =
long
(data_len) + resume_len
+ data_len =
int
(data_len) + resume_len
data_len_str = self.format_bytes(data_len)
byte_counter = 0 + resume_len
block_size = self.params.get('buffersize', 1024)
data_len_str = self.format_bytes(data_len)
byte_counter = 0 + resume_len
block_size = self.params.get('buffersize', 1024)
@@
-665,12
+677,12
@@
class FileDownloader(object):
assert stream is not None
filename = self.undo_temp_name(tmpfilename)
self.report_destination(filename)
assert stream is not None
filename = self.undo_temp_name(tmpfilename)
self.report_destination(filename)
- except (OSError, IOError)
,
err:
+ except (OSError, IOError)
as
err:
self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
return False
try:
stream.write(data_block)
self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
return False
try:
stream.write(data_block)
- except (IOError, OSError)
,
err:
+ except (IOError, OSError)
as
err:
self.trouble(u'\nERROR: unable to write data: %s' % str(err))
return False
if not self.params.get('noresizebuffer', False):
self.trouble(u'\nERROR: unable to write data: %s' % str(err))
return False
if not self.params.get('noresizebuffer', False):
@@
-694,7
+706,7
@@
class FileDownloader(object):
stream.close()
self.report_finish()
if data_len is not None and byte_counter != data_len:
stream.close()
self.report_finish()
if data_len is not None and byte_counter != data_len:
- raise ContentTooShortError(byte_counter,
long
(data_len))
+ raise ContentTooShortError(byte_counter,
int
(data_len))
self.try_rename(tmpfilename, filename)
# Update file modification time
self.try_rename(tmpfilename, filename)
# Update file modification time