projects
/
youtube-dl
/ commitdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
| commitdiff |
tree
raw
|
patch
|
inline
| side by side (parent:
7c360e3
)
[http] Improve download mechanism when Range HTTP header is ignored
author
Sergey M․
<dstftw@gmail.com>
Sun, 27 Apr 2014 02:32:01 +0000
(09:32 +0700)
committer
Sergey M․
<dstftw@gmail.com>
Sun, 27 Apr 2014 02:32:01 +0000
(09:32 +0700)
youtube_dl/downloader/http.py
patch
|
blob
|
history
diff --git
a/youtube_dl/downloader/http.py
b/youtube_dl/downloader/http.py
index cc8b9c9a7f5cac66a4b59d460c7678917aa98cc0..7e1c1d7156fa9a5986dcdb7d41f4c19ab9c790b4 100644
(file)
--- a/
youtube_dl/downloader/http.py
+++ b/
youtube_dl/downloader/http.py
@@
-14,6
+14,8
@@
from ..utils import (
class HttpFD(FileDownloader):
class HttpFD(FileDownloader):
+ _TEST_FILE_SIZE = 10241
+
def real_download(self, filename, info_dict):
url = info_dict['url']
tmpfilename = self.temp_name(filename)
def real_download(self, filename, info_dict):
url = info_dict['url']
tmpfilename = self.temp_name(filename)
@@
-28,8
+30,10
@@
class HttpFD(FileDownloader):
basic_request = compat_urllib_request.Request(url, None, headers)
request = compat_urllib_request.Request(url, None, headers)
basic_request = compat_urllib_request.Request(url, None, headers)
request = compat_urllib_request.Request(url, None, headers)
- if self.params.get('test', False):
- request.add_header('Range', 'bytes=0-10240')
+ is_test = self.params.get('test', False)
+
+ if is_test:
+ request.add_header('Range', 'bytes=0-%s' % str(self._TEST_FILE_SIZE - 1))
# Establish possible resume length
if os.path.isfile(encodeFilename(tmpfilename)):
# Establish possible resume length
if os.path.isfile(encodeFilename(tmpfilename)):
@@
-100,6
+104,15
@@
class HttpFD(FileDownloader):
return False
data_len = data.info().get('Content-length', None)
return False
data_len = data.info().get('Content-length', None)
+
+ # Range HTTP header may be ignored/unsupported by a webserver
+ # (e.g. extractor/scivee.py, extractor/bambuser.py).
+ # However, for a test we still would like to download just a piece of a file.
+ # To achieve this we limit data_len to _TEST_FILE_SIZE and manually control
+ # block size when downloading a file.
+ if is_test and data_len > self._TEST_FILE_SIZE:
+ data_len = self._TEST_FILE_SIZE
+
if data_len is not None:
data_len = int(data_len) + resume_len
min_data_len = self.params.get("min_filesize", None)
if data_len is not None:
data_len = int(data_len) + resume_len
min_data_len = self.params.get("min_filesize", None)
@@
-118,7
+131,7
@@
class HttpFD(FileDownloader):
while True:
# Download and write
before = time.time()
while True:
# Download and write
before = time.time()
- data_block = data.read(block_size)
+ data_block = data.read(block_size
if not is_test else min(block_size, data_len - byte_counter)
)
after = time.time()
if len(data_block) == 0:
break
after = time.time()
if len(data_block) == 0:
break
@@
-162,6
+175,9
@@
class HttpFD(FileDownloader):
'speed': speed,
})
'speed': speed,
})
+ if is_test and byte_counter == data_len:
+ break
+
# Apply rate limit
self.slow_down(start, byte_counter - resume_len)
# Apply rate limit
self.slow_down(start, byte_counter - resume_len)