X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=youtube-dl;h=aa16ff1e4ced11af45527fdf602a85c9a914f37f;hb=110cd3462e4057ccbec5369bf895bdf311fb536b;hp=9d84c3560b5131d5aca4800f59a38c5714b40732;hpb=daa88ccc2e6ac9c184baa62ec411eb13180a00f8;p=youtube-dl diff --git a/youtube-dl b/youtube-dl index 9d84c3560..aa16ff1e4 100755 --- a/youtube-dl +++ b/youtube-dl @@ -19,7 +19,7 @@ import urllib import urllib2 std_headers = { - 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8', + 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5', 'Accept-Language': 'en-us,en;q=0.5', @@ -114,6 +114,7 @@ class FileDownloader(object): ignoreerrors: Do not stop on download errors. ratelimit: Download speed limit, in bytes/sec. nooverwrites: Prevent overwriting files. + continuedl: Try to continue downloads if possible. """ params = None @@ -182,13 +183,13 @@ class FileDownloader(object): new_min = max(bytes / 2.0, 1.0) new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB if elapsed_time < 0.001: - return int(new_max) + return long(new_max) rate = bytes / elapsed_time if rate > new_max: - return int(new_max) + return long(new_max) if rate < new_min: - return int(new_min) - return int(rate) + return long(new_min) + return long(rate) @staticmethod def parse_bytes(bytestr): @@ -226,7 +227,7 @@ class FileDownloader(object): def to_stderr(self, message): """Print message to stderr.""" - print >>sys.stderr, message + print >>sys.stderr, message.encode(locale.getpreferredencoding()) def fixed_template(self): """Checks if the output template is fixed.""" @@ -266,6 +267,18 @@ class FileDownloader(object): """Report download progress.""" self.to_stdout(u'\r[download] %s of %s at %s ETA %s' % (percent_str, data_len_str, speed_str, eta_str), skip_eol=True) + + def report_resuming_byte(self, resume_len): + """Report attemtp to resume at given byte.""" + self.to_stdout(u'[download] Resuming download at byte %s' % resume_len) + + def report_file_already_downloaded(self, file_name): + """Report file has already been fully downloaded.""" + self.to_stdout(u'[download] %s has already been downloaded' % file_name) + + def report_unable_to_resume(self): + """Report it was impossible to resume download.""" + self.to_stdout(u'[download] Unable to resume') def report_finish(self): """Report download finished.""" @@ -292,11 +305,10 @@ class FileDownloader(object): template_dict = dict(info_dict) template_dict['epoch'] = unicode(long(time.time())) filename = self.params['outtmpl'] % template_dict - self.report_destination(filename) except (ValueError, KeyError), err: self.trouble('ERROR: invalid output template or system charset: %s' % str(err)) if self.params['nooverwrites'] and os.path.exists(filename): - self.to_stderr('WARNING: file exists: %s; skipping' % filename) + self.to_stderr(u'WARNING: file exists: %s; skipping' % filename) return try: @@ -306,17 +318,8 @@ class FileDownloader(object): return try: - outstream = open(filename, 'wb') + success = self._do_download(filename, info_dict['url']) except (OSError, IOError), err: - self.trouble('ERROR: unable to open for writing: %s' % str(err)) - return - - try: - self._do_download(outstream, info_dict['url']) - outstream.close() - except (OSError, IOError), err: - outstream.close() - os.remove(filename) raise UnavailableFormatError except (urllib2.URLError, httplib.HTTPException, socket.error), err: self.trouble('ERROR: unable to download video data: %s' % str(err)) @@ -325,11 +328,12 @@ class FileDownloader(object): self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) return - try: - self.post_process(filename, info_dict) - except (PostProcessingError), err: - self.trouble('ERROR: postprocessing: %s' % str(err)) - return + if success: + try: + self.post_process(filename, info_dict) + except (PostProcessingError), err: + self.trouble('ERROR: postprocessing: %s' % str(err)) + return def download(self, url_list): """Download a given list of URLs.""" @@ -366,21 +370,43 @@ class FileDownloader(object): if info is None: break - def _do_download(self, stream, url): + def _do_download(self, filename, url): + stream = None + open_mode = 'ab' + + basic_request = urllib2.Request(url, None, std_headers) request = urllib2.Request(url, None, std_headers) - data = urllib2.urlopen(request) + + # Attempt to resume download with "continuedl" option + if os.path.isfile(filename): + resume_len = os.path.getsize(filename) + else: + resume_len = 0 + if self.params['continuedl'] and resume_len != 0: + self.report_resuming_byte(resume_len) + request.add_header('Range','bytes=%d-' % resume_len) + + # Establish connection + try: + data = urllib2.urlopen(request) + except (urllib2.HTTPError, ), err: + if err.code != 416: # 416 is 'Requested range not satisfiable' + raise + data = urllib2.urlopen(basic_request) + content_length = data.info()['Content-Length'] + if content_length is not None and long(content_length) == resume_len: + self.report_file_already_downloaded(filename) + return True + else: + self.report_unable_to_resume() + open_mode = 'wb' + data_len = data.info().get('Content-length', None) data_len_str = self.format_bytes(data_len) byte_counter = 0 block_size = 1024 start = time.time() while True: - # Progress message - percent_str = self.calc_percent(byte_counter, data_len) - eta_str = self.calc_eta(start, time.time(), data_len, byte_counter) - speed_str = self.calc_speed(start, time.time(), byte_counter) - self.report_progress(percent_str, data_len_str, speed_str, eta_str) - # Download and write before = time.time() data_block = data.read(block_size) @@ -389,15 +415,31 @@ class FileDownloader(object): if data_block_len == 0: break byte_counter += data_block_len + + # Open file just in time + if stream is None: + try: + stream = open(filename, open_mode) + self.report_destination(filename) + except (OSError, IOError), err: + self.trouble('ERROR: unable to open for writing: %s' % str(err)) + return False stream.write(data_block) block_size = self.best_block_size(after - before, data_block_len) + # Progress message + percent_str = self.calc_percent(byte_counter, data_len) + eta_str = self.calc_eta(start, time.time(), data_len, byte_counter) + speed_str = self.calc_speed(start, time.time(), byte_counter) + self.report_progress(percent_str, data_len_str, speed_str, eta_str) + # Apply rate limit self.slow_down(start, byte_counter) self.report_finish() if data_len is not None and str(byte_counter) != data_len: raise ContentTooShortError(byte_counter, long(data_len)) + return True class InfoExtractor(object): """Information Extractor class. @@ -468,7 +510,7 @@ class YoutubeIE(InfoExtractor): _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en' _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en' _NETRC_MACHINE = 'youtube' - _available_formats = ['22', '35', '18', '17', '13'] # listed in order of priority for -b flag + _available_formats = ['22', '35', '18', '5', '17', '13'] # listed in order of priority for -b flag _video_extensions = { '13': '3gp', '17': 'mp4', @@ -779,19 +821,21 @@ class MetacafeIE(InfoExtractor): # Extract URL, uploader and title from webpage self.report_extraction(video_id) - mobj = re.search(r'(?m)&mediaURL=(http.*?\.flv)', webpage) + mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract media URL') return mediaURL = urllib.unquote(mobj.group(1)) - mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage) - if mobj is None: - self._downloader.trouble(u'ERROR: unable to extract gdaKey') - return - gdaKey = mobj.group(1) + #mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage) + #if mobj is None: + # self._downloader.trouble(u'ERROR: unable to extract gdaKey') + # return + #gdaKey = mobj.group(1) + # + #video_url = '%s?__gda__=%s' % (mediaURL, gdaKey) - video_url = '%s?__gda__=%s' % (mediaURL, gdaKey) + video_url = mediaURL mobj = re.search(r'(?im)(.*) - Video', webpage) if mobj is None: @@ -859,7 +903,7 @@ class YoutubeSearchIE(InfoExtractor): return else: try: - n = int(prefix) + n = long(prefix) if n <= 0: self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) return @@ -868,7 +912,7 @@ class YoutubeSearchIE(InfoExtractor): n = self._max_youtube_results self._download_n_results(query, n) return - except ValueError: # parsing prefix as int fails + except ValueError: # parsing prefix as integer fails self._download_n_results(query, 1) return @@ -914,7 +958,7 @@ class YoutubePlaylistIE(InfoExtractor): _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/view_play_list\?p=(.+)' _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en' _VIDEO_INDICATOR = r'/watch\?v=(.+?)&' - _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s' + _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s' _youtube_ie = None def __init__(self, youtube_ie, downloader=None): @@ -960,7 +1004,7 @@ class YoutubePlaylistIE(InfoExtractor): ids_in_page.append(mobj.group(1)) video_ids.extend(ids_in_page) - if (self._MORE_PAGES_INDICATOR % (playlist_id, pagenum + 1)) not in page: + if (self._MORE_PAGES_INDICATOR % (playlist_id.upper(), pagenum + 1)) not in page: break pagenum = pagenum + 1 @@ -1029,7 +1073,7 @@ if __name__ == '__main__': # Parse command line parser = optparse.OptionParser( usage='Usage: %prog [options] url...', - version='INTERNAL', + version='2009.06.29', conflict_handler='resolve', ) @@ -1084,6 +1128,8 @@ if __name__ == '__main__': dest='batchfile', metavar='F', help='file containing URLs to download') filesystem.add_option('-w', '--no-overwrites', action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) + filesystem.add_option('-c', '--continue', + action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False) parser.add_option_group(filesystem) (opts, args) = parser.parse_args() @@ -1141,6 +1187,7 @@ if __name__ == '__main__': 'ignoreerrors': opts.ignoreerrors, 'ratelimit': opts.ratelimit, 'nooverwrites': opts.nooverwrites, + 'continuedl': opts.continue_dl, }) fd.add_info_extractor(youtube_search_ie) fd.add_info_extractor(youtube_pl_ie)