# Author: Benjamin Johnson
# License: Public domain code
import cookielib
+import datetime
import htmlentitydefs
import httplib
import locale
from cgi import parse_qs
std_headers = {
- 'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.11) Gecko/20101019 Firefox/3.6.11',
+ 'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.12) Gecko/20101028 Firefox/3.6.12',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-us,en;q=0.5',
if not os.path.exists(dir):
os.mkdir(dir)
+ @staticmethod
+ def temp_name(filename):
+ """Returns a temporary filename for the given filename."""
+ return filename + '.part'
+
@staticmethod
def format_bytes(bytes):
if bytes is None:
speed = float(byte_counter) / elapsed
if speed > rate_limit:
time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
+
+ def try_rename(self, old_filename, new_filename):
+ try:
+ os.rename(old_filename, new_filename)
+ except (IOError, OSError), err:
+ self.trouble(u'ERROR: unable to rename file')
def report_destination(self, filename):
"""Report destination filename."""
try:
template_dict = dict(info_dict)
template_dict['epoch'] = unicode(long(time.time()))
- template_dict['ord'] = unicode('%05d' % self._num_downloads)
+ template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
filename = self.params['outtmpl'] % template_dict
except (ValueError, KeyError), err:
self.trouble(u'ERROR: invalid system charset or erroneous output template')
def _download_with_rtmpdump(self, filename, url, player_url):
self.report_destination(filename)
+ tmpfilename = self.temp_name(filename)
# Check for rtmpdump first
try:
# Download using rtmpdump. rtmpdump returns exit code 2 when
# the connection was interrumpted and resuming appears to be
# possible. This is part of rtmpdump's normal usage, AFAIK.
- basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', filename]
+ basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
while retval == 2 or retval == 1:
- prevsize = os.path.getsize(filename)
+ prevsize = os.path.getsize(tmpfilename)
self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
time.sleep(5.0) # This seems to be needed
retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
- cursize = os.path.getsize(filename)
+ cursize = os.path.getsize(tmpfilename)
if prevsize == cursize and retval == 1:
break
if retval == 0:
- self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename))
+ self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
+ self.try_rename(tmpfilename, filename)
return True
else:
self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
return False
def _do_download(self, filename, url, player_url):
+ # Check file already present
+ if self.params.get('continuedl', False) and os.path.isfile(filename):
+ self.report_file_already_downloaded(filename)
+ return True
+
# Attempt to download using rtmpdump
if url.startswith('rtmp'):
return self._download_with_rtmpdump(filename, url, player_url)
+ tmpfilename = self.temp_name(filename)
stream = None
open_mode = 'wb'
basic_request = urllib2.Request(url, None, std_headers)
request = urllib2.Request(url, None, std_headers)
# Establish possible resume length
- if os.path.isfile(filename):
- resume_len = os.path.getsize(filename)
+ if os.path.isfile(tmpfilename):
+ resume_len = os.path.getsize(tmpfilename)
else:
resume_len = 0
# completely downloaded if the file size differs less than 100 bytes from
# the one in the hard drive.
self.report_file_already_downloaded(filename)
+ self.try_rename(tmpfilename, filename)
return True
else:
# The length does not match, we start the download over
# Open file just in time
if stream is None:
try:
- (stream, filename) = sanitize_open(filename, open_mode)
+ (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
self.report_destination(filename)
except (OSError, IOError), err:
self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
self.report_finish()
if data_len is not None and str(byte_counter) != data_len:
raise ContentTooShortError(byte_counter, long(data_len))
+ self.try_rename(tmpfilename, filename)
return True
class InfoExtractor(object):
# Get video webpage
self.report_video_webpage_download(video_id)
- request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id, None, std_headers)
+ request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id, None, std_headers)
try:
video_webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
return
# Attempt to extract SWF player URL
- mobj = re.search(r'swfConfig.*"(http://.*?watch.*?-.*?\.swf)"', video_webpage)
+ mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
if mobj is not None:
- player_url = mobj.group(1)
+ player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
else:
player_url = None
else: # don't panic if we can't find it
video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
+ # upload date
+ upload_date = u'NA'
+ mobj = re.search(r'id="eow-date".*?>(.*?)</span>', video_webpage, re.DOTALL)
+ if mobj is not None:
+ upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
+ format_expressions = ['%d %B %Y', '%B %d %Y']
+ for expression in format_expressions:
+ try:
+ upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
+ except:
+ pass
+
# description
video_description = 'No description available.'
if self._downloader.params.get('forcedescription', False):
'id': video_id.decode('utf-8'),
'url': video_real_url.decode('utf-8'),
'uploader': video_uploader.decode('utf-8'),
+ 'upload_date': upload_date,
'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'id': video_id.decode('utf-8'),
'url': video_url.decode('utf-8'),
'uploader': video_uploader.decode('utf-8'),
+ 'upload_date': u'NA',
'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'id': video_id.decode('utf-8'),
'url': video_url.decode('utf-8'),
'uploader': video_uploader.decode('utf-8'),
+ 'upload_date': u'NA',
'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'id': video_id.decode('utf-8'),
'url': video_url.decode('utf-8'),
'uploader': u'NA',
+ 'upload_date': u'NA',
'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'id': video_id.decode('utf-8'),
'url': video_url.decode('utf-8'),
'uploader': video_uploader,
+ 'upload_date': u'NA',
'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'id': video_id.decode('utf-8'),
'url': video_url,
'uploader': video_uploader,
+ 'upload_date': u'NA',
'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
return
+ self.report_extraction(video_id)
# Start with something easy: JW Player in SWFObject
mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
if mobj is None:
'id': video_id.decode('utf-8'),
'url': video_url.decode('utf-8'),
'uploader': video_uploader,
+ 'upload_date': u'NA',
'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
# Parse command line
parser = optparse.OptionParser(
usage='Usage: %prog [options] url...',
- version='2010.10.24',
+ version='2010.11.19',
conflict_handler='resolve',
)
action='store_true', dest='usetitle', help='use title in file name', default=False)
filesystem.add_option('-l', '--literal',
action='store_true', dest='useliteral', help='use literal title in file name', default=False)
+ filesystem.add_option('-A', '--auto-number',
+ action='store_true', dest='autonumber', help='number downloaded files starting from 00000', default=False)
filesystem.add_option('-o', '--output',
dest='outtmpl', metavar='TEMPLATE', help='output filename template')
filesystem.add_option('-a', '--batch-file',
batchfd = open(opts.batchfile, 'r')
batchurls = batchfd.readlines()
batchurls = [x.strip() for x in batchurls]
- batchurls = [x for x in batchurls if len(x) > 0]
+ batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
except IOError:
sys.exit(u'ERROR: batch file could not be read')
all_urls = batchurls + args
parser.error(u'using .netrc conflicts with giving username/password')
if opts.password is not None and opts.username is None:
parser.error(u'account username missing')
- if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
- parser.error(u'using output template conflicts with using title or literal title')
+ if opts.outtmpl is not None and (opts.useliteral or opts.usetitle or opts.autonumber):
+ parser.error(u'using output template conflicts with using title, literal title or auto number')
if opts.usetitle and opts.useliteral:
parser.error(u'using title conflicts with using literal title')
if opts.username is not None and opts.password is None:
or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
+ or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(stitle)s-%(id)s.%(ext)s')
+ or (opts.useliteral and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
+ or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
or u'%(id)s.%(ext)s'),
'ignoreerrors': opts.ignoreerrors,
'ratelimit': opts.ratelimit,
projects / youtube-dl / blobdiff