#!/usr/bin/env python
# -*- coding: utf-8 -*-
-__author__ = (
+__authors__ = (
'Ricardo Garcia Gonzalez',
'Danny Colligan',
'Benjamin Johnson',
'Sören Schulze',
'Kevin Ngo',
'Ori Avtalion',
+ 'shizeeg',
)
__license__ = 'Public Domain'
-__version__ = '2011.11.22'
+__version__ = '2012.01.05'
UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
import msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
return (sys.stdout, filename)
- stream = open(filename, open_mode)
+ stream = open(_encodeFilename(filename), open_mode)
return (stream, filename)
except (IOError, OSError), err:
# In case of error, try to remove win32 forbidden chars
filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
# An exception here should be caught in the caller
- stream = open(filename, open_mode)
+ stream = open(_encodeFilename(filename), open_mode)
return (stream, filename)
expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE)
return expr.sub(u'_', title).strip(u'_')
+def _orderedSet(iterable):
+ """ Remove all duplicates from the input iterable """
+ res = []
+ for el in iterable:
+ if el not in res:
+ res.append(el)
+ return res
+
+def _unescapeHTML(s):
+ """
+ @param s a string (of type unicode)
+ """
+ assert type(s) == type(u'')
+
+ htmlParser = HTMLParser.HTMLParser()
+ return htmlParser.unescape(s)
+
+def _encodeFilename(s):
+ """
+ @param s The name of the file (of type unicode)
+ """
+
+ assert type(s) == type(u'')
+ return s.encode(sys.getfilesystemencoding(), 'ignore')
+
class DownloadError(Exception):
"""Download Error exception.
"""
pass
+class MaxDownloadsReached(Exception):
+ """ --max-downloads limit has been reached. """
+ pass
+
class UnavailableVideoError(Exception):
"""Unavailable Format exception.
self._pps.append(pp)
pp.set_downloader(self)
- def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False):
+ def to_screen(self, message, skip_eol=False):
"""Print message to stdout if not in quiet mode."""
- try:
- if not self.params.get('quiet', False):
- terminator = [u'\n', u''][skip_eol]
- print >>self._screen_file, (u'%s%s' % (message, terminator)).encode(preferredencoding()),
+ assert type(message) == type(u'')
+ if not self.params.get('quiet', False):
+ terminator = [u'\n', u''][skip_eol]
+ output = message + terminator
+
+ if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
+ output = output.encode(preferredencoding(), 'ignore')
+ self._screen_file.write(output)
self._screen_file.flush()
- except (UnicodeEncodeError), err:
- if not ignore_encoding_errors:
- raise
def to_stderr(self, message):
"""Print message to stderr."""
def temp_name(self, filename):
"""Returns a temporary filename for the given filename."""
if self.params.get('nopart', False) or filename == u'-' or \
- (os.path.exists(filename) and not os.path.isfile(filename)):
+ (os.path.exists(_encodeFilename(filename)) and not os.path.isfile(_encodeFilename(filename))):
return filename
return filename + u'.part'
try:
if old_filename == new_filename:
return
- os.rename(old_filename, new_filename)
+ os.rename(_encodeFilename(old_filename), _encodeFilename(new_filename))
except (IOError, OSError), err:
self.trouble(u'ERROR: unable to rename file')
"""Try to set the last-modified time of the given file."""
if last_modified_hdr is None:
return
- if not os.path.isfile(filename):
+ if not os.path.isfile(_encodeFilename(filename)):
return
timestr = last_modified_hdr
if timestr is None:
def report_writedescription(self, descfn):
""" Report that the description file is being written """
- self.to_screen(u'[info] Writing video description to: %s' % descfn, ignore_encoding_errors=True)
+ self.to_screen(u'[info] Writing video description to: ' + descfn)
def report_writeinfojson(self, infofn):
""" Report that the metadata file has been written """
- self.to_screen(u'[info] Video description metadata as JSON to: %s' % infofn, ignore_encoding_errors=True)
+ self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn)
def report_destination(self, filename):
"""Report destination filename."""
- self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
+ self.to_screen(u'[download] Destination: ' + filename)
def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
"""Report download progress."""
self.trouble(u'ERROR: invalid system charset or erroneous output template')
return None
+ def _match_entry(self, info_dict):
+ """ Returns None iff the file should be downloaded """
+
+ title = info_dict['title']
+ matchtitle = self.params.get('matchtitle', False)
+ if matchtitle and not re.search(matchtitle, title, re.IGNORECASE):
+ return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"'
+ rejecttitle = self.params.get('rejecttitle', False)
+ if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE):
+ return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"'
+ return None
+
def process_info(self, info_dict):
"""Process a single dictionary returned by an InfoExtractor."""
+
+ reason = self._match_entry(info_dict)
+ if reason is not None:
+ self.to_screen(u'[download] ' + reason)
+ return
+
+ max_downloads = self.params.get('max_downloads')
+ if max_downloads is not None:
+ if self._num_downloads > int(max_downloads):
+ raise MaxDownloadsReached()
+
filename = self.prepare_filename(info_dict)
# Forced printings
if filename is None:
return
- matchtitle=self.params.get('matchtitle',False)
- rejecttitle=self.params.get('rejecttitle',False)
- title=info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
- if matchtitle and not re.search(matchtitle, title, re.IGNORECASE):
- self.to_screen(u'[download] "%s" title did not match pattern "%s"' % (title, matchtitle))
- return
- if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE):
- self.to_screen(u'[download] "%s" title matched reject pattern "%s"' % (title, rejecttitle))
- return
-
- if self.params.get('nooverwrites', False) and os.path.exists(filename):
- self.to_stderr(u'WARNING: file exists and will be skipped')
- return
-
try:
- dn = os.path.dirname(filename)
- if dn != '' and not os.path.exists(dn):
+ dn = os.path.dirname(_encodeFilename(filename))
+ if dn != '' and not os.path.exists(dn): # dn is already encoded
os.makedirs(dn)
except (OSError, IOError), err:
self.trouble(u'ERROR: unable to create directory ' + unicode(err))
if self.params.get('writedescription', False):
try:
- descfn = filename + '.description'
+ descfn = filename + u'.description'
self.report_writedescription(descfn)
- descfile = open(descfn, 'wb')
+ descfile = open(_encodeFilename(descfn), 'wb')
try:
descfile.write(info_dict['description'].encode('utf-8'))
finally:
return
if self.params.get('writeinfojson', False):
- infofn = filename + '.info.json'
+ infofn = filename + u'.info.json'
self.report_writeinfojson(infofn)
try:
json.dump
self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
return
try:
- infof = open(infofn, 'wb')
+ infof = open(_encodeFilename(infofn), 'wb')
try:
json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
json.dump(json_info_dict, infof)
return
if not self.params.get('skip_download', False):
- try:
- success = self._do_download(filename, info_dict)
- except (OSError, IOError), err:
- raise UnavailableVideoError
- except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self.trouble(u'ERROR: unable to download video data: %s' % str(err))
- return
- except (ContentTooShortError, ), err:
- self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
- return
+ if self.params.get('nooverwrites', False) and os.path.exists(_encodeFilename(filename)):
+ success = True
+ else:
+ try:
+ success = self._do_download(filename, info_dict)
+ except (OSError, IOError), err:
+ raise UnavailableVideoError
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self.trouble(u'ERROR: unable to download video data: %s' % str(err))
+ return
+ except (ContentTooShortError, ), err:
+ self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+ return
if success:
try:
basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
while retval == 2 or retval == 1:
- prevsize = os.path.getsize(tmpfilename)
+ prevsize = os.path.getsize(_encodeFilename(tmpfilename))
self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
time.sleep(5.0) # This seems to be needed
retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
- cursize = os.path.getsize(tmpfilename)
+ cursize = os.path.getsize(_encodeFilename(tmpfilename))
if prevsize == cursize and retval == 1:
break
# Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
retval = 0
break
if retval == 0:
- self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
+ self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(_encodeFilename(tmpfilename)))
self.try_rename(tmpfilename, filename)
return True
else:
player_url = info_dict.get('player_url', None)
# Check file already present
- if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
+ if self.params.get('continuedl', False) and os.path.isfile(_encodeFilename(filename)) and not self.params.get('nopart', False):
self.report_file_already_downloaded(filename)
return True
request = urllib2.Request(url, None, headers)
# Establish possible resume length
- if os.path.isfile(tmpfilename):
- resume_len = os.path.getsize(tmpfilename)
+ if os.path.isfile(_encodeFilename(tmpfilename)):
+ resume_len = os.path.getsize(_encodeFilename(tmpfilename))
else:
resume_len = 0
_NETRC_MACHINE = 'youtube'
# Listed in order of quality
_available_formats = ['38', '37', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
+ _available_formats_prefer_free = ['38', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
_video_extensions = {
'13': '3gp',
'17': 'mp4',
url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data)
format_limit = self._downloader.params.get('format_limit', None)
- if format_limit is not None and format_limit in self._available_formats:
- format_list = self._available_formats[self._available_formats.index(format_limit):]
+ available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
+ if format_limit is not None and format_limit in available_formats:
+ format_list = available_formats[available_formats.index(format_limit):]
else:
- format_list = self._available_formats
+ format_list = available_formats
existing_formats = [x for x in format_list if x in url_map]
if len(existing_formats) == 0:
self._downloader.trouble(u'ERROR: no known formats available for video')
self._downloader.increment_downloads()
video_id = mobj.group(1)
- simple_title = mobj.group(2).decode('utf-8')
video_extension = 'flv'
# Retrieve video webpage to extract further information
video_url = mediaURL
- mobj = re.search(r'(?im)<title>Dailymotion\s*-\s*(.+)\s*-\s*[^<]+?</title>', webpage)
+ mobj = re.search(r'<meta property="og:title" content="(?P<title>[^"]*)" />', webpage)
if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract title')
return
- video_title = mobj.group(1).decode('utf-8')
+ video_title = _unescapeHTML(mobj.group('title').decode('utf-8'))
video_title = sanitize_title(video_title)
+ simple_title = _simplify_title(video_title)
mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
if mobj is None:
self._downloader.trouble(u'ERROR: unable to extract video title')
return
video_title = mobj.group(1).decode('utf-8')
- simple_title = _simple_title(video_title)
+ simple_title = _simplify_title(video_title)
# Extract uploader
mobj = re.search(r'<uploader_url>http://vimeo.com/(.*?)</uploader_url>', webpage)
return
epTitle = mobj.group('episode')
- mMovieParams = re.findall('<param name="movie" value="(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"/>', html)
+ mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"', html)
if len(mMovieParams) == 0:
self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
return
'uploader': showId,
'upload_date': officialDate,
'title': effTitle,
- 'stitle': self._simplify_title(effTitle),
+ 'stitle': _simplify_title(effTitle),
'ext': 'mp4',
'format': format,
'thumbnail': None,
except UnavailableVideoError, err:
self._downloader.trouble(u'\nERROR: unable to download ' + video_url)
+class MixcloudIE(InfoExtractor):
+ """Information extractor for www.mixcloud.com"""
+ _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
+ IE_NAME = u'mixcloud'
+
+ def __init__(self, downloader=None):
+ InfoExtractor.__init__(self, downloader)
+
+ def report_download_json(self, file_id):
+ """Report JSON download."""
+ self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME)
+
+ def report_extraction(self, file_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
+
+ def get_urls(self, jsonData, fmt, bitrate='best'):
+ """Get urls from 'audio_formats' section in json"""
+ file_url = None
+ try:
+ bitrate_list = jsonData[fmt]
+ if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
+ bitrate = max(bitrate_list) # select highest
+
+ url_list = jsonData[fmt][bitrate]
+ except TypeError: # we have no bitrate info.
+ url_list = jsonData[fmt]
+
+ return url_list
+
+ def check_urls(self, url_list):
+ """Returns 1st active url from list"""
+ for url in url_list:
+ try:
+ urllib2.urlopen(url)
+ return url
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ url = None
+
+ return None
+
+ def _print_formats(self, formats):
+ print 'Available formats:'
+ for fmt in formats.keys():
+ for b in formats[fmt]:
+ try:
+ ext = formats[fmt][b][0]
+ print '%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1])
+ except TypeError: # we have no bitrate info
+ ext = formats[fmt][0]
+ print '%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1])
+ break
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+ # extract uploader & filename from url
+ uploader = mobj.group(1).decode('utf-8')
+ file_id = uploader + "-" + mobj.group(2).decode('utf-8')
+
+ # construct API request
+ file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
+ # retrieve .json file with links to files
+ request = urllib2.Request(file_url)
+ try:
+ self.report_download_json(file_url)
+ jsonData = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % str(err))
+ return
+
+ # parse JSON
+ json_data = json.loads(jsonData)
+ player_url = json_data['player_swf_url']
+ formats = dict(json_data['audio_formats'])
+
+ req_format = self._downloader.params.get('format', None)
+ bitrate = None
+
+ if self._downloader.params.get('listformats', None):
+ self._print_formats(formats)
+ return
+
+ if req_format is None or req_format == 'best':
+ for format_param in formats.keys():
+ url_list = self.get_urls(formats, format_param)
+ # check urls
+ file_url = self.check_urls(url_list)
+ if file_url is not None:
+ break # got it!
+ else:
+ if req_format not in formats.keys():
+ self._downloader.trouble(u'ERROR: format is not available')
+ return
+
+ url_list = self.get_urls(formats, req_format)
+ file_url = self.check_urls(url_list)
+ format_param = req_format
+
+ # We have audio
+ self._downloader.increment_downloads()
+ try:
+ # Process file information
+ self._downloader.process_info({
+ 'id': file_id.decode('utf-8'),
+ 'url': file_url.decode('utf-8'),
+ 'uploader': uploader.decode('utf-8'),
+ 'upload_date': u'NA',
+ 'title': json_data['name'],
+ 'stitle': _simplify_title(json_data['name']),
+ 'ext': file_url.split('.')[-1].decode('utf-8'),
+ 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
+ 'thumbnail': json_data['thumbnail_url'],
+ 'description': json_data['description'],
+ 'player_url': player_url.decode('utf-8'),
+ })
+ except UnavailableVideoError, err:
+ self._downloader.trouble(u'ERROR: unable to download file')
+
+class StanfordOpenClassroomIE(InfoExtractor):
+ """Information extractor for Stanford's Open ClassRoom"""
+
+ _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
+ IE_NAME = u'stanfordoc'
+
+ def report_download_webpage(self, objid):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid))
+
+ def report_extraction(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+
+ if mobj.group('course') and mobj.group('video'): # A specific video
+ course = mobj.group('course')
+ video = mobj.group('video')
+ info = {
+ 'id': _simplify_title(course + '_' + video),
+ }
+
+ self.report_extraction(info['id'])
+ baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
+ xmlUrl = baseUrl + video + '.xml'
+ try:
+ metaXml = urllib2.urlopen(xmlUrl).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % unicode(err))
+ return
+ mdoc = xml.etree.ElementTree.fromstring(metaXml)
+ try:
+ info['title'] = mdoc.findall('./title')[0].text
+ info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
+ except IndexError:
+ self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
+ return
+ info['stitle'] = _simplify_title(info['title'])
+ info['ext'] = info['url'].rpartition('.')[2]
+ info['format'] = info['ext']
+ self._downloader.increment_downloads()
+ try:
+ self._downloader.process_info(info)
+ except UnavailableVideoError, err:
+ self._downloader.trouble(u'\nERROR: unable to download video')
+ elif mobj.group('course'): # A course page
+ unescapeHTML = HTMLParser.HTMLParser().unescape
+
+ course = mobj.group('course')
+ info = {
+ 'id': _simplify_title(course),
+ 'type': 'playlist',
+ }
+
+ self.report_download_webpage(info['id'])
+ try:
+ coursepage = urllib2.urlopen(url).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
+ return
+
+ m = re.search('<h1>([^<]+)</h1>', coursepage)
+ if m:
+ info['title'] = unescapeHTML(m.group(1))
+ else:
+ info['title'] = info['id']
+ info['stitle'] = _simplify_title(info['title'])
+
+ m = re.search('<description>([^<]+)</description>', coursepage)
+ if m:
+ info['description'] = unescapeHTML(m.group(1))
+
+ links = _orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
+ info['list'] = [
+ {
+ 'type': 'reference',
+ 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
+ }
+ for vpage in links]
+
+ for entry in info['list']:
+ assert entry['type'] == 'reference'
+ self.extract(entry['url'])
+ else: # Root page
+ unescapeHTML = HTMLParser.HTMLParser().unescape
+
+ info = {
+ 'id': 'Stanford OpenClassroom',
+ 'type': 'playlist',
+ }
+
+ self.report_download_webpage(info['id'])
+ rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
+ try:
+ rootpage = urllib2.urlopen(rootURL).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
+ return
+
+ info['title'] = info['id']
+ info['stitle'] = _simplify_title(info['title'])
+
+ links = _orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
+ info['list'] = [
+ {
+ 'type': 'reference',
+ 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
+ }
+ for cpage in links]
+
+ for entry in info['list']:
+ assert entry['type'] == 'reference'
+ self.extract(entry['url'])
+
+class MTVIE(InfoExtractor):
+ """Information extractor for MTV.com"""
+
+ _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
+ IE_NAME = u'mtv'
+
+ def report_webpage(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+
+ def report_extraction(self, video_id):
+ """Report information extraction."""
+ self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
+ return
+ if not mobj.group('proto'):
+ url = 'http://' + url
+ video_id = mobj.group('videoid')
+ self.report_webpage(video_id)
+
+ request = urllib2.Request(url)
+ try:
+ webpage = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+ return
+
+ mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract song name')
+ return
+ song_name = _unescapeHTML(mobj.group(1).decode('iso-8859-1'))
+ mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract performer')
+ return
+ performer = _unescapeHTML(mobj.group(1).decode('iso-8859-1'))
+ video_title = performer + ' - ' + song_name
+
+ mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to mtvn_uri')
+ return
+ mtvn_uri = mobj.group(1)
+
+ mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract content id')
+ return
+ content_id = mobj.group(1)
+
+ videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri
+ self.report_extraction(video_id)
+ request = urllib2.Request(videogen_url)
+ try:
+ metadataXml = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % str(err))
+ return
+
+ mdoc = xml.etree.ElementTree.fromstring(metadataXml)
+ renditions = mdoc.findall('.//rendition')
+
+ # For now, always pick the highest quality.
+ rendition = renditions[-1]
+
+ try:
+ _,_,ext = rendition.attrib['type'].partition('/')
+ format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
+ video_url = rendition.find('./src').text
+ except KeyError:
+ self._downloader.trouble('Invalid rendition field.')
+ return
+
+ self._downloader.increment_downloads()
+ info = {
+ 'id': video_id,
+ 'url': video_url,
+ 'uploader': performer,
+ 'title': video_title,
+ 'stitle': _simplify_title(video_title),
+ 'ext': ext,
+ 'format': format,
+ }
+
+ try:
+ self._downloader.process_info(info)
+ except UnavailableVideoError, err:
+ self._downloader.trouble(u'\nERROR: unable to download ' + video_id)
class PostProcessor(object):
"""
return information # by default, do nothing
+class AudioConversionError(BaseException):
+ def __init__(self, message):
+ self.message = message
class FFmpegExtractAudioPP(PostProcessor):
@staticmethod
def get_audio_codec(path):
try:
- cmd = ['ffprobe', '-show_streams', '--', path]
+ cmd = ['ffprobe', '-show_streams', '--', _encodeFilename(path)]
handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
output = handle.communicate()[0]
if handle.wait() != 0:
@staticmethod
def run_ffmpeg(path, out_path, codec, more_opts):
+ if codec is None:
+ acodec_opts = []
+ else:
+ acodec_opts = ['-acodec', codec]
+ cmd = ['ffmpeg', '-y', '-i', _encodeFilename(path), '-vn'] + acodec_opts + more_opts + ['--', _encodeFilename(out_path)]
try:
- cmd = ['ffmpeg', '-y', '-i', path, '-vn', '-acodec', codec] + more_opts + ['--', out_path]
- ret = subprocess.call(cmd, stdout=file(os.path.devnull, 'w'), stderr=subprocess.STDOUT)
- return (ret == 0)
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout,stderr = p.communicate()
except (IOError, OSError):
- return False
+ e = sys.exc_info()[1]
+ if isinstance(e, OSError) and e.errno == 2:
+ raise AudioConversionError('ffmpeg not found. Please install ffmpeg.')
+ else:
+ raise e
+ if p.returncode != 0:
+ msg = stderr.strip().split('\n')[-1]
+ raise AudioConversionError(msg)
def run(self, information):
path = information['filepath']
return None
more_opts = []
- if self._preferredcodec == 'best' or self._preferredcodec == filecodec:
- if filecodec in ['aac', 'mp3', 'vorbis']:
+ if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
+ if self._preferredcodec == 'm4a' and filecodec == 'aac':
+ # Lossless, but in another container
+ acodec = 'copy'
+ extension = self._preferredcodec
+ more_opts = ['-absf', 'aac_adtstoasc']
+ elif filecodec in ['aac', 'mp3', 'vorbis']:
# Lossless if possible
acodec = 'copy'
extension = filecodec
more_opts += ['-ab', self._preferredquality]
else:
# We convert the audio (lossy)
- acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'vorbis': 'libvorbis'}[self._preferredcodec]
+ acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
extension = self._preferredcodec
more_opts = []
if self._preferredquality is not None:
more_opts += ['-ab', self._preferredquality]
if self._preferredcodec == 'aac':
more_opts += ['-f', 'adts']
+ if self._preferredcodec == 'm4a':
+ more_opts += ['-absf', 'aac_adtstoasc']
if self._preferredcodec == 'vorbis':
extension = 'ogg'
+ if self._preferredcodec == 'wav':
+ extension = 'wav'
+ more_opts += ['-f', 'wav']
- (prefix, ext) = os.path.splitext(path)
- new_path = prefix + '.' + extension
- self._downloader.to_screen(u'[ffmpeg] Destination: %s' % new_path)
- status = self.run_ffmpeg(path, new_path, acodec, more_opts)
-
- if not status:
- self._downloader.to_stderr(u'WARNING: error running ffmpeg')
+ prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
+ new_path = prefix + sep + extension
+ self._downloader.to_screen(u'[ffmpeg] Destination: ' + new_path)
+ try:
+ self.run_ffmpeg(path, new_path, acodec, more_opts)
+ except:
+ etype,e,tb = sys.exc_info()
+ if isinstance(e, AudioConversionError):
+ self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message)
+ else:
+ self._downloader.to_stderr(u'ERROR: error running ffmpeg')
return None
# Try to update the date time for extracted audio file.
if information.get('filetime') is not None:
try:
- os.utime(new_path, (time.time(), information['filetime']))
+ os.utime(_encodeFilename(new_path), (time.time(), information['filetime']))
except:
self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
if not self._keepvideo:
try:
- os.remove(path)
+ os.remove(_encodeFilename(path))
except (IOError, OSError):
self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
return None
# Deferred imports
import getpass
import optparse
+ import shlex
+
+ def _readOptions(filename_bytes):
+ try:
+ optionf = open(filename_bytes)
+ except IOError:
+ return [] # silently skip if file is not present
+ try:
+ res = []
+ for l in optionf:
+ res += shlex.split(l, comments=True)
+ finally:
+ optionf.close()
+ return res
def _format_option_string(option):
''' ('-o', '--option') -> -o, --format METAVAR'''
dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
+ selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None)
authentication.add_option('-u', '--username',
dest='username', metavar='USERNAME', help='account username')
action='store', dest='format', metavar='FORMAT', help='video format code')
video_format.add_option('--all-formats',
action='store_const', dest='format', help='download all available video formats', const='all')
+ video_format.add_option('--prefer-free-formats',
+ action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
video_format.add_option('--max-quality',
action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
video_format.add_option('-F', '--list-formats',
action='store_true', dest='autonumber',
help='number downloaded files starting from 00000', default=False)
filesystem.add_option('-o', '--output',
- dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(stitle)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, and %% for a literal percent')
+ dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(stitle)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), and %% for a literal percent. Use - to output to stdout.')
filesystem.add_option('-a', '--batch-file',
dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
filesystem.add_option('-w', '--no-overwrites',
postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False,
help='convert video files to audio-only files (requires ffmpeg and ffprobe)')
postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
- help='"best", "aac", "vorbis" or "mp3"; best by default')
+ help='"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default')
postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='128K',
help='ffmpeg audio bitrate specification, 128k by default')
postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
parser.add_option_group(authentication)
parser.add_option_group(postproc)
- opts, args = parser.parse_args()
+ xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
+ if xdg_config_home:
+ userConf = os.path.join(xdg_config_home, 'youtube-dl.conf')
+ else:
+ userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
+ argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:]
+ opts, args = parser.parse_args(argv)
return parser, opts, args
XVideosIE(),
SoundcloudIE(),
InfoQIE(),
+ MixcloudIE(),
+ StanfordOpenClassroomIE(),
+ MTVIE(),
GenericIE()
]
except (TypeError, ValueError), err:
parser.error(u'invalid playlist end number specified')
if opts.extractaudio:
- if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis']:
+ if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']:
parser.error(u'invalid audio format specified')
# File downloader
'writeinfojson': opts.writeinfojson,
'matchtitle': opts.matchtitle,
'rejecttitle': opts.rejecttitle,
+ 'max_downloads': opts.max_downloads,
+ 'prefer_free_formats': opts.prefer_free_formats,
})
for extractor in extractors:
fd.add_info_extractor(extractor)
parser.error(u'you must provide at least one URL')
else:
sys.exit()
- retcode = fd.download(all_urls)
+
+ try:
+ retcode = fd.download(all_urls)
+ except MaxDownloadsReached:
+ fd.to_screen(u'--max-download limit reached, aborting.')
+ retcode = 101
# Dump cookie jar if requested
if opts.cookiefile is not None: