and stitle is created in process_info() and is cross-filesystem sanitized by sanitize_filename();
closes #164
def process_info(self, info_dict):
"""Process a single dictionary returned by an InfoExtractor."""
def process_info(self, info_dict):
"""Process a single dictionary returned by an InfoExtractor."""
+ info_dict['stitle'] = sanitize_filename(info_dict['title'])
+
reason = self._match_entry(info_dict)
if reason is not None:
self.to_screen(u'[download] ' + reason)
reason = self._match_entry(info_dict)
if reason is not None:
self.to_screen(u'[download] ' + reason)
url: Final video URL.
uploader: Nickname of the video uploader.
title: Literal title.
url: Final video URL.
uploader: Nickname of the video uploader.
title: Literal title.
- stitle: Simplified title.
ext: Video filename extension.
format: Video format.
player_url: SWF Player URL (may be None).
ext: Video filename extension.
format: Video format.
player_url: SWF Player URL (may be None).
return
video_title = urllib.unquote_plus(video_info['title'][0])
video_title = video_title.decode('utf-8')
return
video_title = urllib.unquote_plus(video_info['title'][0])
video_title = video_title.decode('utf-8')
- video_title = sanitize_title(video_title)
-
- # simplified title
- simple_title = simplify_title(video_title)
# thumbnail image
if 'thumbnail_url' not in video_info:
# thumbnail image
if 'thumbnail_url' not in video_info:
'uploader': video_uploader.decode('utf-8'),
'upload_date': upload_date,
'title': video_title,
'uploader': video_uploader.decode('utf-8'),
'upload_date': upload_date,
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': video_thumbnail.decode('utf-8'),
'ext': video_extension.decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': video_thumbnail.decode('utf-8'),
self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
return
self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
return
- simple_title = mobj.group(2).decode('utf-8')
-
# Retrieve video webpage to extract further information
request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
try:
# Retrieve video webpage to extract further information
request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
try:
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
- video_title = sanitize_title(video_title)
mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
if mobj is None:
mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
if mobj is None:
'uploader': video_uploader.decode('utf-8'),
'upload_date': u'NA',
'title': video_title,
'uploader': video_uploader.decode('utf-8'),
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
- video_title = sanitize_title(video_title)
- simple_title = simplify_title(video_title)
mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
if mobj is None:
mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
if mobj is None:
'uploader': video_uploader.decode('utf-8'),
'upload_date': u'NA',
'title': video_title,
'uploader': video_uploader.decode('utf-8'),
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
- video_title = sanitize_title(video_title)
- simple_title = simplify_title(video_title)
# Extract video description
mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
# Extract video description
mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
'uploader': u'NA',
'upload_date': u'NA',
'title': video_title,
'uploader': u'NA',
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
- video_title = sanitize_title(video_title)
- simple_title = simplify_title(video_title)
video_uploader = mobj.group(2).decode('utf-8')
video_uploader = mobj.group(2).decode('utf-8')
'uploader': video_uploader,
'upload_date': u'NA',
'title': video_title,
'uploader': video_uploader,
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
self._downloader.trouble(u'ERROR: unable to extract video title')
return
video_title = mobj.group(1).decode('utf-8')
self._downloader.trouble(u'ERROR: unable to extract video title')
return
video_title = mobj.group(1).decode('utf-8')
- simple_title = simplify_title(video_title)
mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
if mobj is None:
mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
if mobj is None:
'uploader': video_uploader,
'upload_date': u'NA',
'title': video_title,
'uploader': video_uploader,
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'thumbnail': video_thumbnail.decode('utf-8'),
'description': video_description,
'ext': video_extension.decode('utf-8'),
'thumbnail': video_thumbnail.decode('utf-8'),
'description': video_description,
# Extract title
video_title = config["video"]["title"]
# Extract title
video_title = config["video"]["title"]
- simple_title = simplify_title(video_title)
# Extract uploader
video_uploader = config["video"]["owner"]["name"]
# Extract uploader
video_uploader = config["video"]["owner"]["name"]
'uploader': video_uploader,
'upload_date': video_upload_date,
'title': video_title,
'uploader': video_uploader,
'upload_date': video_upload_date,
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension,
'thumbnail': video_thumbnail,
'description': video_description,
'ext': video_extension,
'thumbnail': video_thumbnail,
'description': video_description,
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
- video_title = sanitize_title(video_title)
- simple_title = simplify_title(video_title)
# video uploader is domain name
mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
# video uploader is domain name
mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
'uploader': video_uploader,
'upload_date': u'NA',
'title': video_title,
'uploader': video_uploader,
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
'uploader': u'NA',
'upload_date': u'NA',
'title': file_title,
'uploader': u'NA',
'upload_date': u'NA',
'title': file_title,
'ext': file_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
'ext': file_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
return
video_title = video_info['title']
video_title = video_title.decode('utf-8')
return
video_title = video_info['title']
video_title = video_title.decode('utf-8')
- video_title = sanitize_title(video_title)
-
- simple_title = simplify_title(video_title)
# thumbnail image
if 'thumbnail' not in video_info:
# thumbnail image
if 'thumbnail' not in video_info:
'uploader': video_uploader.decode('utf-8'),
'upload_date': upload_date,
'title': video_title,
'uploader': video_uploader.decode('utf-8'),
'upload_date': upload_date,
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': video_thumbnail.decode('utf-8'),
'ext': video_extension.decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': video_thumbnail.decode('utf-8'),
'id': title,
'url': url,
'title': title,
'id': title,
'url': url,
'title': title,
- 'stitle': simplify_title(title),
'ext': ext,
'urlhandle': urlh
}
'ext': ext,
'urlhandle': urlh
}
'uploader': data['display_name'],
'upload_date': upload_date,
'title': data['title'],
'uploader': data['display_name'],
'upload_date': upload_date,
'title': data['title'],
- 'stitle': simplify_title(data['title']),
'ext': ext,
'format': data['media']['mimeType'],
'thumbnail': data['thumbnailUrl'],
'ext': ext,
'format': data['media']['mimeType'],
'thumbnail': data['thumbnailUrl'],
return
video_title = mobj.group(1)
return
video_title = mobj.group(1)
- video_title = sanitize_title(video_title)
-
- simple_title = simplify_title(video_title)
return [{
'id': video_id,
return [{
'id': video_id,
'uploader': u'NA',
'upload_date': u'NA',
'title': video_title,
'uploader': u'NA',
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': u'flv',
'format': u'NA',
'player_url': None,
'ext': u'flv',
'format': u'NA',
'player_url': None,
'uploader': showId,
'upload_date': officialDate,
'title': effTitle,
'uploader': showId,
'upload_date': officialDate,
'title': effTitle,
- 'stitle': simplify_title(effTitle),
'ext': 'mp4',
'format': format,
'thumbnail': None,
'ext': 'mp4',
'format': format,
'thumbnail': None,
'uploader': showName,
'upload_date': None,
'title': showName,
'uploader': showName,
'upload_date': None,
'title': showName,
- 'stitle': simplify_title(showName),
'ext': 'flv',
'format': 'flv',
'thumbnail': imgUrl,
'ext': 'flv',
'format': 'flv',
'thumbnail': imgUrl,
videoNode = mdoc.findall('./video')[0]
info['description'] = videoNode.findall('./description')[0].text
info['title'] = videoNode.findall('./caption')[0].text
videoNode = mdoc.findall('./video')[0]
info['description'] = videoNode.findall('./description')[0].text
info['title'] = videoNode.findall('./caption')[0].text
- info['stitle'] = simplify_title(info['title'])
info['url'] = videoNode.findall('./file')[0].text
info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
info['ext'] = info['url'].rpartition('.')[2]
info['url'] = videoNode.findall('./file')[0].text
info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
info['ext'] = info['url'].rpartition('.')[2]
'uploader': None,
'upload_date': None,
'title': video_title,
'uploader': None,
'upload_date': None,
'title': video_title,
- 'stitle': simplify_title(video_title),
'ext': 'flv',
'format': 'flv',
'thumbnail': video_thumbnail,
'ext': 'flv',
'format': 'flv',
'thumbnail': video_thumbnail,
uploader = mobj.group(1).decode('utf-8')
# extract simple title (uploader + slug of song title)
slug_title = mobj.group(2).decode('utf-8')
uploader = mobj.group(1).decode('utf-8')
# extract simple title (uploader + slug of song title)
slug_title = mobj.group(2).decode('utf-8')
- simple_title = uploader + '-' + slug_title
+ simple_title = uploader + u'-' + slug_title
self.report_webpage('%s/%s' % (uploader, slug_title))
self.report_webpage('%s/%s' % (uploader, slug_title))
# extract unsimplified title
mobj = re.search('"title":"(.*?)",', webpage)
if mobj:
# extract unsimplified title
mobj = re.search('"title":"(.*?)",', webpage)
if mobj:
+ title = mobj.group(1).decode('utf-8')
+ else:
+ title = simple_title
# construct media url (with uid/token)
mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
# construct media url (with uid/token)
mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
'url': mediaURL,
'uploader': uploader.decode('utf-8'),
'upload_date': upload_date,
'url': mediaURL,
'uploader': uploader.decode('utf-8'),
'upload_date': upload_date,
- 'title': simple_title.decode('utf-8'),
- 'stitle': simple_title.decode('utf-8'),
'ext': u'mp3',
'format': u'NA',
'player_url': None,
'ext': u'mp3',
'format': u'NA',
'player_url': None,
'uploader': None,
'upload_date': None,
'title': video_title,
'uploader': None,
'upload_date': None,
'title': video_title,
- 'stitle': simplify_title(video_title),
'ext': extension,
'format': extension, # Extension is always(?) mp4, but seems to be flv
'thumbnail': None,
'ext': extension,
'format': extension, # Extension is always(?) mp4, but seems to be flv
'thumbnail': None,
'uploader': uploader.decode('utf-8'),
'upload_date': u'NA',
'title': json_data['name'],
'uploader': uploader.decode('utf-8'),
'upload_date': u'NA',
'title': json_data['name'],
- 'stitle': simplify_title(json_data['name']),
'ext': file_url.split('.')[-1].decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': json_data['thumbnail_url'],
'ext': file_url.split('.')[-1].decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': json_data['thumbnail_url'],
course = mobj.group('course')
video = mobj.group('video')
info = {
course = mobj.group('course')
video = mobj.group('video')
info = {
- 'id': simplify_title(course + '_' + video),
+ 'id': course + '_' + video,
}
self.report_extraction(info['id'])
}
self.report_extraction(info['id'])
except IndexError:
self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
return
except IndexError:
self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
return
- info['stitle'] = simplify_title(info['title'])
info['ext'] = info['url'].rpartition('.')[2]
info['format'] = info['ext']
return [info]
elif mobj.group('course'): # A course page
course = mobj.group('course')
info = {
info['ext'] = info['url'].rpartition('.')[2]
info['format'] = info['ext']
return [info]
elif mobj.group('course'): # A course page
course = mobj.group('course')
info = {
- 'id': simplify_title(course),
info['title'] = unescapeHTML(m.group(1))
else:
info['title'] = info['id']
info['title'] = unescapeHTML(m.group(1))
else:
info['title'] = info['id']
- info['stitle'] = simplify_title(info['title'])
m = re.search('<description>([^<]+)</description>', coursepage)
if m:
m = re.search('<description>([^<]+)</description>', coursepage)
if m:
return
info['title'] = info['id']
return
info['title'] = info['id']
- info['stitle'] = simplify_title(info['title'])
links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
info['list'] = [
links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
info['list'] = [
'url': video_url,
'uploader': performer,
'title': video_title,
'url': video_url,
'uploader': performer,
'title': video_title,
- 'stitle': simplify_title(video_title),
'ext': ext,
'format': format,
}
'ext': ext,
'format': format,
}
-def sanitize_title(utitle):
- """Sanitizes a video title so it could be used as part of a filename."""
- utitle = unescapeHTML(utitle)
- return utitle.replace(unicode(os.sep), u'%')
-
-
def sanitize_open(filename, open_mode):
"""Try to open the given filename, and slightly tweak it if this fails.
def sanitize_open(filename, open_mode):
"""Try to open the given filename, and slightly tweak it if this fails.
if timetuple is not None:
timestamp = email.utils.mktime_tz(timetuple)
return timestamp
if timetuple is not None:
timestamp = email.utils.mktime_tz(timetuple)
return timestamp
-
-def simplify_title(title):
- expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE)
- return expr.sub(u'_', title).strip(u'_')
+
+def sanitize_filename(s):
+ """Sanitizes a string so it could be used as part of a filename."""
+ def replace_insane(char):
+ if char in u' .\\/|?*<>:"' or ord(char) < 32:
+ return '_'
+ return char
+ return u''.join(map(replace_insane, s)).strip('_')
def orderedSet(iterable):
""" Remove all duplicates from the input iterable """
def orderedSet(iterable):
""" Remove all duplicates from the input iterable """