%(extractor)s for the provider (youtube, metacafe,
etc), %(id)s for the video id and %% for a literal
percent. Use - to output to stdout.
- --restrict-filenames Avoid some characters such as "&" and spaces in
- filenames
+ --restrict-filenames Restrict filenames to only ASCII characters, and
+ avoid "&" and spaces in filenames
-a, --batch-file FILE file containing URLs to download ('-' for stdin)
-w, --no-overwrites do not overwrite files
-c, --continue resume partially downloaded files
from youtube_dl.FileDownloader import FileDownloader
from youtube_dl.InfoExtractors import YoutubeIE, DailymotionIE
from youtube_dl.InfoExtractors import MetacafeIE, BlipTVIE
+from youtube_dl.InfoExtractors import XVideosIE, VimeoIE
+from youtube_dl.InfoExtractors import SoundcloudIE, StanfordOpenClassroomIE
+from youtube_dl.InfoExtractors import CollegeHumorIE, XNXXIE
class DownloadTest(unittest.TestCase):
BLIP_URL = "http://blip.tv/cbr/cbr-exclusive-gotham-city-imposters-bats-vs-jokerz-short-3-5796352"
BLIP_FILE = "5779306.m4v"
- XVIDEO_MD5 = ""
- XVIDEO_URL = ""
- XVIDEO_FILE = ""
+ XVIDEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf"
+ XVIDEO_URL = "http://www.xvideos.com/video939581/funny_porns_by_s_-1"
+ XVIDEO_FILE = "939581.flv"
+ VIMEO_MD5 = "1ab4dedc01f771cb2a65e91caa801aaf"
+ VIMEO_URL = "http://vimeo.com/14160053"
+ VIMEO_FILE = ""
+
+ VIMEO2_MD5 = ""
+ VIMEO2_URL = "http://player.vimeo.com/video/47019590"
+ VIMEO2_FILE = ""
+
+ SOUNDCLOUD_MD5 = "ce3775768ebb6432fa8495d446a078ed"
+ SOUNDCLOUD_URL = "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy"
+ SOUNDCLOUD_FILE = "n6FLbx6ZzMiu.mp3"
+
+ STANDFORD_MD5 = "22c8206291368c4e2c9c1a307f0ea0f4"
+ STANDFORD_URL = "http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100"
+ STANDFORD_FILE = "PracticalUnix_intro-environment.mp4"
+
+ COLLEGEHUMOR_MD5 = ""
+ COLLEGEHUMOR_URL = "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody"
+ COLLEGEHUMOR_FILE = ""
+
+ XNXX_MD5 = "5f0469c8d1dfd1bc38c8e6deb5e0a21d"
+ XNXX_URL = "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_"
+ XNXX_FILE = "1135332.flv"
def test_youtube(self):
#let's download a file from youtube
md5_down_file = md5_for_file(DownloadTest.BLIP_FILE)
self.assertEqual(md5_down_file, DownloadTest.BLIP_MD5)
+ def test_xvideo(self):
+ with open(DownloadTest.PARAMETERS_FILE) as f:
+ fd = FileDownloader(json.load(f))
+ fd.add_info_extractor(XVideosIE())
+ fd.download([DownloadTest.XVIDEO_URL])
+ self.assertTrue(os.path.exists(DownloadTest.XVIDEO_FILE))
+ md5_down_file = md5_for_file(DownloadTest.XVIDEO_FILE)
+ self.assertEqual(md5_down_file, DownloadTest.XVIDEO_MD5)
+
+ def test_vimeo(self):
+ #skipped for the moment produce an error
+ return
+ with open(DownloadTest.PARAMETERS_FILE) as f:
+ fd = FileDownloader(json.load(f))
+ fd.add_info_extractor(VimeoIE())
+ fd.download([DownloadTest.VIMEO_URL])
+ self.assertTrue(os.path.exists(DownloadTest.VIMEO_FILE))
+ md5_down_file = md5_for_file(DownloadTest.VIMEO_FILE)
+ self.assertEqual(md5_down_file, DownloadTest.VIMEO_MD5)
+
+ def test_vimeo2(self):
+ #skipped for the moment produce an error
+ return
+ with open(DownloadTest.PARAMETERS_FILE) as f:
+ fd = FileDownloader(json.load(f))
+ fd.add_info_extractor(VimeoIE())
+ fd.download([DownloadTest.VIMEO2_URL])
+ self.assertTrue(os.path.exists(DownloadTest.VIMEO2_FILE))
+ md5_down_file = md5_for_file(DownloadTest.VIMEO2_FILE)
+ self.assertEqual(md5_down_file, DownloadTest.VIMEO2_MD5)
+
+ def test_soundcloud(self):
+ with open(DownloadTest.PARAMETERS_FILE) as f:
+ fd = FileDownloader(json.load(f))
+ fd.add_info_extractor(SoundcloudIE())
+ fd.download([DownloadTest.SOUNDCLOUD_URL])
+ self.assertTrue(os.path.exists(DownloadTest.SOUNDCLOUD_FILE))
+ md5_down_file = md5_for_file(DownloadTest.SOUNDCLOUD_FILE)
+ self.assertEqual(md5_down_file, DownloadTest.SOUNDCLOUD_MD5)
+
+ def test_standford(self):
+ with open(DownloadTest.PARAMETERS_FILE) as f:
+ fd = FileDownloader(json.load(f))
+ fd.add_info_extractor(StanfordOpenClassroomIE())
+ fd.download([DownloadTest.STANDFORD_URL])
+ self.assertTrue(os.path.exists(DownloadTest.STANDFORD_FILE))
+ md5_down_file = md5_for_file(DownloadTest.STANDFORD_FILE)
+ self.assertEqual(md5_down_file, DownloadTest.STANDFORD_MD5)
+
+ def test_collegehumor(self):
+ with open(DownloadTest.PARAMETERS_FILE) as f:
+ fd = FileDownloader(json.load(f))
+ fd.add_info_extractor(CollegeHumorIE())
+ fd.download([DownloadTest.COLLEGEHUMOR_URL])
+ self.assertTrue(os.path.exists(DownloadTest.COLLEGEHUMOR_FILE))
+ md5_down_file = md5_for_file(DownloadTest.COLLEGEHUMOR_FILE)
+ self.assertEqual(md5_down_file, DownloadTest.COLLEGEHUMOR_MD5)
+
+ def test_xnxx(self):
+ with open(DownloadTest.PARAMETERS_FILE) as f:
+ fd = FileDownloader(json.load(f))
+ fd.add_info_extractor(XNXXIE())
+ fd.download([DownloadTest.XNXX_URL])
+ self.assertTrue(os.path.exists(DownloadTest.XNXX_FILE))
+ md5_down_file = md5_for_file(DownloadTest.XNXX_FILE)
+ self.assertEqual(md5_down_file, DownloadTest.XNXX_MD5)
+
def tearDown(self):
if os.path.exists(DownloadTest.YOUTUBE_FILE):
os.remove(DownloadTest.YOUTUBE_FILE)
os.remove(DownloadTest.METACAFE_FILE)
if os.path.exists(DownloadTest.BLIP_FILE):
os.remove(DownloadTest.BLIP_FILE)
+ if os.path.exists(DownloadTest.XVIDEO_FILE):
+ os.remove(DownloadTest.XVIDEO_FILE)
+ if os.path.exists(DownloadTest.VIMEO_FILE):
+ os.remove(DownloadTest.VIMEO_FILE)
+ if os.path.exists(DownloadTest.SOUNDCLOUD_FILE):
+ os.remove(DownloadTest.SOUNDCLOUD_FILE)
+ if os.path.exists(DownloadTest.STANDFORD_FILE):
+ os.remove(DownloadTest.STANDFORD_FILE)
+ if os.path.exists(DownloadTest.COLLEGEHUMOR_FILE):
+ os.remove(DownloadTest.COLLEGEHUMOR_FILE)
+ if os.path.exists(DownloadTest.XNXX_FILE):
+ os.remove(DownloadTest.XNXX_FILE)
def md5_for_file(filename, block_size=2**20):
- with open(filename) as f:
- md5 = hashlib.md5()
- while True:
- data = f.read(block_size)
- if not data:
- break
- md5.update(data)
- return md5.hexdigest()
+ with open(filename) as f:
+ md5 = hashlib.md5()
+ while True:
+ data = f.read(block_size)
+ if not data:
+ break
+ md5.update(data)
+ return md5.hexdigest()
self.assertEqual(sanitize_filename(u'123'), u'123')
- self.assertEqual(u'abc-de', sanitize_filename(u'abc/de'))
+ self.assertEqual(u'abc_de', sanitize_filename(u'abc/de'))
self.assertFalse(u'/' in sanitize_filename(u'abc/de///'))
- self.assertEqual(u'abc-de', sanitize_filename(u'abc/<>\\*|de'))
+ self.assertEqual(u'abc_de', sanitize_filename(u'abc/<>\\*|de'))
self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|'))
self.assertEqual(u'yes no', sanitize_filename(u'yes? no'))
self.assertEqual(u'this - that', sanitize_filename(u'this: that'))
self.assertEqual(sanitize_filename(u'123', restricted=True), u'123')
- self.assertEqual(u'abc-de', sanitize_filename(u'abc/de', restricted=True))
+ self.assertEqual(u'abc_de', sanitize_filename(u'abc/de', restricted=True))
self.assertFalse(u'/' in sanitize_filename(u'abc/de///', restricted=True))
- self.assertEqual(u'abc-de', sanitize_filename(u'abc/<>\\*|de', restricted=True))
+ self.assertEqual(u'abc_de', sanitize_filename(u'abc/<>\\*|de', restricted=True))
self.assertEqual(u'xxx', sanitize_filename(u'xxx/<>\\*|', restricted=True))
self.assertEqual(u'yes_no', sanitize_filename(u'yes? no', restricted=True))
self.assertEqual(u'this_-_that', sanitize_filename(u'this: that', restricted=True))
+ self.assertEqual(sanitize_filename(u'aäb', restricted=True), u'a_b')
+ self.assertTrue(sanitize_filename(u'ö', restricted=True) != u'') # No empty filename
+
forbidden = u'"\0\\/&: \'\t\n'
for fc in forbidden:
- print('input: ' + fc + ', result: ' + repr(sanitize_filename(fc, restricted=True)))
for fbc in forbidden:
self.assertTrue(fbc not in sanitize_filename(fc, restricted=True))
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(extractor)s\ for\ the\ provider\ (youtube,\ metacafe,
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ etc),\ %(id)s\ for\ the\ video\ id\ and\ %%\ for\ a\ literal
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ percent.\ Use\ -\ to\ output\ to\ stdout.
---restrict-filenames\ \ \ \ \ Avoid\ some\ characters\ such\ as\ "&"\ and\ spaces\ in
-\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ filenames
+--restrict-filenames\ \ \ \ \ Restrict\ filenames\ to\ only\ ASCII\ characters,\ and
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ avoid\ "&"\ and\ spaces\ in\ filenames
-a,\ --batch-file\ FILE\ \ \ \ file\ containing\ URLs\ to\ download\ (\[aq]-\[aq]\ for\ stdin)
-w,\ --no-overwrites\ \ \ \ \ \ do\ not\ overwrite\ files
-c,\ --continue\ \ \ \ \ \ \ \ \ \ \ resume\ partially\ downloaded\ files
self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
self.params = params
+ if '%(stitle)s' in self.params['outtmpl']:
+ self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
+
@staticmethod
def format_bytes(bytes):
if bytes is None:
"""Generate the output filename."""
try:
template_dict = dict(info_dict)
- template_dict['epoch'] = unicode(long(time.time()))
+ template_dict['epoch'] = unicode(int(time.time()))
template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
- template_dict['title'] = template_dict['stitle'] # Keep both for backwards compatibility
filename = self.params['outtmpl'] % template_dict
return filename
except (ValueError, KeyError), err:
def process_info(self, info_dict):
"""Process a single dictionary returned by an InfoExtractor."""
- info_dict['stitle'] = sanitize_filename(info_dict['title'], self.params.get('restrictfilenames'))
+ # Keep for backwards compatibility
+ info_dict['stitle'] = info_dict['title']
reason = self._match_entry(info_dict)
if reason is not None:
raise MaxDownloadsReached()
filename = self.prepare_filename(info_dict)
+ filename = sanitize_filename(filename, self.params.get('restrictfilenames'))
# Forced printings
if self.params.get('forcetitle', False):
print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
-
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
if mobj is None:
continue
if self._downloader.params.get('listformats', None):
- self._print_formats([i[0] for i in turls])
- return
+ self._print_formats([i[0] for i in turls])
+ return
# For now, just pick the highest bitrate
format,video_url = turls[-1]
# Select format if we can find one
for f,v in turls:
- if f == req_format:
- format, video_url = f, v
- break
-
- # Patch to download from alternative CDN, which does not
- # break on current RTMPDump builds
-
+ if f == req_format:
+ format, video_url = f, v
+ break
+ # Patch to download from alternative CDN, which does not
+ # break on current RTMPDump builds
broken_cdn = "rtmpe://viacomccstrmfs.fplive.net/viacomccstrm/gsp.comedystor/"
better_cdn = "rtmpe://cp10740.edgefcs.net/ondemand/mtvnorigin/gsp.comedystor/"
-
+
if video_url.startswith(broken_cdn):
- video_url = video_url.replace(broken_cdn, better_cdn)
-
+ video_url = video_url.replace(broken_cdn, better_cdn)
effTitle = showId + u'-' + epTitle
info = {
dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
filesystem.add_option('--restrict-filenames',
action='store_true', dest='restrictfilenames',
- help='Avoid some characters such as "&" and spaces in filenames', default=False)
+ help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
filesystem.add_option('-a', '--batch-file',
dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
filesystem.add_option('-w', '--no-overwrites',
parser.error(u'you must provide at least one URL')
else:
sys.exit()
-
+
try:
retcode = fd.download(all_urls)
except MaxDownloadsReached:
elif char == ':':
return '_-' if restricted else ' -'
elif char in '\\/|*<>':
- return '-'
+ return '_'
if restricted and (char in '&\'' or char.isspace()):
return '_'
+ if restricted and ord(char) > 127:
+ return '_'
return char
result = u''.join(map(replace_insane, s))
- while '--' in result:
- result = result.replace('--', '-')
- return result.strip('-')
+ while '__' in result:
+ result = result.replace('__', '_')
+ result = result.strip('_')
+ if not result:
+ result = '_'
+ return result
def orderedSet(iterable):
""" Remove all duplicates from the input iterable """
projects / youtube-dl / commitdiff