%(upload_date)s for the upload date (YYYYMMDD),
%(extractor)s for the provider (youtube, metacafe,
etc), %(id)s for the video id and %% for a literal
- percent. Use - to output to stdout.
+ percent. Use - to output to stdout. Can also be
+ used to download to a different directory, for
+ example with -o '/my/downloads/%(uploader)s/%(title
+ )s-%(id)s.%(ext)s' .
--restrict-filenames Restrict filenames to only ASCII characters, and
avoid "&" and spaces in filenames
-a, --batch-file FILE file containing URLs to download ('-' for stdin)
* The output of `youtube-dl --version`
* The output of `python --version`
* The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough).
+
+For discussions, join us in the irc channel #youtube-dl on freenode.
def _file_md5(fn):
with open(fn, 'rb') as f:
return hashlib.md5(f.read()).hexdigest()
-
-def md5_for_file(filename, block_size=2**20):
- with open(filename) as f:
- md5 = hashlib.md5()
- while True:
- data = f.read(block_size)
- if not data:
- break
- md5.update(data)
- return md5.hexdigest()
-_file_md5 = md5_for_file
-
-
try:
_skip_unless = unittest.skipUnless
except AttributeError: # Python 2.6
def _skip_unless(cond, reason='No reason given'):
def resfunc(f):
- def wfunc(*args, **kwargs):
+ # Start the function name with test to appease nosetests-2.6
+ def test_wfunc(*args, **kwargs):
if cond:
return f(*args, **kwargs)
else:
print('Skipped test')
return
- return wfunc
+ return test_wfunc
return resfunc
_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
def _file_md5(fn):
with open(fn, 'rb') as f:
return hashlib.md5(f.read()).hexdigest()
-
-def md5_for_file(filename, block_size=2**20):
- with open(filename) as f:
- md5 = hashlib.md5()
- while True:
- data = f.read(block_size)
- if not data:
- break
- md5.update(data)
- return md5.hexdigest()
-_file_md5 = md5_for_file
-
-
try:
_skip_unless = unittest.skipUnless
except AttributeError: # Python 2.6
def _skip_unless(cond, reason='No reason given'):
def resfunc(f):
- def wfunc(*args, **kwargs):
+ # Start the function name with test to appease nosetests-2.6
+ def test_wfunc(*args, **kwargs):
if cond:
return f(*args, **kwargs)
else:
print('Skipped test')
return
- return wfunc
+ return test_wfunc
return resfunc
_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
@_skip_unless(youtube_dl.InfoExtractors.MetacafeIE._WORKING, "IE marked as not _WORKING")
def test_Metacafe(self):
- filename = 'aUehQsCQtM.flv'
+ filename = '_aUehQsCQtM.flv'
fd = FileDownloader(self.parameters)
fd.add_info_extractor(youtube_dl.InfoExtractors.MetacafeIE())
fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE())
@_skip_unless(youtube_dl.InfoExtractors.SoundcloudIE._WORKING, "IE marked as not _WORKING")
def test_Soundcloud(self):
- filename = 'n6FLbx6ZzMiu.mp3'
+ filename = '62986583.mp3'
fd = FileDownloader(self.parameters)
fd.add_info_extractor(youtube_dl.InfoExtractors.SoundcloudIE())
fd.download(['http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy'])
md5_for_file = _file_md5(filename)
self.assertEqual(md5_for_file, 'c5c67df477eb0d9b058200351448ba4c')
+ @_skip_unless(youtube_dl.InfoExtractors.YoukuIE._WORKING, "IE marked as not _WORKING")
+ def test_Youku(self):
+ filename = 'XNDgyMDQ2NTQw_part00.flv'
+ fd = FileDownloader(self.parameters)
+ fd.add_info_extractor(youtube_dl.InfoExtractors.YoukuIE())
+ fd.download(['http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html'])
+ self.assertTrue(os.path.exists(filename))
+ md5_for_file = _file_md5(filename)
+ self.assertEqual(md5_for_file, 'ffe3f2e435663dc2d1eea34faeff5b5b')
+
def tearDown(self):
if os.path.exists('BaW_jenozKc.mp4'):
os.remove('BaW_jenozKc.mp4')
if os.path.exists('x33vw9.mp4'):
os.remove('x33vw9.mp4')
- if os.path.exists('aUehQsCQtM.flv'):
- os.remove('aUehQsCQtM.flv')
+ if os.path.exists('_aUehQsCQtM.flv'):
+ os.remove('_aUehQsCQtM.flv')
if os.path.exists('5779306.m4v'):
os.remove('5779306.m4v')
if os.path.exists('939581.flv'):
os.remove('939581.flv')
# No file specified for Vimeo
- if os.path.exists('n6FLbx6ZzMiu.mp3'):
- os.remove('n6FLbx6ZzMiu.mp3')
+ if os.path.exists('62986583.mp3'):
+ os.remove('62986583.mp3')
if os.path.exists('PracticalUnix_intro-environment.mp4'):
os.remove('PracticalUnix_intro-environment.mp4')
# No file specified for CollegeHumor
if os.path.exists('1135332.flv'):
os.remove('1135332.flv')
+ if os.path.exists('XNDgyMDQ2NTQw_part00.flv'):
+ os.remove('XNDgyMDQ2NTQw_part00.flv')
--- /dev/null
+import unittest
+
+import sys
+import os
+import subprocess
+
+rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+try:
+ _DEV_NULL = subprocess.DEVNULL
+except AttributeError:
+ _DEV_NULL = open(os.devnull, 'wb')
+
+class TestExecution(unittest.TestCase):
+ def test_import(self):
+ subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
+
+ def test_module_exec(self):
+ if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution
+ subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+ def test_main_exec(self):
+ subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+if __name__ == '__main__':
+ unittest.main()
+++ /dev/null
-import unittest
-
-import sys
-import os.path
-import subprocess
-
-class TestImport(unittest.TestCase):
- def test_import(self):
- rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
-
-if __name__ == '__main__':
- unittest.main()
+#!/usr/bin/env python
+
# Various small unit tests
import sys
self.assertTrue(sanitize_filename('-', restricted=True) != '')
self.assertTrue(sanitize_filename(':', restricted=True) != '')
+ def test_sanitize_ids(self):
+ self.assertEquals(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
+ self.assertEquals(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
+ self.assertEquals(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
+
def test_ordered_set(self):
self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
self.assertEqual(orderedSet([]), [])
--- /dev/null
+#!/usr/bin/env python
+
+import sys
+import unittest
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE
+
+class TestYoutubePlaylistMatching(unittest.TestCase):
+ def test_playlist_matching(self):
+ assert YoutubePlaylistIE().suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
+ assert YoutubePlaylistIE().suitable(u'PL63F0C78739B09958')
+ assert not YoutubePlaylistIE().suitable(u'PLtS2H6bU1M')
+
+ def test_youtube_matching(self):
+ assert YoutubeIE().suitable(u'PLtS2H6bU1M')
+
+if __name__ == '__main__':
+ unittest.main()
"size": 5754305,
"addIEs": ["Youtube"],
"url": "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
- "file": "aUehQsCQtM.flv"
+ "file": "_aUehQsCQtM.flv"
},
{
"name": "BlipTV",
"name": "Soundcloud",
"md5": "c1b9b9ea8bfd620b96b2628664576e1c",
"url": "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy",
- "file": "n6FLbx6ZzMiu.mp3"
+ "file": "62986583.mp3"
},
{
"name": "StanfordOpenClassroom",
"url": "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody",
"file": ""
},
- {
+ {
"name": "XNXX",
"md5": "c5c67df477eb0d9b058200351448ba4c",
"url": "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_",
"file": "1135332.flv"
+ },
+ {
+ "name": "Youku",
+ "url": "http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
+ "file": "XNDgyMDQ2NTQw_part00.flv",
+ "md5": "ffe3f2e435663dc2d1eea34faeff5b5b"
}
-]
\ No newline at end of file
+]
template_dict['epoch'] = int(time.time())
template_dict['autonumber'] = u'%05d' % self._num_downloads
- template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items())
- template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
+ sanitize = lambda k,v: sanitize_filename(
+ u'NA' if v is None else compat_str(v),
+ restricted=self.params.get('restrictfilenames'),
+ is_id=(k==u'id'))
+ template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
filename = self.params['outtmpl'] % template_dict
return filename
class YoutubePlaylistIE(InfoExtractor):
"""Information Extractor for YouTube playlists."""
- _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
+ _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*'
_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
_VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s'
_MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
def __init__(self, downloader=None):
InfoExtractor.__init__(self, downloader)
- def report_webpage(self, video_id):
+ def report_resolve(self, video_id):
"""Report information extraction."""
- self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+ self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id))
def report_extraction(self, video_id):
"""Report information extraction."""
- self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+ self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id))
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
return
# extract uploader (which is in the url)
- uploader = mobj.group(1).decode('utf-8')
+ uploader = mobj.group(1)
# extract simple title (uploader + slug of song title)
- slug_title = mobj.group(2).decode('utf-8')
+ slug_title = mobj.group(2)
simple_title = uploader + u'-' + slug_title
- self.report_webpage('%s/%s' % (uploader, slug_title))
+ self.report_resolve('%s/%s' % (uploader, slug_title))
- request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
+ url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title)
+ resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
+ request = compat_urllib_request.Request(resolv_url)
try:
- webpage = compat_urllib_request.urlopen(request).read()
+ info_json_bytes = compat_urllib_request.urlopen(request).read()
+ info_json = info_json_bytes.decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return
+ info = json.loads(info_json)
+ video_id = info['id']
self.report_extraction('%s/%s' % (uploader, slug_title))
- # extract uid and stream token that soundcloud hands out for access
- mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
- if mobj:
- video_id = mobj.group(1)
- stream_token = mobj.group(2)
-
- # extract unsimplified title
- mobj = re.search('"title":"(.*?)",', webpage)
- if mobj:
- title = mobj.group(1).decode('utf-8')
- else:
- title = simple_title
-
- # construct media url (with uid/token)
- mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
- mediaURL = mediaURL % (video_id, stream_token)
-
- # description
- description = u'No description available'
- mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
- if mobj:
- description = mobj.group(1)
-
- # upload date
- upload_date = None
- mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
- if mobj:
- try:
- upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
- except Exception as err:
- self._downloader.to_stderr(compat_str(err))
+ streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
+ request = compat_urllib_request.Request(streams_url)
+ try:
+ stream_json_bytes = compat_urllib_request.urlopen(request).read()
+ stream_json = stream_json_bytes.decode('utf-8')
+ except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+ return
- # for soundcloud, a request to a cross domain is required for cookies
- request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
+ streams = json.loads(stream_json)
+ mediaURL = streams['http_mp3_128_url']
return [{
- 'id': video_id.decode('utf-8'),
+ 'id': info['id'],
'url': mediaURL,
- 'uploader': uploader.decode('utf-8'),
- 'upload_date': upload_date,
- 'title': title,
+ 'uploader': info['user']['username'],
+ 'upload_date': info['created_at'],
+ 'title': info['title'],
'ext': u'mp3',
- 'description': description.decode('utf-8')
+ 'description': info['description'],
}]
action='store_true', dest='autonumber',
help='number downloaded files starting from 00000', default=False)
filesystem.add_option('-o', '--output',
- dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
+ dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')
filesystem.add_option('--restrict-filenames',
action='store_true', dest='restrictfilenames',
help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
timestamp = email.utils.mktime_tz(timetuple)
return timestamp
-def sanitize_filename(s, restricted=False):
+def sanitize_filename(s, restricted=False, is_id=False):
"""Sanitizes a string so it could be used as part of a filename.
If restricted is set, use a stricter subset of allowed characters.
+ Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
"""
def replace_insane(char):
if char == '?' or ord(char) < 32 or ord(char) == 127:
return char
result = u''.join(map(replace_insane, s))
- while '__' in result:
- result = result.replace('__', '_')
- result = result.strip('_')
- # Common case of "Foreign band name - English song title"
- if restricted and result.startswith('-_'):
- result = result[2:]
- if not result:
- result = '_'
+ if not is_id:
+ while '__' in result:
+ result = result.replace('__', '_')
+ result = result.strip('_')
+ # Common case of "Foreign band name - English song title"
+ if restricted and result.startswith('-_'):
+ result = result[2:]
+ if not result:
+ result = '_'
return result
def orderedSet(iterable):
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
resp.msg = old_resp.msg
return resp
+
+ https_request = http_request
+ https_response = http_response
projects / youtube-dl / commitdiff