3 from __future__ import unicode_literals
5 # Allow direct execution
9 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
13 from test.helper import FakeYDL, assertRegexpMatches
14 from youtube_dl import YoutubeDL
15 from youtube_dl.compat import compat_str, compat_urllib_error
16 from youtube_dl.extractor import YoutubeIE
17 from youtube_dl.postprocessor.common import PostProcessor
18 from youtube_dl.utils import ExtractorError, match_filter_func
20 TEST_URL = 'http://localhost/sample.mp4'
24 def __init__(self, *args, **kwargs):
25 super(YDL, self).__init__(*args, **kwargs)
26 self.downloaded_info_dicts = []
29 def process_info(self, info_dict):
30 self.downloaded_info_dicts.append(info_dict)
32 def to_screen(self, msg):
36 def _make_result(formats, **kwargs):
40 'title': 'testttitle',
41 'extractor': 'testex',
47 class TestFormatSelection(unittest.TestCase):
48 def test_prefer_free_formats(self):
49 # Same resolution => download webm
51 ydl.params['prefer_free_formats'] = True
53 {'ext': 'webm', 'height': 460, 'url': TEST_URL},
54 {'ext': 'mp4', 'height': 460, 'url': TEST_URL},
56 info_dict = _make_result(formats)
58 yie._sort_formats(info_dict['formats'])
59 ydl.process_ie_result(info_dict)
60 downloaded = ydl.downloaded_info_dicts[0]
61 self.assertEqual(downloaded['ext'], 'webm')
63 # Different resolution => download best quality (mp4)
65 ydl.params['prefer_free_formats'] = True
67 {'ext': 'webm', 'height': 720, 'url': TEST_URL},
68 {'ext': 'mp4', 'height': 1080, 'url': TEST_URL},
70 info_dict['formats'] = formats
72 yie._sort_formats(info_dict['formats'])
73 ydl.process_ie_result(info_dict)
74 downloaded = ydl.downloaded_info_dicts[0]
75 self.assertEqual(downloaded['ext'], 'mp4')
77 # No prefer_free_formats => prefer mp4 and flv for greater compatibility
79 ydl.params['prefer_free_formats'] = False
81 {'ext': 'webm', 'height': 720, 'url': TEST_URL},
82 {'ext': 'mp4', 'height': 720, 'url': TEST_URL},
83 {'ext': 'flv', 'height': 720, 'url': TEST_URL},
85 info_dict['formats'] = formats
87 yie._sort_formats(info_dict['formats'])
88 ydl.process_ie_result(info_dict)
89 downloaded = ydl.downloaded_info_dicts[0]
90 self.assertEqual(downloaded['ext'], 'mp4')
93 ydl.params['prefer_free_formats'] = False
95 {'ext': 'flv', 'height': 720, 'url': TEST_URL},
96 {'ext': 'webm', 'height': 720, 'url': TEST_URL},
98 info_dict['formats'] = formats
100 yie._sort_formats(info_dict['formats'])
101 ydl.process_ie_result(info_dict)
102 downloaded = ydl.downloaded_info_dicts[0]
103 self.assertEqual(downloaded['ext'], 'flv')
105 def test_format_selection(self):
107 {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
108 {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL},
109 {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
110 {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
111 {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
113 info_dict = _make_result(formats)
115 ydl = YDL({'format': '20/47'})
116 ydl.process_ie_result(info_dict.copy())
117 downloaded = ydl.downloaded_info_dicts[0]
118 self.assertEqual(downloaded['format_id'], '47')
120 ydl = YDL({'format': '20/71/worst'})
121 ydl.process_ie_result(info_dict.copy())
122 downloaded = ydl.downloaded_info_dicts[0]
123 self.assertEqual(downloaded['format_id'], '35')
126 ydl.process_ie_result(info_dict.copy())
127 downloaded = ydl.downloaded_info_dicts[0]
128 self.assertEqual(downloaded['format_id'], '2')
130 ydl = YDL({'format': 'webm/mp4'})
131 ydl.process_ie_result(info_dict.copy())
132 downloaded = ydl.downloaded_info_dicts[0]
133 self.assertEqual(downloaded['format_id'], '47')
135 ydl = YDL({'format': '3gp/40/mp4'})
136 ydl.process_ie_result(info_dict.copy())
137 downloaded = ydl.downloaded_info_dicts[0]
138 self.assertEqual(downloaded['format_id'], '35')
140 ydl = YDL({'format': 'example-with-dashes'})
141 ydl.process_ie_result(info_dict.copy())
142 downloaded = ydl.downloaded_info_dicts[0]
143 self.assertEqual(downloaded['format_id'], 'example-with-dashes')
145 def test_format_selection_audio(self):
147 {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
148 {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
149 {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': TEST_URL},
150 {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': TEST_URL},
152 info_dict = _make_result(formats)
154 ydl = YDL({'format': 'bestaudio'})
155 ydl.process_ie_result(info_dict.copy())
156 downloaded = ydl.downloaded_info_dicts[0]
157 self.assertEqual(downloaded['format_id'], 'audio-high')
159 ydl = YDL({'format': 'worstaudio'})
160 ydl.process_ie_result(info_dict.copy())
161 downloaded = ydl.downloaded_info_dicts[0]
162 self.assertEqual(downloaded['format_id'], 'audio-low')
165 {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
166 {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': TEST_URL},
168 info_dict = _make_result(formats)
170 ydl = YDL({'format': 'bestaudio/worstaudio/best'})
171 ydl.process_ie_result(info_dict.copy())
172 downloaded = ydl.downloaded_info_dicts[0]
173 self.assertEqual(downloaded['format_id'], 'vid-high')
175 def test_format_selection_audio_exts(self):
177 {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
178 {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
179 {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
180 {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
181 {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
184 info_dict = _make_result(formats)
185 ydl = YDL({'format': 'best'})
187 ie._sort_formats(info_dict['formats'])
188 ydl.process_ie_result(copy.deepcopy(info_dict))
189 downloaded = ydl.downloaded_info_dicts[0]
190 self.assertEqual(downloaded['format_id'], 'aac-64')
192 ydl = YDL({'format': 'mp3'})
194 ie._sort_formats(info_dict['formats'])
195 ydl.process_ie_result(copy.deepcopy(info_dict))
196 downloaded = ydl.downloaded_info_dicts[0]
197 self.assertEqual(downloaded['format_id'], 'mp3-64')
199 ydl = YDL({'prefer_free_formats': True})
201 ie._sort_formats(info_dict['formats'])
202 ydl.process_ie_result(copy.deepcopy(info_dict))
203 downloaded = ydl.downloaded_info_dicts[0]
204 self.assertEqual(downloaded['format_id'], 'ogg-64')
206 def test_format_selection_video(self):
208 {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': TEST_URL},
209 {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': TEST_URL},
210 {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': TEST_URL},
212 info_dict = _make_result(formats)
214 ydl = YDL({'format': 'bestvideo'})
215 ydl.process_ie_result(info_dict.copy())
216 downloaded = ydl.downloaded_info_dicts[0]
217 self.assertEqual(downloaded['format_id'], 'dash-video-high')
219 ydl = YDL({'format': 'worstvideo'})
220 ydl.process_ie_result(info_dict.copy())
221 downloaded = ydl.downloaded_info_dicts[0]
222 self.assertEqual(downloaded['format_id'], 'dash-video-low')
225 {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
227 info_dict = _make_result(formats)
229 ydl = YDL({'format': 'bestvideo[vcodec=avc1.123456]'})
230 ydl.process_ie_result(info_dict.copy())
231 downloaded = ydl.downloaded_info_dicts[0]
232 self.assertEqual(downloaded['format_id'], 'vid-vcodec-dot')
234 def test_youtube_format_selection(self):
236 '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '36', '17', '13',
237 # Apple HTTP Live Streaming
238 '96', '95', '94', '93', '92', '132', '151',
240 '85', '84', '102', '83', '101', '82', '100',
242 '137', '248', '136', '247', '135', '246',
243 '245', '244', '134', '243', '133', '242', '160',
245 '141', '172', '140', '171', '139',
248 def format_info(f_id):
249 info = YoutubeIE._formats[f_id].copy()
250 info['format_id'] = f_id
251 info['url'] = 'url:' + f_id
253 formats_order = [format_info(f_id) for f_id in order]
255 info_dict = _make_result(list(formats_order), extractor='youtube')
256 ydl = YDL({'format': 'bestvideo+bestaudio'})
258 yie._sort_formats(info_dict['formats'])
259 ydl.process_ie_result(info_dict)
260 downloaded = ydl.downloaded_info_dicts[0]
261 self.assertEqual(downloaded['format_id'], '137+141')
262 self.assertEqual(downloaded['ext'], 'mp4')
264 info_dict = _make_result(list(formats_order), extractor='youtube')
265 ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'})
267 yie._sort_formats(info_dict['formats'])
268 ydl.process_ie_result(info_dict)
269 downloaded = ydl.downloaded_info_dicts[0]
270 self.assertEqual(downloaded['format_id'], '38')
272 info_dict = _make_result(list(formats_order), extractor='youtube')
273 ydl = YDL({'format': 'bestvideo/best,bestaudio'})
275 yie._sort_formats(info_dict['formats'])
276 ydl.process_ie_result(info_dict)
277 downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
278 self.assertEqual(downloaded_ids, ['137', '141'])
280 info_dict = _make_result(list(formats_order), extractor='youtube')
281 ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
283 yie._sort_formats(info_dict['formats'])
284 ydl.process_ie_result(info_dict)
285 downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
286 self.assertEqual(downloaded_ids, ['137+141', '248+141'])
288 info_dict = _make_result(list(formats_order), extractor='youtube')
289 ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'})
291 yie._sort_formats(info_dict['formats'])
292 ydl.process_ie_result(info_dict)
293 downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
294 self.assertEqual(downloaded_ids, ['136+141', '247+141'])
296 info_dict = _make_result(list(formats_order), extractor='youtube')
297 ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'})
299 yie._sort_formats(info_dict['formats'])
300 ydl.process_ie_result(info_dict)
301 downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
302 self.assertEqual(downloaded_ids, ['248+141'])
304 for f1, f2 in zip(formats_order, formats_order[1:]):
305 info_dict = _make_result([f1, f2], extractor='youtube')
306 ydl = YDL({'format': 'best/bestvideo'})
308 yie._sort_formats(info_dict['formats'])
309 ydl.process_ie_result(info_dict)
310 downloaded = ydl.downloaded_info_dicts[0]
311 self.assertEqual(downloaded['format_id'], f1['format_id'])
313 info_dict = _make_result([f2, f1], extractor='youtube')
314 ydl = YDL({'format': 'best/bestvideo'})
316 yie._sort_formats(info_dict['formats'])
317 ydl.process_ie_result(info_dict)
318 downloaded = ydl.downloaded_info_dicts[0]
319 self.assertEqual(downloaded['format_id'], f1['format_id'])
321 def test_invalid_format_specs(self):
322 def assert_syntax_error(format_spec):
323 ydl = YDL({'format': format_spec})
324 info_dict = _make_result([{'format_id': 'foo', 'url': TEST_URL}])
325 self.assertRaises(SyntaxError, ydl.process_ie_result, info_dict)
327 assert_syntax_error('bestvideo,,best')
328 assert_syntax_error('+bestaudio')
329 assert_syntax_error('bestvideo+')
330 assert_syntax_error('/')
332 def test_format_filtering(self):
334 {'format_id': 'A', 'filesize': 500, 'width': 1000},
335 {'format_id': 'B', 'filesize': 1000, 'width': 500},
336 {'format_id': 'C', 'filesize': 1000, 'width': 400},
337 {'format_id': 'D', 'filesize': 2000, 'width': 600},
338 {'format_id': 'E', 'filesize': 3000},
340 {'format_id': 'G', 'filesize': 1000000},
343 f['url'] = 'http://_/'
345 info_dict = _make_result(formats)
347 ydl = YDL({'format': 'best[filesize<3000]'})
348 ydl.process_ie_result(info_dict)
349 downloaded = ydl.downloaded_info_dicts[0]
350 self.assertEqual(downloaded['format_id'], 'D')
352 ydl = YDL({'format': 'best[filesize<=3000]'})
353 ydl.process_ie_result(info_dict)
354 downloaded = ydl.downloaded_info_dicts[0]
355 self.assertEqual(downloaded['format_id'], 'E')
357 ydl = YDL({'format': 'best[filesize <= ? 3000]'})
358 ydl.process_ie_result(info_dict)
359 downloaded = ydl.downloaded_info_dicts[0]
360 self.assertEqual(downloaded['format_id'], 'F')
362 ydl = YDL({'format': 'best [filesize = 1000] [width>450]'})
363 ydl.process_ie_result(info_dict)
364 downloaded = ydl.downloaded_info_dicts[0]
365 self.assertEqual(downloaded['format_id'], 'B')
367 ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'})
368 ydl.process_ie_result(info_dict)
369 downloaded = ydl.downloaded_info_dicts[0]
370 self.assertEqual(downloaded['format_id'], 'C')
372 ydl = YDL({'format': '[filesize>?1]'})
373 ydl.process_ie_result(info_dict)
374 downloaded = ydl.downloaded_info_dicts[0]
375 self.assertEqual(downloaded['format_id'], 'G')
377 ydl = YDL({'format': '[filesize<1M]'})
378 ydl.process_ie_result(info_dict)
379 downloaded = ydl.downloaded_info_dicts[0]
380 self.assertEqual(downloaded['format_id'], 'E')
382 ydl = YDL({'format': '[filesize<1MiB]'})
383 ydl.process_ie_result(info_dict)
384 downloaded = ydl.downloaded_info_dicts[0]
385 self.assertEqual(downloaded['format_id'], 'G')
387 ydl = YDL({'format': 'all[width>=400][width<=600]'})
388 ydl.process_ie_result(info_dict)
389 downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
390 self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
392 ydl = YDL({'format': 'best[height<40]'})
394 ydl.process_ie_result(info_dict)
395 except ExtractorError:
397 self.assertEqual(ydl.downloaded_info_dicts, [])
400 class TestYoutubeDL(unittest.TestCase):
401 def test_subtitles(self):
402 def s_formats(lang, autocaption=False):
405 'url': 'http://localhost/video.%s.%s' % (lang, ext),
406 '_auto': autocaption,
407 } for ext in ['vtt', 'srt', 'ass']]
408 subtitles = dict((l, s_formats(l)) for l in ['en', 'fr', 'es'])
409 auto_captions = dict((l, s_formats(l, True)) for l in ['it', 'pt', 'es'])
413 'url': 'http://localhost/video.mp4',
414 'subtitles': subtitles,
415 'automatic_captions': auto_captions,
419 def get_info(params={}):
420 params.setdefault('simulate', True)
422 ydl.report_warning = lambda *args, **kargs: None
423 return ydl.process_video_result(info_dict, download=False)
426 self.assertFalse(result.get('requested_subtitles'))
427 self.assertEqual(result['subtitles'], subtitles)
428 self.assertEqual(result['automatic_captions'], auto_captions)
430 result = get_info({'writesubtitles': True})
431 subs = result['requested_subtitles']
432 self.assertTrue(subs)
433 self.assertEqual(set(subs.keys()), set(['en']))
434 self.assertTrue(subs['en'].get('data') is None)
435 self.assertEqual(subs['en']['ext'], 'ass')
437 result = get_info({'writesubtitles': True, 'subtitlesformat': 'foo/srt'})
438 subs = result['requested_subtitles']
439 self.assertEqual(subs['en']['ext'], 'srt')
441 result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']})
442 subs = result['requested_subtitles']
443 self.assertTrue(subs)
444 self.assertEqual(set(subs.keys()), set(['es', 'fr']))
446 result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
447 subs = result['requested_subtitles']
448 self.assertTrue(subs)
449 self.assertEqual(set(subs.keys()), set(['es', 'pt']))
450 self.assertFalse(subs['es']['_auto'])
451 self.assertTrue(subs['pt']['_auto'])
453 result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
454 subs = result['requested_subtitles']
455 self.assertTrue(subs)
456 self.assertEqual(set(subs.keys()), set(['es', 'pt']))
457 self.assertTrue(subs['es']['_auto'])
458 self.assertTrue(subs['pt']['_auto'])
460 def test_add_extra_info(self):
466 'playlist': 'funny videos',
468 YDL.add_extra_info(test_dict, extra_info)
469 self.assertEqual(test_dict['extractor'], 'Foo')
470 self.assertEqual(test_dict['playlist'], 'funny videos')
472 def test_prepare_filename(self):
480 ydl = YoutubeDL({'outtmpl': templ})
481 return ydl.prepare_filename(info)
482 self.assertEqual(fname('%(id)s.%(ext)s'), '1234.mp4')
483 self.assertEqual(fname('%(id)s-%(width)s.%(ext)s'), '1234-NA.mp4')
484 # Replace missing fields with 'NA'
485 self.assertEqual(fname('%(uploader_date)s-%(id)s.%(ext)s'), 'NA-1234.mp4')
487 def test_format_note(self):
489 self.assertEqual(ydl._format_note({}), '')
490 assertRegexpMatches(self, ydl._format_note({
494 def test_postprocessors(self):
495 filename = 'post-processor-testfile.mp4'
496 audiofile = filename + '.mp3'
498 class SimplePP(PostProcessor):
500 with open(audiofile, 'wt') as f:
502 return [info['filepath']], info
504 def run_pp(params, PP):
505 with open(filename, 'wt') as f:
507 ydl = YoutubeDL(params)
508 ydl.add_post_processor(PP())
509 ydl.post_process(filename, {'filepath': filename})
511 run_pp({'keepvideo': True}, SimplePP)
512 self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
513 self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
517 run_pp({'keepvideo': False}, SimplePP)
518 self.assertFalse(os.path.exists(filename), '%s exists' % filename)
519 self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
522 class ModifierPP(PostProcessor):
524 with open(info['filepath'], 'wt') as f:
528 run_pp({'keepvideo': False}, ModifierPP)
529 self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
532 def test_match_filter(self):
533 class FilterYDL(YDL):
534 def __init__(self, *args, **kwargs):
535 super(FilterYDL, self).__init__(*args, **kwargs)
536 self.params['simulate'] = True
538 def process_info(self, info_dict):
539 super(YDL, self).process_info(info_dict)
541 def _match_entry(self, info_dict, incomplete):
542 res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
544 self.downloaded_info_dicts.append(info_dict)
553 'filesize': 10 * 1024,
561 'description': 'foo',
562 'filesize': 5 * 1024,
564 videos = [first, second]
566 def get_videos(filter_=None):
567 ydl = FilterYDL({'match_filter': filter_})
569 ydl.process_ie_result(v, download=True)
570 return [v['id'] for v in ydl.downloaded_info_dicts]
573 self.assertEqual(res, ['1', '2'])
579 return 'Video id is not 1'
581 self.assertEqual(res, ['1'])
583 f = match_filter_func('duration < 30')
585 self.assertEqual(res, ['2'])
587 f = match_filter_func('description = foo')
589 self.assertEqual(res, ['2'])
591 f = match_filter_func('description =? foo')
593 self.assertEqual(res, ['1', '2'])
595 f = match_filter_func('filesize > 5KiB')
597 self.assertEqual(res, ['1'])
599 def test_playlist_items_selection(self):
602 'title': compat_str(i),
604 } for i in range(1, 5)]
609 'extractor': 'test:playlist',
610 'extractor_key': 'test:playlist',
611 'webpage_url': 'http://example.com',
616 # make a copy because the dictionary can be modified
617 ydl.process_ie_result(playlist.copy())
618 return [int(v['id']) for v in ydl.downloaded_info_dicts]
621 self.assertEqual(result, [1, 2, 3, 4])
623 result = get_ids({'playlistend': 10})
624 self.assertEqual(result, [1, 2, 3, 4])
626 result = get_ids({'playlistend': 2})
627 self.assertEqual(result, [1, 2])
629 result = get_ids({'playliststart': 10})
630 self.assertEqual(result, [])
632 result = get_ids({'playliststart': 2})
633 self.assertEqual(result, [2, 3, 4])
635 result = get_ids({'playlist_items': '2-4'})
636 self.assertEqual(result, [2, 3, 4])
638 result = get_ids({'playlist_items': '2,4'})
639 self.assertEqual(result, [2, 4])
641 result = get_ids({'playlist_items': '10'})
642 self.assertEqual(result, [])
644 def test_urlopen_no_file_protocol(self):
645 # see https://github.com/rg3/youtube-dl/issues/8227
647 self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
650 if __name__ == '__main__':