]> git.bitcoin.ninja Git - youtube-dl/blob - test/test_YoutubeDL.py
[ard] Better format handling
[youtube-dl] / test / test_YoutubeDL.py
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 from __future__ import unicode_literals
5
6 # Allow direct execution
7 import os
8 import sys
9 import unittest
10 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
11
12 import copy
13
14 from test.helper import FakeYDL, assertRegexpMatches
15 from youtube_dl import YoutubeDL
16 from youtube_dl.compat import compat_str, compat_urllib_error
17 from youtube_dl.extractor import YoutubeIE
18 from youtube_dl.extractor.common import InfoExtractor
19 from youtube_dl.postprocessor.common import PostProcessor
20 from youtube_dl.utils import ExtractorError, match_filter_func
21
22 TEST_URL = 'http://localhost/sample.mp4'
23
24
25 class YDL(FakeYDL):
26     def __init__(self, *args, **kwargs):
27         super(YDL, self).__init__(*args, **kwargs)
28         self.downloaded_info_dicts = []
29         self.msgs = []
30
31     def process_info(self, info_dict):
32         self.downloaded_info_dicts.append(info_dict)
33
34     def to_screen(self, msg):
35         self.msgs.append(msg)
36
37
38 def _make_result(formats, **kwargs):
39     res = {
40         'formats': formats,
41         'id': 'testid',
42         'title': 'testttitle',
43         'extractor': 'testex',
44         'extractor_key': 'TestEx',
45     }
46     res.update(**kwargs)
47     return res
48
49
50 class TestFormatSelection(unittest.TestCase):
51     def test_prefer_free_formats(self):
52         # Same resolution => download webm
53         ydl = YDL()
54         ydl.params['prefer_free_formats'] = True
55         formats = [
56             {'ext': 'webm', 'height': 460, 'url': TEST_URL},
57             {'ext': 'mp4', 'height': 460, 'url': TEST_URL},
58         ]
59         info_dict = _make_result(formats)
60         yie = YoutubeIE(ydl)
61         yie._sort_formats(info_dict['formats'])
62         ydl.process_ie_result(info_dict)
63         downloaded = ydl.downloaded_info_dicts[0]
64         self.assertEqual(downloaded['ext'], 'webm')
65
66         # Different resolution => download best quality (mp4)
67         ydl = YDL()
68         ydl.params['prefer_free_formats'] = True
69         formats = [
70             {'ext': 'webm', 'height': 720, 'url': TEST_URL},
71             {'ext': 'mp4', 'height': 1080, 'url': TEST_URL},
72         ]
73         info_dict['formats'] = formats
74         yie = YoutubeIE(ydl)
75         yie._sort_formats(info_dict['formats'])
76         ydl.process_ie_result(info_dict)
77         downloaded = ydl.downloaded_info_dicts[0]
78         self.assertEqual(downloaded['ext'], 'mp4')
79
80         # No prefer_free_formats => prefer mp4 and flv for greater compatibility
81         ydl = YDL()
82         ydl.params['prefer_free_formats'] = False
83         formats = [
84             {'ext': 'webm', 'height': 720, 'url': TEST_URL},
85             {'ext': 'mp4', 'height': 720, 'url': TEST_URL},
86             {'ext': 'flv', 'height': 720, 'url': TEST_URL},
87         ]
88         info_dict['formats'] = formats
89         yie = YoutubeIE(ydl)
90         yie._sort_formats(info_dict['formats'])
91         ydl.process_ie_result(info_dict)
92         downloaded = ydl.downloaded_info_dicts[0]
93         self.assertEqual(downloaded['ext'], 'mp4')
94
95         ydl = YDL()
96         ydl.params['prefer_free_formats'] = False
97         formats = [
98             {'ext': 'flv', 'height': 720, 'url': TEST_URL},
99             {'ext': 'webm', 'height': 720, 'url': TEST_URL},
100         ]
101         info_dict['formats'] = formats
102         yie = YoutubeIE(ydl)
103         yie._sort_formats(info_dict['formats'])
104         ydl.process_ie_result(info_dict)
105         downloaded = ydl.downloaded_info_dicts[0]
106         self.assertEqual(downloaded['ext'], 'flv')
107
108     def test_format_selection(self):
109         formats = [
110             {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
111             {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL},
112             {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
113             {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
114             {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
115         ]
116         info_dict = _make_result(formats)
117
118         ydl = YDL({'format': '20/47'})
119         ydl.process_ie_result(info_dict.copy())
120         downloaded = ydl.downloaded_info_dicts[0]
121         self.assertEqual(downloaded['format_id'], '47')
122
123         ydl = YDL({'format': '20/71/worst'})
124         ydl.process_ie_result(info_dict.copy())
125         downloaded = ydl.downloaded_info_dicts[0]
126         self.assertEqual(downloaded['format_id'], '35')
127
128         ydl = YDL()
129         ydl.process_ie_result(info_dict.copy())
130         downloaded = ydl.downloaded_info_dicts[0]
131         self.assertEqual(downloaded['format_id'], '2')
132
133         ydl = YDL({'format': 'webm/mp4'})
134         ydl.process_ie_result(info_dict.copy())
135         downloaded = ydl.downloaded_info_dicts[0]
136         self.assertEqual(downloaded['format_id'], '47')
137
138         ydl = YDL({'format': '3gp/40/mp4'})
139         ydl.process_ie_result(info_dict.copy())
140         downloaded = ydl.downloaded_info_dicts[0]
141         self.assertEqual(downloaded['format_id'], '35')
142
143         ydl = YDL({'format': 'example-with-dashes'})
144         ydl.process_ie_result(info_dict.copy())
145         downloaded = ydl.downloaded_info_dicts[0]
146         self.assertEqual(downloaded['format_id'], 'example-with-dashes')
147
148     def test_format_selection_audio(self):
149         formats = [
150             {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
151             {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
152             {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': TEST_URL},
153             {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': TEST_URL},
154         ]
155         info_dict = _make_result(formats)
156
157         ydl = YDL({'format': 'bestaudio'})
158         ydl.process_ie_result(info_dict.copy())
159         downloaded = ydl.downloaded_info_dicts[0]
160         self.assertEqual(downloaded['format_id'], 'audio-high')
161
162         ydl = YDL({'format': 'worstaudio'})
163         ydl.process_ie_result(info_dict.copy())
164         downloaded = ydl.downloaded_info_dicts[0]
165         self.assertEqual(downloaded['format_id'], 'audio-low')
166
167         formats = [
168             {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
169             {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': TEST_URL},
170         ]
171         info_dict = _make_result(formats)
172
173         ydl = YDL({'format': 'bestaudio/worstaudio/best'})
174         ydl.process_ie_result(info_dict.copy())
175         downloaded = ydl.downloaded_info_dicts[0]
176         self.assertEqual(downloaded['format_id'], 'vid-high')
177
178     def test_format_selection_audio_exts(self):
179         formats = [
180             {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
181             {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
182             {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
183             {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
184             {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
185         ]
186
187         info_dict = _make_result(formats)
188         ydl = YDL({'format': 'best'})
189         ie = YoutubeIE(ydl)
190         ie._sort_formats(info_dict['formats'])
191         ydl.process_ie_result(copy.deepcopy(info_dict))
192         downloaded = ydl.downloaded_info_dicts[0]
193         self.assertEqual(downloaded['format_id'], 'aac-64')
194
195         ydl = YDL({'format': 'mp3'})
196         ie = YoutubeIE(ydl)
197         ie._sort_formats(info_dict['formats'])
198         ydl.process_ie_result(copy.deepcopy(info_dict))
199         downloaded = ydl.downloaded_info_dicts[0]
200         self.assertEqual(downloaded['format_id'], 'mp3-64')
201
202         ydl = YDL({'prefer_free_formats': True})
203         ie = YoutubeIE(ydl)
204         ie._sort_formats(info_dict['formats'])
205         ydl.process_ie_result(copy.deepcopy(info_dict))
206         downloaded = ydl.downloaded_info_dicts[0]
207         self.assertEqual(downloaded['format_id'], 'ogg-64')
208
209     def test_format_selection_video(self):
210         formats = [
211             {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': TEST_URL},
212             {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': TEST_URL},
213             {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': TEST_URL},
214         ]
215         info_dict = _make_result(formats)
216
217         ydl = YDL({'format': 'bestvideo'})
218         ydl.process_ie_result(info_dict.copy())
219         downloaded = ydl.downloaded_info_dicts[0]
220         self.assertEqual(downloaded['format_id'], 'dash-video-high')
221
222         ydl = YDL({'format': 'worstvideo'})
223         ydl.process_ie_result(info_dict.copy())
224         downloaded = ydl.downloaded_info_dicts[0]
225         self.assertEqual(downloaded['format_id'], 'dash-video-low')
226
227         ydl = YDL({'format': 'bestvideo[format_id^=dash][format_id$=low]'})
228         ydl.process_ie_result(info_dict.copy())
229         downloaded = ydl.downloaded_info_dicts[0]
230         self.assertEqual(downloaded['format_id'], 'dash-video-low')
231
232         formats = [
233             {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
234         ]
235         info_dict = _make_result(formats)
236
237         ydl = YDL({'format': 'bestvideo[vcodec=avc1.123456]'})
238         ydl.process_ie_result(info_dict.copy())
239         downloaded = ydl.downloaded_info_dicts[0]
240         self.assertEqual(downloaded['format_id'], 'vid-vcodec-dot')
241
242     def test_youtube_format_selection(self):
243         order = [
244             '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '17', '36', '13',
245             # Apple HTTP Live Streaming
246             '96', '95', '94', '93', '92', '132', '151',
247             # 3D
248             '85', '84', '102', '83', '101', '82', '100',
249             # Dash video
250             '137', '248', '136', '247', '135', '246',
251             '245', '244', '134', '243', '133', '242', '160',
252             # Dash audio
253             '141', '172', '140', '171', '139',
254         ]
255
256         def format_info(f_id):
257             info = YoutubeIE._formats[f_id].copy()
258
259             # XXX: In real cases InfoExtractor._parse_mpd_formats() fills up 'acodec'
260             # and 'vcodec', while in tests such information is incomplete since
261             # commit a6c2c24479e5f4827ceb06f64d855329c0a6f593
262             # test_YoutubeDL.test_youtube_format_selection is broken without
263             # this fix
264             if 'acodec' in info and 'vcodec' not in info:
265                 info['vcodec'] = 'none'
266             elif 'vcodec' in info and 'acodec' not in info:
267                 info['acodec'] = 'none'
268
269             info['format_id'] = f_id
270             info['url'] = 'url:' + f_id
271             return info
272         formats_order = [format_info(f_id) for f_id in order]
273
274         info_dict = _make_result(list(formats_order), extractor='youtube')
275         ydl = YDL({'format': 'bestvideo+bestaudio'})
276         yie = YoutubeIE(ydl)
277         yie._sort_formats(info_dict['formats'])
278         ydl.process_ie_result(info_dict)
279         downloaded = ydl.downloaded_info_dicts[0]
280         self.assertEqual(downloaded['format_id'], '137+141')
281         self.assertEqual(downloaded['ext'], 'mp4')
282
283         info_dict = _make_result(list(formats_order), extractor='youtube')
284         ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'})
285         yie = YoutubeIE(ydl)
286         yie._sort_formats(info_dict['formats'])
287         ydl.process_ie_result(info_dict)
288         downloaded = ydl.downloaded_info_dicts[0]
289         self.assertEqual(downloaded['format_id'], '38')
290
291         info_dict = _make_result(list(formats_order), extractor='youtube')
292         ydl = YDL({'format': 'bestvideo/best,bestaudio'})
293         yie = YoutubeIE(ydl)
294         yie._sort_formats(info_dict['formats'])
295         ydl.process_ie_result(info_dict)
296         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
297         self.assertEqual(downloaded_ids, ['137', '141'])
298
299         info_dict = _make_result(list(formats_order), extractor='youtube')
300         ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
301         yie = YoutubeIE(ydl)
302         yie._sort_formats(info_dict['formats'])
303         ydl.process_ie_result(info_dict)
304         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
305         self.assertEqual(downloaded_ids, ['137+141', '248+141'])
306
307         info_dict = _make_result(list(formats_order), extractor='youtube')
308         ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'})
309         yie = YoutubeIE(ydl)
310         yie._sort_formats(info_dict['formats'])
311         ydl.process_ie_result(info_dict)
312         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
313         self.assertEqual(downloaded_ids, ['136+141', '247+141'])
314
315         info_dict = _make_result(list(formats_order), extractor='youtube')
316         ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'})
317         yie = YoutubeIE(ydl)
318         yie._sort_formats(info_dict['formats'])
319         ydl.process_ie_result(info_dict)
320         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
321         self.assertEqual(downloaded_ids, ['248+141'])
322
323         for f1, f2 in zip(formats_order, formats_order[1:]):
324             info_dict = _make_result([f1, f2], extractor='youtube')
325             ydl = YDL({'format': 'best/bestvideo'})
326             yie = YoutubeIE(ydl)
327             yie._sort_formats(info_dict['formats'])
328             ydl.process_ie_result(info_dict)
329             downloaded = ydl.downloaded_info_dicts[0]
330             self.assertEqual(downloaded['format_id'], f1['format_id'])
331
332             info_dict = _make_result([f2, f1], extractor='youtube')
333             ydl = YDL({'format': 'best/bestvideo'})
334             yie = YoutubeIE(ydl)
335             yie._sort_formats(info_dict['formats'])
336             ydl.process_ie_result(info_dict)
337             downloaded = ydl.downloaded_info_dicts[0]
338             self.assertEqual(downloaded['format_id'], f1['format_id'])
339
340     def test_audio_only_extractor_format_selection(self):
341         # For extractors with incomplete formats (all formats are audio-only or
342         # video-only) best and worst should fallback to corresponding best/worst
343         # video-only or audio-only formats (as per
344         # https://github.com/rg3/youtube-dl/pull/5556)
345         formats = [
346             {'format_id': 'low', 'ext': 'mp3', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
347             {'format_id': 'high', 'ext': 'mp3', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
348         ]
349         info_dict = _make_result(formats)
350
351         ydl = YDL({'format': 'best'})
352         ydl.process_ie_result(info_dict.copy())
353         downloaded = ydl.downloaded_info_dicts[0]
354         self.assertEqual(downloaded['format_id'], 'high')
355
356         ydl = YDL({'format': 'worst'})
357         ydl.process_ie_result(info_dict.copy())
358         downloaded = ydl.downloaded_info_dicts[0]
359         self.assertEqual(downloaded['format_id'], 'low')
360
361     def test_format_not_available(self):
362         formats = [
363             {'format_id': 'regular', 'ext': 'mp4', 'height': 360, 'url': TEST_URL},
364             {'format_id': 'video', 'ext': 'mp4', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
365         ]
366         info_dict = _make_result(formats)
367
368         # This must fail since complete video-audio format does not match filter
369         # and extractor does not provide incomplete only formats (i.e. only
370         # video-only or audio-only).
371         ydl = YDL({'format': 'best[height>360]'})
372         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
373
374     def test_format_selection_issue_10083(self):
375         # See https://github.com/rg3/youtube-dl/issues/10083
376         formats = [
377             {'format_id': 'regular', 'height': 360, 'url': TEST_URL},
378             {'format_id': 'video', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
379             {'format_id': 'audio', 'vcodec': 'none', 'url': TEST_URL},
380         ]
381         info_dict = _make_result(formats)
382
383         ydl = YDL({'format': 'best[height>360]/bestvideo[height>360]+bestaudio'})
384         ydl.process_ie_result(info_dict.copy())
385         self.assertEqual(ydl.downloaded_info_dicts[0]['format_id'], 'video+audio')
386
387     def test_invalid_format_specs(self):
388         def assert_syntax_error(format_spec):
389             ydl = YDL({'format': format_spec})
390             info_dict = _make_result([{'format_id': 'foo', 'url': TEST_URL}])
391             self.assertRaises(SyntaxError, ydl.process_ie_result, info_dict)
392
393         assert_syntax_error('bestvideo,,best')
394         assert_syntax_error('+bestaudio')
395         assert_syntax_error('bestvideo+')
396         assert_syntax_error('/')
397
398     def test_format_filtering(self):
399         formats = [
400             {'format_id': 'A', 'filesize': 500, 'width': 1000},
401             {'format_id': 'B', 'filesize': 1000, 'width': 500},
402             {'format_id': 'C', 'filesize': 1000, 'width': 400},
403             {'format_id': 'D', 'filesize': 2000, 'width': 600},
404             {'format_id': 'E', 'filesize': 3000},
405             {'format_id': 'F'},
406             {'format_id': 'G', 'filesize': 1000000},
407         ]
408         for f in formats:
409             f['url'] = 'http://_/'
410             f['ext'] = 'unknown'
411         info_dict = _make_result(formats)
412
413         ydl = YDL({'format': 'best[filesize<3000]'})
414         ydl.process_ie_result(info_dict)
415         downloaded = ydl.downloaded_info_dicts[0]
416         self.assertEqual(downloaded['format_id'], 'D')
417
418         ydl = YDL({'format': 'best[filesize<=3000]'})
419         ydl.process_ie_result(info_dict)
420         downloaded = ydl.downloaded_info_dicts[0]
421         self.assertEqual(downloaded['format_id'], 'E')
422
423         ydl = YDL({'format': 'best[filesize <= ? 3000]'})
424         ydl.process_ie_result(info_dict)
425         downloaded = ydl.downloaded_info_dicts[0]
426         self.assertEqual(downloaded['format_id'], 'F')
427
428         ydl = YDL({'format': 'best [filesize = 1000] [width>450]'})
429         ydl.process_ie_result(info_dict)
430         downloaded = ydl.downloaded_info_dicts[0]
431         self.assertEqual(downloaded['format_id'], 'B')
432
433         ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'})
434         ydl.process_ie_result(info_dict)
435         downloaded = ydl.downloaded_info_dicts[0]
436         self.assertEqual(downloaded['format_id'], 'C')
437
438         ydl = YDL({'format': '[filesize>?1]'})
439         ydl.process_ie_result(info_dict)
440         downloaded = ydl.downloaded_info_dicts[0]
441         self.assertEqual(downloaded['format_id'], 'G')
442
443         ydl = YDL({'format': '[filesize<1M]'})
444         ydl.process_ie_result(info_dict)
445         downloaded = ydl.downloaded_info_dicts[0]
446         self.assertEqual(downloaded['format_id'], 'E')
447
448         ydl = YDL({'format': '[filesize<1MiB]'})
449         ydl.process_ie_result(info_dict)
450         downloaded = ydl.downloaded_info_dicts[0]
451         self.assertEqual(downloaded['format_id'], 'G')
452
453         ydl = YDL({'format': 'all[width>=400][width<=600]'})
454         ydl.process_ie_result(info_dict)
455         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
456         self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
457
458         ydl = YDL({'format': 'best[height<40]'})
459         try:
460             ydl.process_ie_result(info_dict)
461         except ExtractorError:
462             pass
463         self.assertEqual(ydl.downloaded_info_dicts, [])
464
465     def test_default_format_spec(self):
466         ydl = YDL({'simulate': True})
467         self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
468
469         ydl = YDL({})
470         self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
471
472         ydl = YDL({'simulate': True})
473         self.assertEqual(ydl._default_format_spec({'is_live': True}), 'bestvideo+bestaudio/best')
474
475         ydl = YDL({'outtmpl': '-'})
476         self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')
477
478         ydl = YDL({})
479         self.assertEqual(ydl._default_format_spec({}, download=False), 'bestvideo+bestaudio/best')
480         self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
481
482
483 class TestYoutubeDL(unittest.TestCase):
484     def test_subtitles(self):
485         def s_formats(lang, autocaption=False):
486             return [{
487                 'ext': ext,
488                 'url': 'http://localhost/video.%s.%s' % (lang, ext),
489                 '_auto': autocaption,
490             } for ext in ['vtt', 'srt', 'ass']]
491         subtitles = dict((l, s_formats(l)) for l in ['en', 'fr', 'es'])
492         auto_captions = dict((l, s_formats(l, True)) for l in ['it', 'pt', 'es'])
493         info_dict = {
494             'id': 'test',
495             'title': 'Test',
496             'url': 'http://localhost/video.mp4',
497             'subtitles': subtitles,
498             'automatic_captions': auto_captions,
499             'extractor': 'TEST',
500         }
501
502         def get_info(params={}):
503             params.setdefault('simulate', True)
504             ydl = YDL(params)
505             ydl.report_warning = lambda *args, **kargs: None
506             return ydl.process_video_result(info_dict, download=False)
507
508         result = get_info()
509         self.assertFalse(result.get('requested_subtitles'))
510         self.assertEqual(result['subtitles'], subtitles)
511         self.assertEqual(result['automatic_captions'], auto_captions)
512
513         result = get_info({'writesubtitles': True})
514         subs = result['requested_subtitles']
515         self.assertTrue(subs)
516         self.assertEqual(set(subs.keys()), set(['en']))
517         self.assertTrue(subs['en'].get('data') is None)
518         self.assertEqual(subs['en']['ext'], 'ass')
519
520         result = get_info({'writesubtitles': True, 'subtitlesformat': 'foo/srt'})
521         subs = result['requested_subtitles']
522         self.assertEqual(subs['en']['ext'], 'srt')
523
524         result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']})
525         subs = result['requested_subtitles']
526         self.assertTrue(subs)
527         self.assertEqual(set(subs.keys()), set(['es', 'fr']))
528
529         result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
530         subs = result['requested_subtitles']
531         self.assertTrue(subs)
532         self.assertEqual(set(subs.keys()), set(['es', 'pt']))
533         self.assertFalse(subs['es']['_auto'])
534         self.assertTrue(subs['pt']['_auto'])
535
536         result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
537         subs = result['requested_subtitles']
538         self.assertTrue(subs)
539         self.assertEqual(set(subs.keys()), set(['es', 'pt']))
540         self.assertTrue(subs['es']['_auto'])
541         self.assertTrue(subs['pt']['_auto'])
542
543     def test_add_extra_info(self):
544         test_dict = {
545             'extractor': 'Foo',
546         }
547         extra_info = {
548             'extractor': 'Bar',
549             'playlist': 'funny videos',
550         }
551         YDL.add_extra_info(test_dict, extra_info)
552         self.assertEqual(test_dict['extractor'], 'Foo')
553         self.assertEqual(test_dict['playlist'], 'funny videos')
554
555     def test_prepare_filename(self):
556         info = {
557             'id': '1234',
558             'ext': 'mp4',
559             'width': None,
560             'height': 1080,
561             'title1': '$PATH',
562             'title2': '%PATH%',
563         }
564
565         def fname(templ):
566             ydl = YoutubeDL({'outtmpl': templ})
567             return ydl.prepare_filename(info)
568         self.assertEqual(fname('%(id)s.%(ext)s'), '1234.mp4')
569         self.assertEqual(fname('%(id)s-%(width)s.%(ext)s'), '1234-NA.mp4')
570         # Replace missing fields with 'NA'
571         self.assertEqual(fname('%(uploader_date)s-%(id)s.%(ext)s'), 'NA-1234.mp4')
572         self.assertEqual(fname('%(height)d.%(ext)s'), '1080.mp4')
573         self.assertEqual(fname('%(height)6d.%(ext)s'), '  1080.mp4')
574         self.assertEqual(fname('%(height)-6d.%(ext)s'), '1080  .mp4')
575         self.assertEqual(fname('%(height)06d.%(ext)s'), '001080.mp4')
576         self.assertEqual(fname('%(height) 06d.%(ext)s'), ' 01080.mp4')
577         self.assertEqual(fname('%(height)   06d.%(ext)s'), ' 01080.mp4')
578         self.assertEqual(fname('%(height)0 6d.%(ext)s'), ' 01080.mp4')
579         self.assertEqual(fname('%(height)0   6d.%(ext)s'), ' 01080.mp4')
580         self.assertEqual(fname('%(height)   0   6d.%(ext)s'), ' 01080.mp4')
581         self.assertEqual(fname('%%'), '%')
582         self.assertEqual(fname('%%%%'), '%%')
583         self.assertEqual(fname('%%(height)06d.%(ext)s'), '%(height)06d.mp4')
584         self.assertEqual(fname('%(width)06d.%(ext)s'), 'NA.mp4')
585         self.assertEqual(fname('%(width)06d.%%(ext)s'), 'NA.%(ext)s')
586         self.assertEqual(fname('%%(width)06d.%(ext)s'), '%(width)06d.mp4')
587         self.assertEqual(fname('Hello %(title1)s'), 'Hello $PATH')
588         self.assertEqual(fname('Hello %(title2)s'), 'Hello %PATH%')
589
590     def test_format_note(self):
591         ydl = YoutubeDL()
592         self.assertEqual(ydl._format_note({}), '')
593         assertRegexpMatches(self, ydl._format_note({
594             'vbr': 10,
595         }), r'^\s*10k$')
596         assertRegexpMatches(self, ydl._format_note({
597             'fps': 30,
598         }), r'^30fps$')
599
600     def test_postprocessors(self):
601         filename = 'post-processor-testfile.mp4'
602         audiofile = filename + '.mp3'
603
604         class SimplePP(PostProcessor):
605             def run(self, info):
606                 with open(audiofile, 'wt') as f:
607                     f.write('EXAMPLE')
608                 return [info['filepath']], info
609
610         def run_pp(params, PP):
611             with open(filename, 'wt') as f:
612                 f.write('EXAMPLE')
613             ydl = YoutubeDL(params)
614             ydl.add_post_processor(PP())
615             ydl.post_process(filename, {'filepath': filename})
616
617         run_pp({'keepvideo': True}, SimplePP)
618         self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
619         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
620         os.unlink(filename)
621         os.unlink(audiofile)
622
623         run_pp({'keepvideo': False}, SimplePP)
624         self.assertFalse(os.path.exists(filename), '%s exists' % filename)
625         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
626         os.unlink(audiofile)
627
628         class ModifierPP(PostProcessor):
629             def run(self, info):
630                 with open(info['filepath'], 'wt') as f:
631                     f.write('MODIFIED')
632                 return [], info
633
634         run_pp({'keepvideo': False}, ModifierPP)
635         self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
636         os.unlink(filename)
637
638     def test_match_filter(self):
639         class FilterYDL(YDL):
640             def __init__(self, *args, **kwargs):
641                 super(FilterYDL, self).__init__(*args, **kwargs)
642                 self.params['simulate'] = True
643
644             def process_info(self, info_dict):
645                 super(YDL, self).process_info(info_dict)
646
647             def _match_entry(self, info_dict, incomplete):
648                 res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
649                 if res is None:
650                     self.downloaded_info_dicts.append(info_dict)
651                 return res
652
653         first = {
654             'id': '1',
655             'url': TEST_URL,
656             'title': 'one',
657             'extractor': 'TEST',
658             'duration': 30,
659             'filesize': 10 * 1024,
660             'playlist_id': '42',
661             'uploader': "變態妍字幕版 太妍 тест",
662             'creator': "тест ' 123 ' тест--",
663         }
664         second = {
665             'id': '2',
666             'url': TEST_URL,
667             'title': 'two',
668             'extractor': 'TEST',
669             'duration': 10,
670             'description': 'foo',
671             'filesize': 5 * 1024,
672             'playlist_id': '43',
673             'uploader': "тест 123",
674         }
675         videos = [first, second]
676
677         def get_videos(filter_=None):
678             ydl = FilterYDL({'match_filter': filter_})
679             for v in videos:
680                 ydl.process_ie_result(v, download=True)
681             return [v['id'] for v in ydl.downloaded_info_dicts]
682
683         res = get_videos()
684         self.assertEqual(res, ['1', '2'])
685
686         def f(v):
687             if v['id'] == '1':
688                 return None
689             else:
690                 return 'Video id is not 1'
691         res = get_videos(f)
692         self.assertEqual(res, ['1'])
693
694         f = match_filter_func('duration < 30')
695         res = get_videos(f)
696         self.assertEqual(res, ['2'])
697
698         f = match_filter_func('description = foo')
699         res = get_videos(f)
700         self.assertEqual(res, ['2'])
701
702         f = match_filter_func('description =? foo')
703         res = get_videos(f)
704         self.assertEqual(res, ['1', '2'])
705
706         f = match_filter_func('filesize > 5KiB')
707         res = get_videos(f)
708         self.assertEqual(res, ['1'])
709
710         f = match_filter_func('playlist_id = 42')
711         res = get_videos(f)
712         self.assertEqual(res, ['1'])
713
714         f = match_filter_func('uploader = "變態妍字幕版 太妍 тест"')
715         res = get_videos(f)
716         self.assertEqual(res, ['1'])
717
718         f = match_filter_func('uploader != "變態妍字幕版 太妍 тест"')
719         res = get_videos(f)
720         self.assertEqual(res, ['2'])
721
722         f = match_filter_func('creator = "тест \' 123 \' тест--"')
723         res = get_videos(f)
724         self.assertEqual(res, ['1'])
725
726         f = match_filter_func("creator = 'тест \\' 123 \\' тест--'")
727         res = get_videos(f)
728         self.assertEqual(res, ['1'])
729
730         f = match_filter_func(r"creator = 'тест \' 123 \' тест--' & duration > 30")
731         res = get_videos(f)
732         self.assertEqual(res, [])
733
734     def test_playlist_items_selection(self):
735         entries = [{
736             'id': compat_str(i),
737             'title': compat_str(i),
738             'url': TEST_URL,
739         } for i in range(1, 5)]
740         playlist = {
741             '_type': 'playlist',
742             'id': 'test',
743             'entries': entries,
744             'extractor': 'test:playlist',
745             'extractor_key': 'test:playlist',
746             'webpage_url': 'http://example.com',
747         }
748
749         def get_ids(params):
750             ydl = YDL(params)
751             # make a copy because the dictionary can be modified
752             ydl.process_ie_result(playlist.copy())
753             return [int(v['id']) for v in ydl.downloaded_info_dicts]
754
755         result = get_ids({})
756         self.assertEqual(result, [1, 2, 3, 4])
757
758         result = get_ids({'playlistend': 10})
759         self.assertEqual(result, [1, 2, 3, 4])
760
761         result = get_ids({'playlistend': 2})
762         self.assertEqual(result, [1, 2])
763
764         result = get_ids({'playliststart': 10})
765         self.assertEqual(result, [])
766
767         result = get_ids({'playliststart': 2})
768         self.assertEqual(result, [2, 3, 4])
769
770         result = get_ids({'playlist_items': '2-4'})
771         self.assertEqual(result, [2, 3, 4])
772
773         result = get_ids({'playlist_items': '2,4'})
774         self.assertEqual(result, [2, 4])
775
776         result = get_ids({'playlist_items': '10'})
777         self.assertEqual(result, [])
778
779         result = get_ids({'playlist_items': '3-10'})
780         self.assertEqual(result, [3, 4])
781
782         result = get_ids({'playlist_items': '2-4,3-4,3'})
783         self.assertEqual(result, [2, 3, 4])
784
785     def test_urlopen_no_file_protocol(self):
786         # see https://github.com/rg3/youtube-dl/issues/8227
787         ydl = YDL()
788         self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
789
790     def test_do_not_override_ie_key_in_url_transparent(self):
791         ydl = YDL()
792
793         class Foo1IE(InfoExtractor):
794             _VALID_URL = r'foo1:'
795
796             def _real_extract(self, url):
797                 return {
798                     '_type': 'url_transparent',
799                     'url': 'foo2:',
800                     'ie_key': 'Foo2',
801                     'title': 'foo1 title',
802                     'id': 'foo1_id',
803                 }
804
805         class Foo2IE(InfoExtractor):
806             _VALID_URL = r'foo2:'
807
808             def _real_extract(self, url):
809                 return {
810                     '_type': 'url',
811                     'url': 'foo3:',
812                     'ie_key': 'Foo3',
813                 }
814
815         class Foo3IE(InfoExtractor):
816             _VALID_URL = r'foo3:'
817
818             def _real_extract(self, url):
819                 return _make_result([{'url': TEST_URL}], title='foo3 title')
820
821         ydl.add_info_extractor(Foo1IE(ydl))
822         ydl.add_info_extractor(Foo2IE(ydl))
823         ydl.add_info_extractor(Foo3IE(ydl))
824         ydl.extract_info('foo1:')
825         downloaded = ydl.downloaded_info_dicts[0]
826         self.assertEqual(downloaded['url'], TEST_URL)
827         self.assertEqual(downloaded['title'], 'foo1 title')
828         self.assertEqual(downloaded['id'], 'testid')
829         self.assertEqual(downloaded['extractor'], 'testex')
830         self.assertEqual(downloaded['extractor_key'], 'TestEx')
831
832
833 if __name__ == '__main__':
834     unittest.main()