]> git.bitcoin.ninja Git - youtube-dl/blob - test/test_YoutubeDL.py
[yourporn] Add support for sxyprn.com (#20646)
[youtube-dl] / test / test_YoutubeDL.py
1 #!/usr/bin/env python
2 # coding: utf-8
3
4 from __future__ import unicode_literals
5
6 # Allow direct execution
7 import os
8 import sys
9 import unittest
10 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
11
12 import copy
13
14 from test.helper import FakeYDL, assertRegexpMatches
15 from youtube_dl import YoutubeDL
16 from youtube_dl.compat import compat_str, compat_urllib_error
17 from youtube_dl.extractor import YoutubeIE
18 from youtube_dl.extractor.common import InfoExtractor
19 from youtube_dl.postprocessor.common import PostProcessor
20 from youtube_dl.utils import ExtractorError, match_filter_func
21
22 TEST_URL = 'http://localhost/sample.mp4'
23
24
25 class YDL(FakeYDL):
26     def __init__(self, *args, **kwargs):
27         super(YDL, self).__init__(*args, **kwargs)
28         self.downloaded_info_dicts = []
29         self.msgs = []
30
31     def process_info(self, info_dict):
32         self.downloaded_info_dicts.append(info_dict)
33
34     def to_screen(self, msg):
35         self.msgs.append(msg)
36
37
38 def _make_result(formats, **kwargs):
39     res = {
40         'formats': formats,
41         'id': 'testid',
42         'title': 'testttitle',
43         'extractor': 'testex',
44         'extractor_key': 'TestEx',
45     }
46     res.update(**kwargs)
47     return res
48
49
50 class TestFormatSelection(unittest.TestCase):
51     def test_prefer_free_formats(self):
52         # Same resolution => download webm
53         ydl = YDL()
54         ydl.params['prefer_free_formats'] = True
55         formats = [
56             {'ext': 'webm', 'height': 460, 'url': TEST_URL},
57             {'ext': 'mp4', 'height': 460, 'url': TEST_URL},
58         ]
59         info_dict = _make_result(formats)
60         yie = YoutubeIE(ydl)
61         yie._sort_formats(info_dict['formats'])
62         ydl.process_ie_result(info_dict)
63         downloaded = ydl.downloaded_info_dicts[0]
64         self.assertEqual(downloaded['ext'], 'webm')
65
66         # Different resolution => download best quality (mp4)
67         ydl = YDL()
68         ydl.params['prefer_free_formats'] = True
69         formats = [
70             {'ext': 'webm', 'height': 720, 'url': TEST_URL},
71             {'ext': 'mp4', 'height': 1080, 'url': TEST_URL},
72         ]
73         info_dict['formats'] = formats
74         yie = YoutubeIE(ydl)
75         yie._sort_formats(info_dict['formats'])
76         ydl.process_ie_result(info_dict)
77         downloaded = ydl.downloaded_info_dicts[0]
78         self.assertEqual(downloaded['ext'], 'mp4')
79
80         # No prefer_free_formats => prefer mp4 and flv for greater compatibility
81         ydl = YDL()
82         ydl.params['prefer_free_formats'] = False
83         formats = [
84             {'ext': 'webm', 'height': 720, 'url': TEST_URL},
85             {'ext': 'mp4', 'height': 720, 'url': TEST_URL},
86             {'ext': 'flv', 'height': 720, 'url': TEST_URL},
87         ]
88         info_dict['formats'] = formats
89         yie = YoutubeIE(ydl)
90         yie._sort_formats(info_dict['formats'])
91         ydl.process_ie_result(info_dict)
92         downloaded = ydl.downloaded_info_dicts[0]
93         self.assertEqual(downloaded['ext'], 'mp4')
94
95         ydl = YDL()
96         ydl.params['prefer_free_formats'] = False
97         formats = [
98             {'ext': 'flv', 'height': 720, 'url': TEST_URL},
99             {'ext': 'webm', 'height': 720, 'url': TEST_URL},
100         ]
101         info_dict['formats'] = formats
102         yie = YoutubeIE(ydl)
103         yie._sort_formats(info_dict['formats'])
104         ydl.process_ie_result(info_dict)
105         downloaded = ydl.downloaded_info_dicts[0]
106         self.assertEqual(downloaded['ext'], 'flv')
107
108     def test_format_selection(self):
109         formats = [
110             {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
111             {'format_id': 'example-with-dashes', 'ext': 'webm', 'preference': 1, 'url': TEST_URL},
112             {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
113             {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
114             {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
115         ]
116         info_dict = _make_result(formats)
117
118         ydl = YDL({'format': '20/47'})
119         ydl.process_ie_result(info_dict.copy())
120         downloaded = ydl.downloaded_info_dicts[0]
121         self.assertEqual(downloaded['format_id'], '47')
122
123         ydl = YDL({'format': '20/71/worst'})
124         ydl.process_ie_result(info_dict.copy())
125         downloaded = ydl.downloaded_info_dicts[0]
126         self.assertEqual(downloaded['format_id'], '35')
127
128         ydl = YDL()
129         ydl.process_ie_result(info_dict.copy())
130         downloaded = ydl.downloaded_info_dicts[0]
131         self.assertEqual(downloaded['format_id'], '2')
132
133         ydl = YDL({'format': 'webm/mp4'})
134         ydl.process_ie_result(info_dict.copy())
135         downloaded = ydl.downloaded_info_dicts[0]
136         self.assertEqual(downloaded['format_id'], '47')
137
138         ydl = YDL({'format': '3gp/40/mp4'})
139         ydl.process_ie_result(info_dict.copy())
140         downloaded = ydl.downloaded_info_dicts[0]
141         self.assertEqual(downloaded['format_id'], '35')
142
143         ydl = YDL({'format': 'example-with-dashes'})
144         ydl.process_ie_result(info_dict.copy())
145         downloaded = ydl.downloaded_info_dicts[0]
146         self.assertEqual(downloaded['format_id'], 'example-with-dashes')
147
148     def test_format_selection_audio(self):
149         formats = [
150             {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
151             {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
152             {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': TEST_URL},
153             {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': TEST_URL},
154         ]
155         info_dict = _make_result(formats)
156
157         ydl = YDL({'format': 'bestaudio'})
158         ydl.process_ie_result(info_dict.copy())
159         downloaded = ydl.downloaded_info_dicts[0]
160         self.assertEqual(downloaded['format_id'], 'audio-high')
161
162         ydl = YDL({'format': 'worstaudio'})
163         ydl.process_ie_result(info_dict.copy())
164         downloaded = ydl.downloaded_info_dicts[0]
165         self.assertEqual(downloaded['format_id'], 'audio-low')
166
167         formats = [
168             {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
169             {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': TEST_URL},
170         ]
171         info_dict = _make_result(formats)
172
173         ydl = YDL({'format': 'bestaudio/worstaudio/best'})
174         ydl.process_ie_result(info_dict.copy())
175         downloaded = ydl.downloaded_info_dicts[0]
176         self.assertEqual(downloaded['format_id'], 'vid-high')
177
178     def test_format_selection_audio_exts(self):
179         formats = [
180             {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
181             {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
182             {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
183             {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
184             {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
185         ]
186
187         info_dict = _make_result(formats)
188         ydl = YDL({'format': 'best'})
189         ie = YoutubeIE(ydl)
190         ie._sort_formats(info_dict['formats'])
191         ydl.process_ie_result(copy.deepcopy(info_dict))
192         downloaded = ydl.downloaded_info_dicts[0]
193         self.assertEqual(downloaded['format_id'], 'aac-64')
194
195         ydl = YDL({'format': 'mp3'})
196         ie = YoutubeIE(ydl)
197         ie._sort_formats(info_dict['formats'])
198         ydl.process_ie_result(copy.deepcopy(info_dict))
199         downloaded = ydl.downloaded_info_dicts[0]
200         self.assertEqual(downloaded['format_id'], 'mp3-64')
201
202         ydl = YDL({'prefer_free_formats': True})
203         ie = YoutubeIE(ydl)
204         ie._sort_formats(info_dict['formats'])
205         ydl.process_ie_result(copy.deepcopy(info_dict))
206         downloaded = ydl.downloaded_info_dicts[0]
207         self.assertEqual(downloaded['format_id'], 'ogg-64')
208
209     def test_format_selection_video(self):
210         formats = [
211             {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': TEST_URL},
212             {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': TEST_URL},
213             {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': TEST_URL},
214         ]
215         info_dict = _make_result(formats)
216
217         ydl = YDL({'format': 'bestvideo'})
218         ydl.process_ie_result(info_dict.copy())
219         downloaded = ydl.downloaded_info_dicts[0]
220         self.assertEqual(downloaded['format_id'], 'dash-video-high')
221
222         ydl = YDL({'format': 'worstvideo'})
223         ydl.process_ie_result(info_dict.copy())
224         downloaded = ydl.downloaded_info_dicts[0]
225         self.assertEqual(downloaded['format_id'], 'dash-video-low')
226
227         ydl = YDL({'format': 'bestvideo[format_id^=dash][format_id$=low]'})
228         ydl.process_ie_result(info_dict.copy())
229         downloaded = ydl.downloaded_info_dicts[0]
230         self.assertEqual(downloaded['format_id'], 'dash-video-low')
231
232         formats = [
233             {'format_id': 'vid-vcodec-dot', 'ext': 'mp4', 'preference': 1, 'vcodec': 'avc1.123456', 'acodec': 'none', 'url': TEST_URL},
234         ]
235         info_dict = _make_result(formats)
236
237         ydl = YDL({'format': 'bestvideo[vcodec=avc1.123456]'})
238         ydl.process_ie_result(info_dict.copy())
239         downloaded = ydl.downloaded_info_dicts[0]
240         self.assertEqual(downloaded['format_id'], 'vid-vcodec-dot')
241
242     def test_format_selection_string_ops(self):
243         formats = [
244             {'format_id': 'abc-cba', 'ext': 'mp4', 'url': TEST_URL},
245             {'format_id': 'zxc-cxz', 'ext': 'webm', 'url': TEST_URL},
246         ]
247         info_dict = _make_result(formats)
248
249         # equals (=)
250         ydl = YDL({'format': '[format_id=abc-cba]'})
251         ydl.process_ie_result(info_dict.copy())
252         downloaded = ydl.downloaded_info_dicts[0]
253         self.assertEqual(downloaded['format_id'], 'abc-cba')
254
255         # does not equal (!=)
256         ydl = YDL({'format': '[format_id!=abc-cba]'})
257         ydl.process_ie_result(info_dict.copy())
258         downloaded = ydl.downloaded_info_dicts[0]
259         self.assertEqual(downloaded['format_id'], 'zxc-cxz')
260
261         ydl = YDL({'format': '[format_id!=abc-cba][format_id!=zxc-cxz]'})
262         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
263
264         # starts with (^=)
265         ydl = YDL({'format': '[format_id^=abc]'})
266         ydl.process_ie_result(info_dict.copy())
267         downloaded = ydl.downloaded_info_dicts[0]
268         self.assertEqual(downloaded['format_id'], 'abc-cba')
269
270         # does not start with (!^=)
271         ydl = YDL({'format': '[format_id!^=abc]'})
272         ydl.process_ie_result(info_dict.copy())
273         downloaded = ydl.downloaded_info_dicts[0]
274         self.assertEqual(downloaded['format_id'], 'zxc-cxz')
275
276         ydl = YDL({'format': '[format_id!^=abc][format_id!^=zxc]'})
277         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
278
279         # ends with ($=)
280         ydl = YDL({'format': '[format_id$=cba]'})
281         ydl.process_ie_result(info_dict.copy())
282         downloaded = ydl.downloaded_info_dicts[0]
283         self.assertEqual(downloaded['format_id'], 'abc-cba')
284
285         # does not end with (!$=)
286         ydl = YDL({'format': '[format_id!$=cba]'})
287         ydl.process_ie_result(info_dict.copy())
288         downloaded = ydl.downloaded_info_dicts[0]
289         self.assertEqual(downloaded['format_id'], 'zxc-cxz')
290
291         ydl = YDL({'format': '[format_id!$=cba][format_id!$=cxz]'})
292         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
293
294         # contains (*=)
295         ydl = YDL({'format': '[format_id*=bc-cb]'})
296         ydl.process_ie_result(info_dict.copy())
297         downloaded = ydl.downloaded_info_dicts[0]
298         self.assertEqual(downloaded['format_id'], 'abc-cba')
299
300         # does not contain (!*=)
301         ydl = YDL({'format': '[format_id!*=bc-cb]'})
302         ydl.process_ie_result(info_dict.copy())
303         downloaded = ydl.downloaded_info_dicts[0]
304         self.assertEqual(downloaded['format_id'], 'zxc-cxz')
305
306         ydl = YDL({'format': '[format_id!*=abc][format_id!*=zxc]'})
307         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
308
309         ydl = YDL({'format': '[format_id!*=-]'})
310         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
311
312     def test_youtube_format_selection(self):
313         order = [
314             '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '17', '36', '13',
315             # Apple HTTP Live Streaming
316             '96', '95', '94', '93', '92', '132', '151',
317             # 3D
318             '85', '84', '102', '83', '101', '82', '100',
319             # Dash video
320             '137', '248', '136', '247', '135', '246',
321             '245', '244', '134', '243', '133', '242', '160',
322             # Dash audio
323             '141', '172', '140', '171', '139',
324         ]
325
326         def format_info(f_id):
327             info = YoutubeIE._formats[f_id].copy()
328
329             # XXX: In real cases InfoExtractor._parse_mpd_formats() fills up 'acodec'
330             # and 'vcodec', while in tests such information is incomplete since
331             # commit a6c2c24479e5f4827ceb06f64d855329c0a6f593
332             # test_YoutubeDL.test_youtube_format_selection is broken without
333             # this fix
334             if 'acodec' in info and 'vcodec' not in info:
335                 info['vcodec'] = 'none'
336             elif 'vcodec' in info and 'acodec' not in info:
337                 info['acodec'] = 'none'
338
339             info['format_id'] = f_id
340             info['url'] = 'url:' + f_id
341             return info
342         formats_order = [format_info(f_id) for f_id in order]
343
344         info_dict = _make_result(list(formats_order), extractor='youtube')
345         ydl = YDL({'format': 'bestvideo+bestaudio'})
346         yie = YoutubeIE(ydl)
347         yie._sort_formats(info_dict['formats'])
348         ydl.process_ie_result(info_dict)
349         downloaded = ydl.downloaded_info_dicts[0]
350         self.assertEqual(downloaded['format_id'], '137+141')
351         self.assertEqual(downloaded['ext'], 'mp4')
352
353         info_dict = _make_result(list(formats_order), extractor='youtube')
354         ydl = YDL({'format': 'bestvideo[height>=999999]+bestaudio/best'})
355         yie = YoutubeIE(ydl)
356         yie._sort_formats(info_dict['formats'])
357         ydl.process_ie_result(info_dict)
358         downloaded = ydl.downloaded_info_dicts[0]
359         self.assertEqual(downloaded['format_id'], '38')
360
361         info_dict = _make_result(list(formats_order), extractor='youtube')
362         ydl = YDL({'format': 'bestvideo/best,bestaudio'})
363         yie = YoutubeIE(ydl)
364         yie._sort_formats(info_dict['formats'])
365         ydl.process_ie_result(info_dict)
366         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
367         self.assertEqual(downloaded_ids, ['137', '141'])
368
369         info_dict = _make_result(list(formats_order), extractor='youtube')
370         ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])+bestaudio'})
371         yie = YoutubeIE(ydl)
372         yie._sort_formats(info_dict['formats'])
373         ydl.process_ie_result(info_dict)
374         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
375         self.assertEqual(downloaded_ids, ['137+141', '248+141'])
376
377         info_dict = _make_result(list(formats_order), extractor='youtube')
378         ydl = YDL({'format': '(bestvideo[ext=mp4],bestvideo[ext=webm])[height<=720]+bestaudio'})
379         yie = YoutubeIE(ydl)
380         yie._sort_formats(info_dict['formats'])
381         ydl.process_ie_result(info_dict)
382         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
383         self.assertEqual(downloaded_ids, ['136+141', '247+141'])
384
385         info_dict = _make_result(list(formats_order), extractor='youtube')
386         ydl = YDL({'format': '(bestvideo[ext=none]/bestvideo[ext=webm])+bestaudio'})
387         yie = YoutubeIE(ydl)
388         yie._sort_formats(info_dict['formats'])
389         ydl.process_ie_result(info_dict)
390         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
391         self.assertEqual(downloaded_ids, ['248+141'])
392
393         for f1, f2 in zip(formats_order, formats_order[1:]):
394             info_dict = _make_result([f1, f2], extractor='youtube')
395             ydl = YDL({'format': 'best/bestvideo'})
396             yie = YoutubeIE(ydl)
397             yie._sort_formats(info_dict['formats'])
398             ydl.process_ie_result(info_dict)
399             downloaded = ydl.downloaded_info_dicts[0]
400             self.assertEqual(downloaded['format_id'], f1['format_id'])
401
402             info_dict = _make_result([f2, f1], extractor='youtube')
403             ydl = YDL({'format': 'best/bestvideo'})
404             yie = YoutubeIE(ydl)
405             yie._sort_formats(info_dict['formats'])
406             ydl.process_ie_result(info_dict)
407             downloaded = ydl.downloaded_info_dicts[0]
408             self.assertEqual(downloaded['format_id'], f1['format_id'])
409
410     def test_audio_only_extractor_format_selection(self):
411         # For extractors with incomplete formats (all formats are audio-only or
412         # video-only) best and worst should fallback to corresponding best/worst
413         # video-only or audio-only formats (as per
414         # https://github.com/ytdl-org/youtube-dl/pull/5556)
415         formats = [
416             {'format_id': 'low', 'ext': 'mp3', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
417             {'format_id': 'high', 'ext': 'mp3', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
418         ]
419         info_dict = _make_result(formats)
420
421         ydl = YDL({'format': 'best'})
422         ydl.process_ie_result(info_dict.copy())
423         downloaded = ydl.downloaded_info_dicts[0]
424         self.assertEqual(downloaded['format_id'], 'high')
425
426         ydl = YDL({'format': 'worst'})
427         ydl.process_ie_result(info_dict.copy())
428         downloaded = ydl.downloaded_info_dicts[0]
429         self.assertEqual(downloaded['format_id'], 'low')
430
431     def test_format_not_available(self):
432         formats = [
433             {'format_id': 'regular', 'ext': 'mp4', 'height': 360, 'url': TEST_URL},
434             {'format_id': 'video', 'ext': 'mp4', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
435         ]
436         info_dict = _make_result(formats)
437
438         # This must fail since complete video-audio format does not match filter
439         # and extractor does not provide incomplete only formats (i.e. only
440         # video-only or audio-only).
441         ydl = YDL({'format': 'best[height>360]'})
442         self.assertRaises(ExtractorError, ydl.process_ie_result, info_dict.copy())
443
444     def test_format_selection_issue_10083(self):
445         # See https://github.com/ytdl-org/youtube-dl/issues/10083
446         formats = [
447             {'format_id': 'regular', 'height': 360, 'url': TEST_URL},
448             {'format_id': 'video', 'height': 720, 'acodec': 'none', 'url': TEST_URL},
449             {'format_id': 'audio', 'vcodec': 'none', 'url': TEST_URL},
450         ]
451         info_dict = _make_result(formats)
452
453         ydl = YDL({'format': 'best[height>360]/bestvideo[height>360]+bestaudio'})
454         ydl.process_ie_result(info_dict.copy())
455         self.assertEqual(ydl.downloaded_info_dicts[0]['format_id'], 'video+audio')
456
457     def test_invalid_format_specs(self):
458         def assert_syntax_error(format_spec):
459             ydl = YDL({'format': format_spec})
460             info_dict = _make_result([{'format_id': 'foo', 'url': TEST_URL}])
461             self.assertRaises(SyntaxError, ydl.process_ie_result, info_dict)
462
463         assert_syntax_error('bestvideo,,best')
464         assert_syntax_error('+bestaudio')
465         assert_syntax_error('bestvideo+')
466         assert_syntax_error('/')
467
468     def test_format_filtering(self):
469         formats = [
470             {'format_id': 'A', 'filesize': 500, 'width': 1000},
471             {'format_id': 'B', 'filesize': 1000, 'width': 500},
472             {'format_id': 'C', 'filesize': 1000, 'width': 400},
473             {'format_id': 'D', 'filesize': 2000, 'width': 600},
474             {'format_id': 'E', 'filesize': 3000},
475             {'format_id': 'F'},
476             {'format_id': 'G', 'filesize': 1000000},
477         ]
478         for f in formats:
479             f['url'] = 'http://_/'
480             f['ext'] = 'unknown'
481         info_dict = _make_result(formats)
482
483         ydl = YDL({'format': 'best[filesize<3000]'})
484         ydl.process_ie_result(info_dict)
485         downloaded = ydl.downloaded_info_dicts[0]
486         self.assertEqual(downloaded['format_id'], 'D')
487
488         ydl = YDL({'format': 'best[filesize<=3000]'})
489         ydl.process_ie_result(info_dict)
490         downloaded = ydl.downloaded_info_dicts[0]
491         self.assertEqual(downloaded['format_id'], 'E')
492
493         ydl = YDL({'format': 'best[filesize <= ? 3000]'})
494         ydl.process_ie_result(info_dict)
495         downloaded = ydl.downloaded_info_dicts[0]
496         self.assertEqual(downloaded['format_id'], 'F')
497
498         ydl = YDL({'format': 'best [filesize = 1000] [width>450]'})
499         ydl.process_ie_result(info_dict)
500         downloaded = ydl.downloaded_info_dicts[0]
501         self.assertEqual(downloaded['format_id'], 'B')
502
503         ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'})
504         ydl.process_ie_result(info_dict)
505         downloaded = ydl.downloaded_info_dicts[0]
506         self.assertEqual(downloaded['format_id'], 'C')
507
508         ydl = YDL({'format': '[filesize>?1]'})
509         ydl.process_ie_result(info_dict)
510         downloaded = ydl.downloaded_info_dicts[0]
511         self.assertEqual(downloaded['format_id'], 'G')
512
513         ydl = YDL({'format': '[filesize<1M]'})
514         ydl.process_ie_result(info_dict)
515         downloaded = ydl.downloaded_info_dicts[0]
516         self.assertEqual(downloaded['format_id'], 'E')
517
518         ydl = YDL({'format': '[filesize<1MiB]'})
519         ydl.process_ie_result(info_dict)
520         downloaded = ydl.downloaded_info_dicts[0]
521         self.assertEqual(downloaded['format_id'], 'G')
522
523         ydl = YDL({'format': 'all[width>=400][width<=600]'})
524         ydl.process_ie_result(info_dict)
525         downloaded_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
526         self.assertEqual(downloaded_ids, ['B', 'C', 'D'])
527
528         ydl = YDL({'format': 'best[height<40]'})
529         try:
530             ydl.process_ie_result(info_dict)
531         except ExtractorError:
532             pass
533         self.assertEqual(ydl.downloaded_info_dicts, [])
534
535     def test_default_format_spec(self):
536         ydl = YDL({'simulate': True})
537         self.assertEqual(ydl._default_format_spec({}), 'bestvideo+bestaudio/best')
538
539         ydl = YDL({})
540         self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
541
542         ydl = YDL({'simulate': True})
543         self.assertEqual(ydl._default_format_spec({'is_live': True}), 'bestvideo+bestaudio/best')
544
545         ydl = YDL({'outtmpl': '-'})
546         self.assertEqual(ydl._default_format_spec({}), 'best/bestvideo+bestaudio')
547
548         ydl = YDL({})
549         self.assertEqual(ydl._default_format_spec({}, download=False), 'bestvideo+bestaudio/best')
550         self.assertEqual(ydl._default_format_spec({'is_live': True}), 'best/bestvideo+bestaudio')
551
552
553 class TestYoutubeDL(unittest.TestCase):
554     def test_subtitles(self):
555         def s_formats(lang, autocaption=False):
556             return [{
557                 'ext': ext,
558                 'url': 'http://localhost/video.%s.%s' % (lang, ext),
559                 '_auto': autocaption,
560             } for ext in ['vtt', 'srt', 'ass']]
561         subtitles = dict((l, s_formats(l)) for l in ['en', 'fr', 'es'])
562         auto_captions = dict((l, s_formats(l, True)) for l in ['it', 'pt', 'es'])
563         info_dict = {
564             'id': 'test',
565             'title': 'Test',
566             'url': 'http://localhost/video.mp4',
567             'subtitles': subtitles,
568             'automatic_captions': auto_captions,
569             'extractor': 'TEST',
570         }
571
572         def get_info(params={}):
573             params.setdefault('simulate', True)
574             ydl = YDL(params)
575             ydl.report_warning = lambda *args, **kargs: None
576             return ydl.process_video_result(info_dict, download=False)
577
578         result = get_info()
579         self.assertFalse(result.get('requested_subtitles'))
580         self.assertEqual(result['subtitles'], subtitles)
581         self.assertEqual(result['automatic_captions'], auto_captions)
582
583         result = get_info({'writesubtitles': True})
584         subs = result['requested_subtitles']
585         self.assertTrue(subs)
586         self.assertEqual(set(subs.keys()), set(['en']))
587         self.assertTrue(subs['en'].get('data') is None)
588         self.assertEqual(subs['en']['ext'], 'ass')
589
590         result = get_info({'writesubtitles': True, 'subtitlesformat': 'foo/srt'})
591         subs = result['requested_subtitles']
592         self.assertEqual(subs['en']['ext'], 'srt')
593
594         result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']})
595         subs = result['requested_subtitles']
596         self.assertTrue(subs)
597         self.assertEqual(set(subs.keys()), set(['es', 'fr']))
598
599         result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
600         subs = result['requested_subtitles']
601         self.assertTrue(subs)
602         self.assertEqual(set(subs.keys()), set(['es', 'pt']))
603         self.assertFalse(subs['es']['_auto'])
604         self.assertTrue(subs['pt']['_auto'])
605
606         result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
607         subs = result['requested_subtitles']
608         self.assertTrue(subs)
609         self.assertEqual(set(subs.keys()), set(['es', 'pt']))
610         self.assertTrue(subs['es']['_auto'])
611         self.assertTrue(subs['pt']['_auto'])
612
613     def test_add_extra_info(self):
614         test_dict = {
615             'extractor': 'Foo',
616         }
617         extra_info = {
618             'extractor': 'Bar',
619             'playlist': 'funny videos',
620         }
621         YDL.add_extra_info(test_dict, extra_info)
622         self.assertEqual(test_dict['extractor'], 'Foo')
623         self.assertEqual(test_dict['playlist'], 'funny videos')
624
625     def test_prepare_filename(self):
626         info = {
627             'id': '1234',
628             'ext': 'mp4',
629             'width': None,
630             'height': 1080,
631             'title1': '$PATH',
632             'title2': '%PATH%',
633         }
634
635         def fname(templ):
636             ydl = YoutubeDL({'outtmpl': templ})
637             return ydl.prepare_filename(info)
638         self.assertEqual(fname('%(id)s.%(ext)s'), '1234.mp4')
639         self.assertEqual(fname('%(id)s-%(width)s.%(ext)s'), '1234-NA.mp4')
640         # Replace missing fields with 'NA'
641         self.assertEqual(fname('%(uploader_date)s-%(id)s.%(ext)s'), 'NA-1234.mp4')
642         self.assertEqual(fname('%(height)d.%(ext)s'), '1080.mp4')
643         self.assertEqual(fname('%(height)6d.%(ext)s'), '  1080.mp4')
644         self.assertEqual(fname('%(height)-6d.%(ext)s'), '1080  .mp4')
645         self.assertEqual(fname('%(height)06d.%(ext)s'), '001080.mp4')
646         self.assertEqual(fname('%(height) 06d.%(ext)s'), ' 01080.mp4')
647         self.assertEqual(fname('%(height)   06d.%(ext)s'), ' 01080.mp4')
648         self.assertEqual(fname('%(height)0 6d.%(ext)s'), ' 01080.mp4')
649         self.assertEqual(fname('%(height)0   6d.%(ext)s'), ' 01080.mp4')
650         self.assertEqual(fname('%(height)   0   6d.%(ext)s'), ' 01080.mp4')
651         self.assertEqual(fname('%%'), '%')
652         self.assertEqual(fname('%%%%'), '%%')
653         self.assertEqual(fname('%%(height)06d.%(ext)s'), '%(height)06d.mp4')
654         self.assertEqual(fname('%(width)06d.%(ext)s'), 'NA.mp4')
655         self.assertEqual(fname('%(width)06d.%%(ext)s'), 'NA.%(ext)s')
656         self.assertEqual(fname('%%(width)06d.%(ext)s'), '%(width)06d.mp4')
657         self.assertEqual(fname('Hello %(title1)s'), 'Hello $PATH')
658         self.assertEqual(fname('Hello %(title2)s'), 'Hello %PATH%')
659
660     def test_format_note(self):
661         ydl = YoutubeDL()
662         self.assertEqual(ydl._format_note({}), '')
663         assertRegexpMatches(self, ydl._format_note({
664             'vbr': 10,
665         }), r'^\s*10k$')
666         assertRegexpMatches(self, ydl._format_note({
667             'fps': 30,
668         }), r'^30fps$')
669
670     def test_postprocessors(self):
671         filename = 'post-processor-testfile.mp4'
672         audiofile = filename + '.mp3'
673
674         class SimplePP(PostProcessor):
675             def run(self, info):
676                 with open(audiofile, 'wt') as f:
677                     f.write('EXAMPLE')
678                 return [info['filepath']], info
679
680         def run_pp(params, PP):
681             with open(filename, 'wt') as f:
682                 f.write('EXAMPLE')
683             ydl = YoutubeDL(params)
684             ydl.add_post_processor(PP())
685             ydl.post_process(filename, {'filepath': filename})
686
687         run_pp({'keepvideo': True}, SimplePP)
688         self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
689         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
690         os.unlink(filename)
691         os.unlink(audiofile)
692
693         run_pp({'keepvideo': False}, SimplePP)
694         self.assertFalse(os.path.exists(filename), '%s exists' % filename)
695         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
696         os.unlink(audiofile)
697
698         class ModifierPP(PostProcessor):
699             def run(self, info):
700                 with open(info['filepath'], 'wt') as f:
701                     f.write('MODIFIED')
702                 return [], info
703
704         run_pp({'keepvideo': False}, ModifierPP)
705         self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
706         os.unlink(filename)
707
708     def test_match_filter(self):
709         class FilterYDL(YDL):
710             def __init__(self, *args, **kwargs):
711                 super(FilterYDL, self).__init__(*args, **kwargs)
712                 self.params['simulate'] = True
713
714             def process_info(self, info_dict):
715                 super(YDL, self).process_info(info_dict)
716
717             def _match_entry(self, info_dict, incomplete):
718                 res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
719                 if res is None:
720                     self.downloaded_info_dicts.append(info_dict)
721                 return res
722
723         first = {
724             'id': '1',
725             'url': TEST_URL,
726             'title': 'one',
727             'extractor': 'TEST',
728             'duration': 30,
729             'filesize': 10 * 1024,
730             'playlist_id': '42',
731             'uploader': "變態妍字幕版 太妍 тест",
732             'creator': "тест ' 123 ' тест--",
733         }
734         second = {
735             'id': '2',
736             'url': TEST_URL,
737             'title': 'two',
738             'extractor': 'TEST',
739             'duration': 10,
740             'description': 'foo',
741             'filesize': 5 * 1024,
742             'playlist_id': '43',
743             'uploader': "тест 123",
744         }
745         videos = [first, second]
746
747         def get_videos(filter_=None):
748             ydl = FilterYDL({'match_filter': filter_})
749             for v in videos:
750                 ydl.process_ie_result(v, download=True)
751             return [v['id'] for v in ydl.downloaded_info_dicts]
752
753         res = get_videos()
754         self.assertEqual(res, ['1', '2'])
755
756         def f(v):
757             if v['id'] == '1':
758                 return None
759             else:
760                 return 'Video id is not 1'
761         res = get_videos(f)
762         self.assertEqual(res, ['1'])
763
764         f = match_filter_func('duration < 30')
765         res = get_videos(f)
766         self.assertEqual(res, ['2'])
767
768         f = match_filter_func('description = foo')
769         res = get_videos(f)
770         self.assertEqual(res, ['2'])
771
772         f = match_filter_func('description =? foo')
773         res = get_videos(f)
774         self.assertEqual(res, ['1', '2'])
775
776         f = match_filter_func('filesize > 5KiB')
777         res = get_videos(f)
778         self.assertEqual(res, ['1'])
779
780         f = match_filter_func('playlist_id = 42')
781         res = get_videos(f)
782         self.assertEqual(res, ['1'])
783
784         f = match_filter_func('uploader = "變態妍字幕版 太妍 тест"')
785         res = get_videos(f)
786         self.assertEqual(res, ['1'])
787
788         f = match_filter_func('uploader != "變態妍字幕版 太妍 тест"')
789         res = get_videos(f)
790         self.assertEqual(res, ['2'])
791
792         f = match_filter_func('creator = "тест \' 123 \' тест--"')
793         res = get_videos(f)
794         self.assertEqual(res, ['1'])
795
796         f = match_filter_func("creator = 'тест \\' 123 \\' тест--'")
797         res = get_videos(f)
798         self.assertEqual(res, ['1'])
799
800         f = match_filter_func(r"creator = 'тест \' 123 \' тест--' & duration > 30")
801         res = get_videos(f)
802         self.assertEqual(res, [])
803
804     def test_playlist_items_selection(self):
805         entries = [{
806             'id': compat_str(i),
807             'title': compat_str(i),
808             'url': TEST_URL,
809         } for i in range(1, 5)]
810         playlist = {
811             '_type': 'playlist',
812             'id': 'test',
813             'entries': entries,
814             'extractor': 'test:playlist',
815             'extractor_key': 'test:playlist',
816             'webpage_url': 'http://example.com',
817         }
818
819         def get_ids(params):
820             ydl = YDL(params)
821             # make a copy because the dictionary can be modified
822             ydl.process_ie_result(playlist.copy())
823             return [int(v['id']) for v in ydl.downloaded_info_dicts]
824
825         result = get_ids({})
826         self.assertEqual(result, [1, 2, 3, 4])
827
828         result = get_ids({'playlistend': 10})
829         self.assertEqual(result, [1, 2, 3, 4])
830
831         result = get_ids({'playlistend': 2})
832         self.assertEqual(result, [1, 2])
833
834         result = get_ids({'playliststart': 10})
835         self.assertEqual(result, [])
836
837         result = get_ids({'playliststart': 2})
838         self.assertEqual(result, [2, 3, 4])
839
840         result = get_ids({'playlist_items': '2-4'})
841         self.assertEqual(result, [2, 3, 4])
842
843         result = get_ids({'playlist_items': '2,4'})
844         self.assertEqual(result, [2, 4])
845
846         result = get_ids({'playlist_items': '10'})
847         self.assertEqual(result, [])
848
849         result = get_ids({'playlist_items': '3-10'})
850         self.assertEqual(result, [3, 4])
851
852         result = get_ids({'playlist_items': '2-4,3-4,3'})
853         self.assertEqual(result, [2, 3, 4])
854
855     def test_urlopen_no_file_protocol(self):
856         # see https://github.com/ytdl-org/youtube-dl/issues/8227
857         ydl = YDL()
858         self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
859
860     def test_do_not_override_ie_key_in_url_transparent(self):
861         ydl = YDL()
862
863         class Foo1IE(InfoExtractor):
864             _VALID_URL = r'foo1:'
865
866             def _real_extract(self, url):
867                 return {
868                     '_type': 'url_transparent',
869                     'url': 'foo2:',
870                     'ie_key': 'Foo2',
871                     'title': 'foo1 title',
872                     'id': 'foo1_id',
873                 }
874
875         class Foo2IE(InfoExtractor):
876             _VALID_URL = r'foo2:'
877
878             def _real_extract(self, url):
879                 return {
880                     '_type': 'url',
881                     'url': 'foo3:',
882                     'ie_key': 'Foo3',
883                 }
884
885         class Foo3IE(InfoExtractor):
886             _VALID_URL = r'foo3:'
887
888             def _real_extract(self, url):
889                 return _make_result([{'url': TEST_URL}], title='foo3 title')
890
891         ydl.add_info_extractor(Foo1IE(ydl))
892         ydl.add_info_extractor(Foo2IE(ydl))
893         ydl.add_info_extractor(Foo3IE(ydl))
894         ydl.extract_info('foo1:')
895         downloaded = ydl.downloaded_info_dicts[0]
896         self.assertEqual(downloaded['url'], TEST_URL)
897         self.assertEqual(downloaded['title'], 'foo1 title')
898         self.assertEqual(downloaded['id'], 'testid')
899         self.assertEqual(downloaded['extractor'], 'testex')
900         self.assertEqual(downloaded['extractor_key'], 'TestEx')
901
902
903 if __name__ == '__main__':
904     unittest.main()