Allow playlist test definitions in test_download.

This moves playlist tests where they belong, i.e. to the extractors themselves.
Additionally, all our network interaction configuration for tests in test_download now applies to playlist tests as well.

diff --git a/test/helper.py b/test/helper.py
index b7299fb82c2e541fc520ba11c5c52d9edcc972e3..22d7638606841bc0250665402e09fb49655e5d89 100644 (file)
--- a/test/helper.py
+++ b/test/helper.py
@@ -117,8 +117,9 @@ def expect_info_dict(self, expected_dict, got_dict):
                 u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
 
     # Check for the presence of mandatory fields
                 u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
 
     # Check for the presence of mandatory fields

-    for key in ('id', 'url', 'title', 'ext'):
-        self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key)
+    if got_dict.get('_type') != 'playlist':
+        for key in ('id', 'url', 'title', 'ext'):
+            self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key)

     # Check for mandatory fields that are automatically set by YoutubeDL
     for key in ['webpage_url', 'extractor', 'extractor_key']:
         self.assertTrue(got_dict.get(key), u'Missing field: %s' % key)
     # Check for mandatory fields that are automatically set by YoutubeDL
     for key in ['webpage_url', 'extractor', 'extractor_key']:
         self.assertTrue(got_dict.get(key), u'Missing field: %s' % key)

diff --git a/test/test_download.py b/test/test_download.py
index d6540588c130f6bafacd4ef7d077e6debf8d911d..c8d4ec2c87c97773d60c52ffb342809b8e2a0ffb 100644 (file)
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -63,15 +63,21 @@ def generator(test_case):
     def test_template(self):
         ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
         other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
     def test_template(self):
         ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
         other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]

+        is_playlist = any(k.startswith('playlist') for k in test_case)
+        test_cases = test_case.get(
+            'playlist', [] if is_playlist else [test_case])
+

         def print_skipping(reason):
             print('Skipping %s: %s' % (test_case['name'], reason))
         if not ie.working():
             print_skipping('IE marked as not _WORKING')
             return
         def print_skipping(reason):
             print('Skipping %s: %s' % (test_case['name'], reason))
         if not ie.working():
             print_skipping('IE marked as not _WORKING')
             return

-        if 'playlist' not in test_case:
-            info_dict = test_case.get('info_dict', {})
-            if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
+
+        for tc in test_cases:
+            info_dict = tc.get('info_dict', {})
+            if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')):

                 raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
                 raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')

+

         if 'skip' in test_case:
             print_skipping(test_case['skip'])
             return
         if 'skip' in test_case:
             print_skipping(test_case['skip'])
             return


@@ -81,6 +87,9 @@ def generator(test_case):
                 return
 
         params = get_params(test_case.get('params', {}))
                 return
 
         params = get_params(test_case.get('params', {}))

+        if is_playlist and 'playlist' not in test_case:
+            params.setdefault('extract_flat', True)
+            params.setdefault('skip_download', True)

 
         ydl = YoutubeDL(params)
         ydl.add_default_info_extractors()
 
         ydl = YoutubeDL(params)
         ydl.add_default_info_extractors()


@@ -93,7 +102,6 @@ def generator(test_case):
         def get_tc_filename(tc):
             return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
 
         def get_tc_filename(tc):
             return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
 

-        test_cases = test_case.get('playlist', [test_case])

         def try_rm_tcs_files():
             for tc in test_cases:
                 tc_filename = get_tc_filename(tc)
         def try_rm_tcs_files():
             for tc in test_cases:
                 tc_filename = get_tc_filename(tc)


@@ -105,7 +113,10 @@ def generator(test_case):
             try_num = 1
             while True:
                 try:
             try_num = 1
             while True:
                 try:

-                    ydl.download([test_case['url']])
+                    # We're not using .download here sine that is just a shim
+                    # for outside error handling, and returns the exit code
+                    # instead of the result dict.
+                    res_dict = ydl.extract_info(test_case['url'])

                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
                     if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
                     if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):


@@ -121,6 +132,17 @@ def generator(test_case):
                 else:
                     break
 
                 else:
                     break
 

+            if is_playlist:
+                self.assertEqual(res_dict['_type'], 'playlist')
+                expect_info_dict(self, test_case.get('info_dict', {}), res_dict)
+            if 'playlist_mincount' in test_case:
+                self.assertGreaterEqual(
+                    len(res_dict['entries']),
+                    test_case['playlist_mincount'],
+                    'Expected at least %d in playlist %s, but got only %d' % (
+                        test_case['playlist_mincount'], test_case['url'],
+                        len(res_dict['entries'])))
+

             for tc in test_cases:
                 tc_filename = get_tc_filename(tc)
                 if not test_case.get('params', {}).get('skip_download', False):
             for tc in test_cases:
                 tc_filename = get_tc_filename(tc)
                 if not test_case.get('params', {}).get('skip_download', False):

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 14a1d06ab1ed3350547822cac71501745a14842a..e7194f3e314b78bd44cb71b755e2de9c666829fe 100755 (executable)
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -162,6 +162,7 @@ class YoutubeDL(object):
     default_search:    Prepend this string if an input url is not valid.
                        'auto' for elaborate guessing
     encoding:          Use this encoding instead of the system-specified.
     default_search:    Prepend this string if an input url is not valid.
                        'auto' for elaborate guessing
     encoding:          Use this encoding instead of the system-specified.

+    extract_flat:      Do not resolve URLs, return the immediate result.

 
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:


@@ -558,7 +559,12 @@ class YoutubeDL(object):
         Returns the resolved ie_result.
         """
 
         Returns the resolved ie_result.
         """
 

-        result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
+        result_type = ie_result.get('_type', 'video')
+
+        if self.params.get('extract_flat', False):
+            if result_type in ('url', 'url_transparent'):
+                return ie_result
+

         if result_type == 'video':
             self.add_extra_info(ie_result, extra_info)
             return self.process_video_result(ie_result, download=download)
         if result_type == 'video':
             self.add_extra_info(ie_result, extra_info)
             return self.process_video_result(ie_result, download=download)