[youtube] Fix playlist and feed extraction (closes #25675)

[youtube-dl] / devscripts / check-porn.py

diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py
index 63401fe18a12d3600ee741888656d28b8e52e9df..740f04de0f22ad3ac6352b114b0e8f99cf717a9f 100644 (file)
--- a/devscripts/check-porn.py
+++ b/devscripts/check-porn.py
@@ -1,8 +1,12 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
 
 """
 This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check
 if we are not 'age_limit' tagging some porn site
+
+A second approach implemented relies on a list of porn domains, to activate it
+pass the list filename as the only argument
 """
 
 # Allow direct execution
@@ -10,26 +14,43 @@ import os
 import sys
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import get_testcases
+from test.helper import gettestcases
+from youtube_dl.utils import compat_urllib_parse_urlparse
 from youtube_dl.utils import compat_urllib_request
 
-for test in get_testcases():
-    try:
-        webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
-    except:
-        print('\nFail: {0}'.format(test['name']))
-        continue
+if len(sys.argv) > 1:
+    METHOD = 'LIST'
+    LIST = open(sys.argv[1]).read().decode('utf8').strip()
+else:
+    METHOD = 'EURISTIC'
+
+for test in gettestcases():
+    if METHOD == 'EURISTIC':
+        try:
+            webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read()
+        except Exception:
+            print('\nFail: {0}'.format(test['name']))
+            continue
+
+        webpage = webpage.decode('utf8', 'replace')
+
+        RESULT = 'porn' in webpage.lower()
+
+    elif METHOD == 'LIST':
+        domain = compat_urllib_parse_urlparse(test['url']).netloc
+        if not domain:
+            print('\nFail: {0}'.format(test['name']))
+            continue
+        domain = '.'.join(domain.split('.')[-2:])
 
-    webpage = webpage.decode('utf8', 'replace')
+        RESULT = ('.' + domain + '\n' in LIST or '\n' + domain + '\n' in LIST)
 
-    if 'porn' in webpage.lower() and ('info_dict' not in test
-                                      or 'age_limit' not in test['info_dict']
-                                      or test['info_dict']['age_limit'] != 18):
+    if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict']
+                   or test['info_dict']['age_limit'] != 18):
         print('\nPotential missing age_limit check: {0}'.format(test['name']))
 
-    elif 'porn' not in webpage.lower() and ('info_dict' in test and
-                                            'age_limit' in test['info_dict'] and
-                                            test['info_dict']['age_limit'] == 18):
+    elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict']
+                         and test['info_dict']['age_limit'] == 18):
         print('\nPotential false negative: {0}'.format(test['name']))
 
     else: