Merge pull request #12906 from Tithen-Firion/clean-html-fix

[utils] Fix inconsistent output of clean_html

diff --git a/test/test_utils.py b/test/test_utils.py
index 4cd8188506ad296a407622b759d0ccf89c3c7117..9e1808e6bd10dbdab55884476c6a7d7446128f15 100644 (file)
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -899,6 +899,7 @@ class TestUtil(unittest.TestCase):
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
         self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
+        self.assertEqual(clean_html('a<br>\xa0b'), 'a\nb')
 
     def test_intlist_to_bytes(self):
         self.assertEqual(

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 91e235ff2f6166106c93e4b8de5bbeb18a6d8b7a..41bc205446a7594ece9f14dc43b0617808962053 100644 (file)
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -421,8 +421,8 @@ def clean_html(html):
 
     # Newline vs <br />
     html = html.replace('\n', ' ')
-    html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
-    html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
+    html = re.sub(r'(?u)\s*<\s*br\s*/?\s*>\s*', '\n', html)
+    html = re.sub(r'(?u)<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
     # Strip html tags
     html = re.sub('<.*?>', '', html)
     # Replace html entities