[compat] Add compat_urllib_request_Request

[youtube-dl] / youtube_dl / compat.py

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f39d4e9a9ee1e35e45be35ae122778a0910e3238..4e3de7f5170022bbab637b8163cd0d67cc00719e 100644 (file)
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -198,6 +198,14 @@ except ImportError:  # Python < 3.4
 
             return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
 
+
+# Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
+# unwanted failures due to missing protocol
+def compat_urllib_request_Request(url, *args, **kwargs):
+    return compat_urllib_request.Request(
+        'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
+
+
 try:
     compat_basestring = basestring  # Python 2
 except NameError:
@@ -216,9 +224,19 @@ except ImportError:  # Python 2.6
 if sys.version_info[0] >= 3:
     compat_etree_fromstring = xml.etree.ElementTree.fromstring
 else:
-    # on python 2.x the the attributes of a node aren't always unicode objects
+    # python 2.x tries to encode unicode strings with ascii (see the
+    # XMLParser._fixtext method)
     etree = xml.etree.ElementTree
 
+    try:
+        _etree_iter = etree.Element.iter
+    except AttributeError:  # Python <=2.6
+        def _etree_iter(root):
+            for el in root.findall('*'):
+                yield el
+                for sub in _etree_iter(el):
+                    yield sub
+
     # on 2.6 XML doesn't have a parser argument, function copied from CPython
     # 2.7 source
     def _XML(text, parser=None):
@@ -235,7 +253,11 @@ else:
         return el
 
     def compat_etree_fromstring(text):
-        return _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
+        doc = _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
+        for el in _etree_iter(doc):
+            if el.text is not None and isinstance(el.text, bytes):
+                el.text = el.text.decode('utf-8')
+        return doc
 
 try:
     from urllib.parse import parse_qs as compat_parse_qs