[common] ignore ISO 639-2 generic codes

[youtube-dl] / youtube_dl / extractor / common.py

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1758ab29b89e01f2077b617df7f04119a7197072..ee0e3d8d162b709dde59e01fe0372ef73d78bad9 100644 (file)
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1350,25 +1350,30 @@ class InfoExtractor(object):
         if mpd_doc.get('type') == 'dynamic':
             return []
 
+        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace')
+
+        def _add_ns(path):
+            return self._xpath_ns(path, namespace)
+
         def extract_multisegment_info(element, ms_parent_info):
             ms_info = ms_parent_info.copy()
-            segment_list = element.find(self._xpath_ns('SegmentList', namespace))
+            segment_list = element.find(_add_ns('SegmentList'))
             if segment_list is not None:
-                segment_urls_e = segment_list.findall(self._xpath_ns('SegmentURL', namespace))
+                segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
                 if segment_urls_e:
                     ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
-                initialization = segment_list.find(self._xpath_ns('Initialization', namespace))
+                initialization = segment_list.find(_add_ns('Initialization'))
                 if initialization is not None:
                     ms_info['initialization_url'] = initialization.attrib['sourceURL']
             else:
-                segment_template = element.find(self._xpath_ns('SegmentTemplate', namespace))
+                segment_template = element.find(_add_ns('SegmentTemplate'))
                 if segment_template is not None:
                     start_number = segment_template.get('startNumber')
                     if start_number:
                         ms_info['start_number'] = int(start_number)
-                    segment_timeline = segment_template.find(self._xpath_ns('SegmentTimeline', namespace))
+                    segment_timeline = segment_template.find(_add_ns('SegmentTimeline'))
                     if segment_timeline is not None:
-                        s_e = segment_timeline.findall(self._xpath_ns('S', namespace))
+                        s_e = segment_timeline.findall(_add_ns('S'))
                         if s_e:
                             ms_info['total_number'] = 0
                             for s in s_e:
@@ -1387,23 +1392,22 @@ class InfoExtractor(object):
                     if initialization:
                         ms_info['initialization_url'] = initialization
                     else:
-                        initialization = segment_template.find(self._xpath_ns('Initialization', namespace))
+                        initialization = segment_template.find(_add_ns('Initialization'))
                         if initialization is not None:
                             ms_info['initialization_url'] = initialization.attrib['sourceURL']
             return ms_info
 
-        namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace')
         mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
         formats = []
-        for period in mpd_doc.findall(self._xpath_ns('Period', namespace)):
+        for period in mpd_doc.findall(_add_ns('Period')):
             period_duration = parse_duration(period.get('duration')) or mpd_duration
             period_ms_info = extract_multisegment_info(period, {
                 'start_number': 1,
                 'timescale': 1,
             })
-            for adaptation_set in period.findall(self._xpath_ns('AdaptationSet', namespace)):
+            for adaptation_set in period.findall(_add_ns('AdaptationSet')):
                 adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
-                for representation in adaptation_set.findall(self._xpath_ns('Representation', namespace)):
+                for representation in adaptation_set.findall(_add_ns('Representation')):
                     representation_attrib = adaptation_set.attrib.copy()
                     representation_attrib.update(representation.attrib)
                     mime_type = representation_attrib.get('mimeType')
@@ -1414,7 +1418,7 @@ class InfoExtractor(object):
                     elif content_type == 'video' or content_type == 'audio':
                         base_url = ''
                         for element in (representation, adaptation_set, period, mpd_doc):
-                            base_url_e = element.find(self._xpath_ns('BaseURL', namespace))
+                            base_url_e = element.find(_add_ns('BaseURL'))
                             if base_url_e is not None:
                                 base_url = base_url_e.text + base_url
                                 if re.match(r'^https?://', base_url):
@@ -1422,6 +1426,7 @@ class InfoExtractor(object):
                         if not re.match(r'^https?://', base_url):
                             base_url = mpd_base_url + base_url
                         representation_id = representation_attrib.get('id')
+                        lang = representation_attrib.get('lang')
                         f = {
                             'format_id': mpd_id or representation_id,
                             'url': base_url,
@@ -1432,7 +1437,7 @@ class InfoExtractor(object):
                             'fps': int_or_none(representation_attrib.get('frameRate')),
                             'vcodec': 'none' if content_type == 'audio' else representation_attrib.get('codecs'),
                             'acodec': 'none' if content_type == 'video' else representation_attrib.get('codecs'),
-                            'language': representation_attrib.get('lang'),
+                            'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
                             'format_note': 'DASH %s' % content_type,
                         }
                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
@@ -1442,9 +1447,9 @@ class InfoExtractor(object):
                                 representation_ms_info['total_number'] = int(math.ceil(period_duration / segment_duration))
                             media_template = representation_ms_info['media_template']
                             media_template = media_template.replace('$RepresentationID$', representation_id)
-                            media_template = re.sub(r'\$(Bandwidth)(?:%(0\d+d))?\$', r'%(\1)\2', media_template)
+                            media_template = re.sub(r'\$(Bandwidth)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
                             media_template = media_template % {'Bandwidth': representation_attrib.get('bandwidth')}
-                            media_template = re.sub(r'\$(Number)(?:%(0\d+d))?\$', r'%(\1)\2', media_template)
+                            media_template = re.sub(r'\$(Number)(?:%(0\d+)d)?\$', r'%(\1)\2d', media_template)
                             media_template.replace('$$', '$')
                             representation_ms_info['segment_urls'] = [media_template % {'Number': segment_number} for segment_number in range(representation_ms_info['start_number'], representation_ms_info['total_number'] + representation_ms_info['start_number'])]
                         if 'segment_urls' in representation_ms_info: