Merge branch 'lecture2go' of https://github.com/nichdu/youtube-dl into nichdu-lecture2go

[youtube-dl] / youtube_dl / extractor / cspan.py

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 795ccd926a8ba5fc648ae91a2eb92bd3e2789e8c..fbefd37d09a98bb19c82b4c09b7b08c99d147d35 100644 (file)
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -4,9 +4,13 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    int_or_none,
     unescapeHTML,
     find_xpath_attr,
+    smuggle_url,
+    determine_ext,
 )
+from .senateisvp import SenateISVPIE
 
 
 class CSpanIE(InfoExtractor):
@@ -26,13 +30,30 @@ class CSpanIE(InfoExtractor):
         'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
         # For whatever reason, the served video alternates between
         # two different ones
-        #'md5': 'dbb0f047376d457f2ab8b3929cbb2d0c',
         'info_dict': {
             'id': '340723',
             'ext': 'mp4',
             'title': 'International Health Care Models',
             'description': 'md5:7a985a2d595dba00af3d9c9f0783c967',
         }
+    }, {
+        'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
+        'md5': '446562a736c6bf97118e389433ed88d4',
+        'info_dict': {
+            'id': '342759',
+            'ext': 'mp4',
+            'title': 'General Motors Ignition Switch Recall',
+            'duration': 14848,
+            'description': 'md5:70c7c3b8fa63fa60d42772440596034c'
+        },
+    }, {
+        # Video from senate.gov
+        'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
+        'info_dict': {
+            'id': 'judiciary031715',
+            'ext': 'flv',
+            'title': 'Immigration Reforms Needed to Protect Skilled American Workers',
+        }
     }]
 
     def _real_extract(self, url):
@@ -49,68 +70,54 @@ class CSpanIE(InfoExtractor):
                 # present, otherwise this is a stripped version
                 r'<p class=\'initial\'>(.*?)</p>'
             ],
-            webpage, 'description', flags=re.DOTALL)
+            webpage, 'description', flags=re.DOTALL, default=None)
 
         info_url = 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=program&id=' + video_id
         data = self._download_json(info_url, video_id)
 
-        url = unescapeHTML(data['video']['files'][0]['path']['#text'])
-
-        doc = self._download_xml('http://www.c-span.org/common/services/flashXml.php?programid=' + video_id + '&version=2014-01-23',
+        doc = self._download_xml(
+            'http://www.c-span.org/common/services/flashXml.php?programid=' + video_id,
             video_id)
 
-        formats = [
-            {
-                'url': url,
-            }
-        ]
-
-        def find_string(node, s):
-            return find_xpath_attr(node, './/string', 'name', s).text
-
-        def find_number(node, s):
-            return int(find_xpath_attr(node, './/number', 'name', s).text)
-
-        def find_array(node, s):
-            return find_xpath_attr(node, './/array', 'name', s)
-
-        def process_files(files, url, formats):
-            for file in files:
-                path = find_string(file, 'path')
-                #duration = find_number(file, './number', 'name', 'length')
-                hd = find_number(file, 'hd')
-                formats.append({
-                    'url': url,
-                    'play_path': path,
-                    'ext': 'flv',
-                    'quality': hd,
-                })
-
-        def process_node(node, formats):
-            url = find_xpath_attr(node, './string', 'name', 'url')
-            if url is None:
-                url = find_xpath_attr(node, './string', 'name', 'URL')
-                if url is None:
-                    return
-            url = url.text.replace('$(protocol)', 'rtmp').replace('$(port)', '1935')
-            files = find_array(node, 'files')
-            if files is None:
-                return
-            process_files(files, url, formats)
-
-        process_node(doc.find('./media-link'), formats)
-
-        streams = find_array(doc, 'streams')
-        if streams is not None:
-            for stream in streams:
-                if find_string(stream, 'name') != 'vod':
-                    continue
-                process_node(stream, formats)
-
-        return {
-            'id': video_id,
-            'title': find_string(doc, 'title'),
+        title = find_xpath_attr(doc, './/string', 'name', 'title').text
+        thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
+
+        senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
+        if senate_isvp_url:
+            surl = smuggle_url(senate_isvp_url, {'force_title': title})
+            return self.url_result(surl, 'SenateISVP', video_id, title)
+
+        files = data['video']['files']
+        try:
+            capfile = data['video']['capfile']['#text']
+        except KeyError:
+            capfile = None
+
+        entries = [{
+            'id': '%s_%d' % (video_id, partnum + 1),
+            'title': (
+                title if len(files) == 1 else
+                '%s part %d' % (title, partnum + 1)),
+            'url': unescapeHTML(f['path']['#text']),
             'description': description,
-            'thumbnail': find_string(doc, 'poster'),
-            'formats': formats,
-        }
+            'thumbnail': thumbnail,
+            'duration': int_or_none(f.get('length', {}).get('#text')),
+            'subtitles': {
+                'en': [{
+                    'url': capfile,
+                    'ext': determine_ext(capfile, 'dfxp')
+                }],
+            } if capfile else None,
+        } for partnum, f in enumerate(files)]
+
+        if len(entries) == 1:
+            entry = dict(entries[0])
+            entry['id'] = video_id
+            return entry
+        else:
+            return {
+                '_type': 'playlist',
+                'entries': entries,
+                'title': title,
+                'id': video_id,
+            }