_ Git - youtube-dl/blob - youtube_dl/extractor/stanfordoc.py

   1 import re
   2 import socket
   3 import xml.etree.ElementTree
   4
   5 from .common import InfoExtractor
   6 from ..utils import (
   7     compat_http_client,
   8     compat_str,
   9     compat_urllib_error,
  10     compat_urllib_request,
  11
  12     ExtractorError,
  13     orderedSet,
  14     unescapeHTML,
  15 )
  16
  17
  18 class StanfordOpenClassroomIE(InfoExtractor):
  19     """Information extractor for Stanford's Open ClassRoom"""
  20
  21     _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
  22     IE_NAME = u'stanfordoc'
  23     _TEST = {
  24         u'url': u'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100',
  25         u'file': u'PracticalUnix_intro-environment.mp4',
  26         u'md5': u'544a9468546059d4e80d76265b0443b8',
  27         u'info_dict': {
  28             u"title": u"Intro Environment"
  29         }
  30     }
  31
  32     def _real_extract(self, url):
  33         mobj = re.match(self._VALID_URL, url)
  34         if mobj is None:
  35             raise ExtractorError(u'Invalid URL: %s' % url)
  36
  37         if mobj.group('course') and mobj.group('video'): # A specific video
  38             course = mobj.group('course')
  39             video = mobj.group('video')
  40             info = {
  41                 'id': course + '_' + video,
  42                 'uploader': None,
  43                 'upload_date': None,
  44             }
  45
  46             self.report_extraction(info['id'])
  47             baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
  48             xmlUrl = baseUrl + video + '.xml'
  49             try:
  50                 metaXml = compat_urllib_request.urlopen(xmlUrl).read()
  51             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
  52                 raise ExtractorError(u'Unable to download video info XML: %s' % compat_str(err))
  53             mdoc = xml.etree.ElementTree.fromstring(metaXml)
  54             try:
  55                 info['title'] = mdoc.findall('./title')[0].text
  56                 info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
  57             except IndexError:
  58                 raise ExtractorError(u'Invalid metadata XML file')
  59             info['ext'] = info['url'].rpartition('.')[2]
  60             return [info]
  61         elif mobj.group('course'): # A course page
  62             course = mobj.group('course')
  63             info = {
  64                 'id': course,
  65                 'type': 'playlist',
  66                 'uploader': None,
  67                 'upload_date': None,
  68             }
  69
  70             coursepage = self._download_webpage(url, info['id'],
  71                                         note='Downloading course info page',
  72                                         errnote='Unable to download course info page')
  73
  74             info['title'] = self._html_search_regex('<h1>([^<]+)</h1>', coursepage, 'title', default=info['id'])
  75
  76             info['description'] = self._html_search_regex('<description>([^<]+)</description>',
  77                 coursepage, u'description', fatal=False)
  78
  79             links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
  80             info['list'] = [
  81                 {
  82                     'type': 'reference',
  83                     'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
  84                 }
  85                     for vpage in links]
  86             results = []
  87             for entry in info['list']:
  88                 assert entry['type'] == 'reference'
  89                 results += self.extract(entry['url'])
  90             return results
  91         else: # Root page
  92             info = {
  93                 'id': 'Stanford OpenClassroom',
  94                 'type': 'playlist',
  95                 'uploader': None,
  96                 'upload_date': None,
  97             }
  98
  99             self.report_download_webpage(info['id'])
 100             rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
 101             try:
 102                 rootpage = compat_urllib_request.urlopen(rootURL).read()
 103             except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 104                 raise ExtractorError(u'Unable to download course info page: ' + compat_str(err))
 105
 106             info['title'] = info['id']
 107
 108             links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
 109             info['list'] = [
 110                 {
 111                     'type': 'reference',
 112                     'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
 113                 }
 114                     for cpage in links]
 115
 116             results = []
 117             for entry in info['list']:
 118                 assert entry['type'] == 'reference'
 119                 results += self.extract(entry['url'])
 120             return results