_ Git - youtube-dl/blob - youtube_dl/extractor/frontendmasters.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import (
   8     compat_str,
   9     compat_urlparse,
  10 )
  11 from ..utils import (
  12     ExtractorError,
  13     parse_duration,
  14     urlencode_postdata,
  15 )
  16
  17
  18 class FrontendMastersBaseIE(InfoExtractor):
  19     _API_BASE = 'https://api.frontendmasters.com/v1/kabuki'
  20     _LOGIN_URL = 'https://frontendmasters.com/login/'
  21
  22     _NETRC_MACHINE = 'frontendmasters'
  23
  24     _QUALITIES = {
  25         'low': {'width': 480, 'height': 360},
  26         'mid': {'width': 1280, 'height': 720},
  27         'high': {'width': 1920, 'height': 1080}
  28     }
  29
  30     def _real_initialize(self):
  31         self._login()
  32
  33     def _login(self):
  34         (username, password) = self._get_login_info()
  35         if username is None:
  36             return
  37
  38         login_page = self._download_webpage(
  39             self._LOGIN_URL, None, 'Downloading login page')
  40
  41         login_form = self._hidden_inputs(login_page)
  42
  43         login_form.update({
  44             'username': username,
  45             'password': password
  46         })
  47
  48         post_url = self._search_regex(
  49             r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
  50             'post_url', default=self._LOGIN_URL, group='url')
  51
  52         if not post_url.startswith('http'):
  53             post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
  54
  55         response = self._download_webpage(
  56             post_url, None, 'Logging in', data=urlencode_postdata(login_form),
  57             headers={'Content-Type': 'application/x-www-form-urlencoded'})
  58
  59         # Successful login
  60         if any(p in response for p in (
  61                 'wp-login.php?action=logout', '>Logout')):
  62             return
  63
  64         error = self._html_search_regex(
  65             r'class=(["\'])(?:(?!\1).)*\bMessageAlert\b(?:(?!\1).)*\1[^>]*>(?P<error>[^<]+)<',
  66             response, 'error message', default=None, group='error')
  67         if error:
  68             raise ExtractorError('Unable to login: %s' % error, expected=True)
  69         raise ExtractorError('Unable to log in')
  70
  71
  72 class FrontendMastersPageBaseIE(FrontendMastersBaseIE):
  73     def _download_course(self, course_name, url):
  74         return self._download_json(
  75             '%s/courses/%s' % (self._API_BASE, course_name), course_name,
  76             'Downloading course JSON', headers={'Referer': url})
  77
  78     @staticmethod
  79     def _extract_chapters(course):
  80         chapters = []
  81         lesson_elements = course.get('lessonElements')
  82         if isinstance(lesson_elements, list):
  83             chapters = [e for e in lesson_elements if isinstance(e, compat_str)]
  84         return chapters
  85
  86     @staticmethod
  87     def _extract_lesson(chapters, lesson_id, lesson):
  88         title = lesson.get('title') or lesson_id
  89         display_id = lesson.get('slug')
  90         description = lesson.get('description')
  91         thumbnail = lesson.get('thumbnail')
  92
  93         chapter_number = None
  94         index = lesson.get('index')
  95         element_index = lesson.get('elementIndex')
  96         if (isinstance(index, int) and isinstance(element_index, int) and
  97                 index < element_index):
  98             chapter_number = element_index - index
  99         chapter = (chapters[chapter_number - 1]
 100                    if chapter_number - 1 < len(chapters) else None)
 101
 102         duration = None
 103         timestamp = lesson.get('timestamp')
 104         if isinstance(timestamp, compat_str):
 105             mobj = re.search(
 106                 r'(?P<start>\d{1,2}:\d{1,2}:\d{1,2})\s*-(?P<end>\s*\d{1,2}:\d{1,2}:\d{1,2})',
 107                 timestamp)
 108             if mobj:
 109                 duration = parse_duration(mobj.group('end')) - parse_duration(
 110                     mobj.group('start'))
 111
 112         return {
 113             '_type': 'url_transparent',
 114             'url': 'frontendmasters:%s' % lesson_id,
 115             'ie_key': FrontendMastersIE.ie_key(),
 116             'id': lesson_id,
 117             'display_id': display_id,
 118             'title': title,
 119             'description': description,
 120             'thumbnail': thumbnail,
 121             'duration': duration,
 122             'chapter': chapter,
 123             'chapter_number': chapter_number,
 124         }
 125
 126
 127 class FrontendMastersIE(FrontendMastersBaseIE):
 128     _VALID_URL = r'(?:frontendmasters:|https?://api\.frontendmasters\.com/v\d+/kabuki/video/)(?P<id>[^/]+)'
 129     _TESTS = [{
 130         'url': 'https://api.frontendmasters.com/v1/kabuki/video/a2qogef6ba',
 131         'md5': '7f161159710d6b7016a4f4af6fcb05e2',
 132         'info_dict': {
 133             'id': 'a2qogef6ba',
 134             'ext': 'mp4',
 135             'title': 'a2qogef6ba',
 136         },
 137         'skip': 'Requires FrontendMasters account credentials',
 138     }, {
 139         'url': 'frontendmasters:a2qogef6ba',
 140         'only_matching': True,
 141     }]
 142
 143     def _real_extract(self, url):
 144         lesson_id = self._match_id(url)
 145
 146         source_url = '%s/video/%s/source' % (self._API_BASE, lesson_id)
 147
 148         formats = []
 149         for ext in ('webm', 'mp4'):
 150             for quality in ('low', 'mid', 'high'):
 151                 resolution = self._QUALITIES[quality].copy()
 152                 format_id = '%s-%s' % (ext, quality)
 153                 format_url = self._download_json(
 154                     source_url, lesson_id,
 155                     'Downloading %s source JSON' % format_id, query={
 156                         'f': ext,
 157                         'r': resolution['height'],
 158                     }, headers={
 159                         'Referer': url,
 160                     }, fatal=False)['url']
 161
 162                 if not format_url:
 163                     continue
 164
 165                 f = resolution.copy()
 166                 f.update({
 167                     'url': format_url,
 168                     'ext': ext,
 169                     'format_id': format_id,
 170                 })
 171                 formats.append(f)
 172         self._sort_formats(formats)
 173
 174         subtitles = {
 175             'en': [{
 176                 'url': '%s/transcripts/%s.vtt' % (self._API_BASE, lesson_id),
 177             }]
 178         }
 179
 180         return {
 181             'id': lesson_id,
 182             'title': lesson_id,
 183             'formats': formats,
 184             'subtitles': subtitles
 185         }
 186
 187
 188 class FrontendMastersLessonIE(FrontendMastersPageBaseIE):
 189     _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<course_name>[^/]+)/(?P<lesson_name>[^/]+)'
 190     _TEST = {
 191         'url': 'https://frontendmasters.com/courses/web-development/tools',
 192         'info_dict': {
 193             'id': 'a2qogef6ba',
 194             'display_id': 'tools',
 195             'ext': 'mp4',
 196             'title': 'Tools',
 197             'description': 'md5:82c1ea6472e88ed5acd1829fe992e4f7',
 198             'thumbnail': r're:^https?://.*\.jpg$',
 199             'chapter': 'Introduction',
 200             'chapter_number': 1,
 201         },
 202         'params': {
 203             'skip_download': True,
 204         },
 205         'skip': 'Requires FrontendMasters account credentials',
 206     }
 207
 208     def _real_extract(self, url):
 209         mobj = re.match(self._VALID_URL, url)
 210         course_name, lesson_name = mobj.group('course_name', 'lesson_name')
 211
 212         course = self._download_course(course_name, url)
 213
 214         lesson_id, lesson = next(
 215             (video_id, data)
 216             for video_id, data in course['lessonData'].items()
 217             if data.get('slug') == lesson_name)
 218
 219         chapters = self._extract_chapters(course)
 220         return self._extract_lesson(chapters, lesson_id, lesson)
 221
 222
 223 class FrontendMastersCourseIE(FrontendMastersPageBaseIE):
 224     _VALID_URL = r'https?://(?:www\.)?frontendmasters\.com/courses/(?P<id>[^/]+)'
 225     _TEST = {
 226         'url': 'https://frontendmasters.com/courses/web-development/',
 227         'info_dict': {
 228             'id': 'web-development',
 229             'title': 'Introduction to Web Development',
 230             'description': 'md5:9317e6e842098bf725d62360e52d49a6',
 231         },
 232         'playlist_count': 81,
 233         'skip': 'Requires FrontendMasters account credentials',
 234     }
 235
 236     @classmethod
 237     def suitable(cls, url):
 238         return False if FrontendMastersLessonIE.suitable(url) else super(
 239             FrontendMastersBaseIE, cls).suitable(url)
 240
 241     def _real_extract(self, url):
 242         course_name = self._match_id(url)
 243
 244         course = self._download_course(course_name, url)
 245
 246         chapters = self._extract_chapters(course)
 247
 248         lessons = sorted(
 249             course['lessonData'].values(), key=lambda data: data['index'])
 250
 251         entries = []
 252         for lesson in lessons:
 253             lesson_name = lesson.get('slug')
 254             if not lesson_name:
 255                 continue
 256             lesson_id = lesson.get('hash') or lesson.get('statsId')
 257             entries.append(self._extract_lesson(chapters, lesson_id, lesson))
 258
 259         title = course.get('title')
 260         description = course.get('description')
 261
 262         return self.playlist_result(entries, course_name, title, description)