_ Git - youtube-dl/blob - youtube_dl/extractor/animeondemand.py

   1 from __future__ import unicode_literals
   2
   3 import re
   4
   5 from .common import InfoExtractor
   6 from ..compat import (
   7     compat_urlparse,
   8     compat_str,
   9 )
  10 from ..utils import (
  11     determine_ext,
  12     extract_attributes,
  13     ExtractorError,
  14     sanitized_Request,
  15     urlencode_postdata,
  16 )
  17
  18
  19 class AnimeOnDemandIE(InfoExtractor):
  20     _VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
  21     _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
  22     _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
  23     _NETRC_MACHINE = 'animeondemand'
  24     # German-speaking countries of Europe
  25     _GEO_COUNTRIES = ['AT', 'CH', 'DE', 'LI', 'LU']
  26     _TESTS = [{
  27         # jap, OmU
  28         'url': 'https://www.anime-on-demand.de/anime/161',
  29         'info_dict': {
  30             'id': '161',
  31             'title': 'Grimgar, Ashes and Illusions (OmU)',
  32             'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
  33         },
  34         'playlist_mincount': 4,
  35     }, {
  36         # Film wording is used instead of Episode, ger/jap, Dub/OmU
  37         'url': 'https://www.anime-on-demand.de/anime/39',
  38         'only_matching': True,
  39     }, {
  40         # Episodes without titles, jap, OmU
  41         'url': 'https://www.anime-on-demand.de/anime/162',
  42         'only_matching': True,
  43     }, {
  44         # ger/jap, Dub/OmU, account required
  45         'url': 'https://www.anime-on-demand.de/anime/169',
  46         'only_matching': True,
  47     }, {
  48         # Full length film, non-series, ger/jap, Dub/OmU, account required
  49         'url': 'https://www.anime-on-demand.de/anime/185',
  50         'only_matching': True,
  51     }, {
  52         # Flash videos
  53         'url': 'https://www.anime-on-demand.de/anime/12',
  54         'only_matching': True,
  55     }]
  56
  57     def _login(self):
  58         (username, password) = self._get_login_info()
  59         if username is None:
  60             return
  61
  62         login_page = self._download_webpage(
  63             self._LOGIN_URL, None, 'Downloading login page')
  64
  65         if '>Our licensing terms allow the distribution of animes only to German-speaking countries of Europe' in login_page:
  66             self.raise_geo_restricted(
  67                 '%s is only available in German-speaking countries of Europe' % self.IE_NAME)
  68
  69         login_form = self._form_hidden_inputs('new_user', login_page)
  70
  71         login_form.update({
  72             'user[login]': username,
  73             'user[password]': password,
  74         })
  75
  76         post_url = self._search_regex(
  77             r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
  78             'post url', default=self._LOGIN_URL, group='url')
  79
  80         if not post_url.startswith('http'):
  81             post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
  82
  83         request = sanitized_Request(
  84             post_url, urlencode_postdata(login_form))
  85         request.add_header('Referer', self._LOGIN_URL)
  86
  87         response = self._download_webpage(
  88             request, None, 'Logging in as %s' % username)
  89
  90         if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
  91             error = self._search_regex(
  92                 r'<p class="alert alert-danger">(.+?)</p>',
  93                 response, 'error', default=None)
  94             if error:
  95                 raise ExtractorError('Unable to login: %s' % error, expected=True)
  96             raise ExtractorError('Unable to log in')
  97
  98     def _real_initialize(self):
  99         self._login()
 100
 101     def _real_extract(self, url):
 102         anime_id = self._match_id(url)
 103
 104         webpage = self._download_webpage(url, anime_id)
 105
 106         if 'data-playlist=' not in webpage:
 107             self._download_webpage(
 108                 self._APPLY_HTML5_URL, anime_id,
 109                 'Activating HTML5 beta', 'Unable to apply HTML5 beta')
 110             webpage = self._download_webpage(url, anime_id)
 111
 112         csrf_token = self._html_search_meta(
 113             'csrf-token', webpage, 'csrf token', fatal=True)
 114
 115         anime_title = self._html_search_regex(
 116             r'(?s)<h1[^>]+itemprop="name"[^>]*>(.+?)</h1>',
 117             webpage, 'anime name')
 118         anime_description = self._html_search_regex(
 119             r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
 120             webpage, 'anime description', default=None)
 121
 122         entries = []
 123
 124         def extract_info(html, video_id, num=None):
 125             title, description = [None] * 2
 126             formats = []
 127
 128             for input_ in re.findall(
 129                     r'<input[^>]+class=["\'].*?streamstarter[^>]+>', html):
 130                 attributes = extract_attributes(input_)
 131                 title = attributes.get('data-dialog-header')
 132                 playlist_urls = []
 133                 for playlist_key in ('data-playlist', 'data-otherplaylist', 'data-stream'):
 134                     playlist_url = attributes.get(playlist_key)
 135                     if isinstance(playlist_url, compat_str) and re.match(
 136                             r'/?[\da-zA-Z]+', playlist_url):
 137                         playlist_urls.append(attributes[playlist_key])
 138                 if not playlist_urls:
 139                     continue
 140
 141                 lang = attributes.get('data-lang')
 142                 lang_note = attributes.get('value')
 143
 144                 for playlist_url in playlist_urls:
 145                     kind = self._search_regex(
 146                         r'videomaterialurl/\d+/([^/]+)/',
 147                         playlist_url, 'media kind', default=None)
 148                     format_id_list = []
 149                     if lang:
 150                         format_id_list.append(lang)
 151                     if kind:
 152                         format_id_list.append(kind)
 153                     if not format_id_list and num is not None:
 154                         format_id_list.append(compat_str(num))
 155                     format_id = '-'.join(format_id_list)
 156                     format_note = ', '.join(filter(None, (kind, lang_note)))
 157                     request = sanitized_Request(
 158                         compat_urlparse.urljoin(url, playlist_url),
 159                         headers={
 160                             'X-Requested-With': 'XMLHttpRequest',
 161                             'X-CSRF-Token': csrf_token,
 162                             'Referer': url,
 163                             'Accept': 'application/json, text/javascript, */*; q=0.01',
 164                         })
 165                     playlist = self._download_json(
 166                         request, video_id, 'Downloading %s playlist JSON' % format_id,
 167                         fatal=False)
 168                     if not playlist:
 169                         continue
 170                     stream_url = playlist.get('streamurl')
 171                     if stream_url:
 172                         rtmp = re.search(
 173                             r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+/))(?P<playpath>mp[34]:.+)',
 174                             stream_url)
 175                         if rtmp:
 176                             formats.append({
 177                                 'url': rtmp.group('url'),
 178                                 'app': rtmp.group('app'),
 179                                 'play_path': rtmp.group('playpath'),
 180                                 'page_url': url,
 181                                 'player_url': 'https://www.anime-on-demand.de/assets/jwplayer.flash-55abfb34080700304d49125ce9ffb4a6.swf',
 182                                 'rtmp_real_time': True,
 183                                 'format_id': 'rtmp',
 184                                 'ext': 'flv',
 185                             })
 186                             continue
 187                     start_video = playlist.get('startvideo', 0)
 188                     playlist = playlist.get('playlist')
 189                     if not playlist or not isinstance(playlist, list):
 190                         continue
 191                     playlist = playlist[start_video]
 192                     title = playlist.get('title')
 193                     if not title:
 194                         continue
 195                     description = playlist.get('description')
 196                     for source in playlist.get('sources', []):
 197                         file_ = source.get('file')
 198                         if not file_:
 199                             continue
 200                         ext = determine_ext(file_)
 201                         format_id_list = [lang, kind]
 202                         if ext == 'm3u8':
 203                             format_id_list.append('hls')
 204                         elif source.get('type') == 'video/dash' or ext == 'mpd':
 205                             format_id_list.append('dash')
 206                         format_id = '-'.join(filter(None, format_id_list))
 207                         if ext == 'm3u8':
 208                             file_formats = self._extract_m3u8_formats(
 209                                 file_, video_id, 'mp4',
 210                                 entry_protocol='m3u8_native', m3u8_id=format_id, fatal=False)
 211                         elif source.get('type') == 'video/dash' or ext == 'mpd':
 212                             continue
 213                             file_formats = self._extract_mpd_formats(
 214                                 file_, video_id, mpd_id=format_id, fatal=False)
 215                         else:
 216                             continue
 217                         for f in file_formats:
 218                             f.update({
 219                                 'language': lang,
 220                                 'format_note': format_note,
 221                             })
 222                         formats.extend(file_formats)
 223
 224             return {
 225                 'title': title,
 226                 'description': description,
 227                 'formats': formats,
 228             }
 229
 230         def extract_entries(html, video_id, common_info, num=None):
 231             info = extract_info(html, video_id, num)
 232
 233             if info['formats']:
 234                 self._sort_formats(info['formats'])
 235                 f = common_info.copy()
 236                 f.update(info)
 237                 entries.append(f)
 238
 239             # Extract teaser/trailer only when full episode is not available
 240             if not info['formats']:
 241                 m = re.search(
 242                     r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>(?P<kind>Teaser|Trailer)<',
 243                     html)
 244                 if m:
 245                     f = common_info.copy()
 246                     f.update({
 247                         'id': '%s-%s' % (f['id'], m.group('kind').lower()),
 248                         'title': m.group('title'),
 249                         'url': compat_urlparse.urljoin(url, m.group('href')),
 250                     })
 251                     entries.append(f)
 252
 253         def extract_episodes(html):
 254             for num, episode_html in enumerate(re.findall(
 255                     r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', html), 1):
 256                 episodebox_title = self._search_regex(
 257                     (r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
 258                      r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
 259                     episode_html, 'episodebox title', default=None, group='title')
 260                 if not episodebox_title:
 261                     continue
 262
 263                 episode_number = int(self._search_regex(
 264                     r'(?:Episode|Film)\s*(\d+)',
 265                     episodebox_title, 'episode number', default=num))
 266                 episode_title = self._search_regex(
 267                     r'(?:Episode|Film)\s*\d+\s*-\s*(.+)',
 268                     episodebox_title, 'episode title', default=None)
 269
 270                 video_id = 'episode-%d' % episode_number
 271
 272                 common_info = {
 273                     'id': video_id,
 274                     'series': anime_title,
 275                     'episode': episode_title,
 276                     'episode_number': episode_number,
 277                 }
 278
 279                 extract_entries(episode_html, video_id, common_info)
 280
 281         def extract_film(html, video_id):
 282             common_info = {
 283                 'id': anime_id,
 284                 'title': anime_title,
 285                 'description': anime_description,
 286             }
 287             extract_entries(html, video_id, common_info)
 288
 289         extract_episodes(webpage)
 290
 291         if not entries:
 292             extract_film(webpage, anime_id)
 293
 294         return self.playlist_result(entries, anime_id, anime_title, anime_description)