]> git.bitcoin.ninja Git - youtube-dl/blob - youtube_dl/extractor/animeondemand.py
[scrippsnetworks] Add new extractor(closes #19857)(closes #22981)
[youtube-dl] / youtube_dl / extractor / animeondemand.py
1 from __future__ import unicode_literals
2
3 import re
4
5 from .common import InfoExtractor
6 from ..compat import compat_str
7 from ..utils import (
8     determine_ext,
9     extract_attributes,
10     ExtractorError,
11     url_or_none,
12     urlencode_postdata,
13     urljoin,
14 )
15
16
17 class AnimeOnDemandIE(InfoExtractor):
18     _VALID_URL = r'https?://(?:www\.)?anime-on-demand\.de/anime/(?P<id>\d+)'
19     _LOGIN_URL = 'https://www.anime-on-demand.de/users/sign_in'
20     _APPLY_HTML5_URL = 'https://www.anime-on-demand.de/html5apply'
21     _NETRC_MACHINE = 'animeondemand'
22     # German-speaking countries of Europe
23     _GEO_COUNTRIES = ['AT', 'CH', 'DE', 'LI', 'LU']
24     _TESTS = [{
25         # jap, OmU
26         'url': 'https://www.anime-on-demand.de/anime/161',
27         'info_dict': {
28             'id': '161',
29             'title': 'Grimgar, Ashes and Illusions (OmU)',
30             'description': 'md5:6681ce3c07c7189d255ac6ab23812d31',
31         },
32         'playlist_mincount': 4,
33     }, {
34         # Film wording is used instead of Episode, ger/jap, Dub/OmU
35         'url': 'https://www.anime-on-demand.de/anime/39',
36         'only_matching': True,
37     }, {
38         # Episodes without titles, jap, OmU
39         'url': 'https://www.anime-on-demand.de/anime/162',
40         'only_matching': True,
41     }, {
42         # ger/jap, Dub/OmU, account required
43         'url': 'https://www.anime-on-demand.de/anime/169',
44         'only_matching': True,
45     }, {
46         # Full length film, non-series, ger/jap, Dub/OmU, account required
47         'url': 'https://www.anime-on-demand.de/anime/185',
48         'only_matching': True,
49     }, {
50         # Flash videos
51         'url': 'https://www.anime-on-demand.de/anime/12',
52         'only_matching': True,
53     }]
54
55     def _login(self):
56         username, password = self._get_login_info()
57         if username is None:
58             return
59
60         login_page = self._download_webpage(
61             self._LOGIN_URL, None, 'Downloading login page')
62
63         if '>Our licensing terms allow the distribution of animes only to German-speaking countries of Europe' in login_page:
64             self.raise_geo_restricted(
65                 '%s is only available in German-speaking countries of Europe' % self.IE_NAME)
66
67         login_form = self._form_hidden_inputs('new_user', login_page)
68
69         login_form.update({
70             'user[login]': username,
71             'user[password]': password,
72         })
73
74         post_url = self._search_regex(
75             r'<form[^>]+action=(["\'])(?P<url>.+?)\1', login_page,
76             'post url', default=self._LOGIN_URL, group='url')
77
78         if not post_url.startswith('http'):
79             post_url = urljoin(self._LOGIN_URL, post_url)
80
81         response = self._download_webpage(
82             post_url, None, 'Logging in',
83             data=urlencode_postdata(login_form), headers={
84                 'Referer': self._LOGIN_URL,
85             })
86
87         if all(p not in response for p in ('>Logout<', 'href="/users/sign_out"')):
88             error = self._search_regex(
89                 r'<p[^>]+\bclass=(["\'])(?:(?!\1).)*\balert\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</p>',
90                 response, 'error', default=None, group='error')
91             if error:
92                 raise ExtractorError('Unable to login: %s' % error, expected=True)
93             raise ExtractorError('Unable to log in')
94
95     def _real_initialize(self):
96         self._login()
97
98     def _real_extract(self, url):
99         anime_id = self._match_id(url)
100
101         webpage = self._download_webpage(url, anime_id)
102
103         if 'data-playlist=' not in webpage:
104             self._download_webpage(
105                 self._APPLY_HTML5_URL, anime_id,
106                 'Activating HTML5 beta', 'Unable to apply HTML5 beta')
107             webpage = self._download_webpage(url, anime_id)
108
109         csrf_token = self._html_search_meta(
110             'csrf-token', webpage, 'csrf token', fatal=True)
111
112         anime_title = self._html_search_regex(
113             r'(?s)<h1[^>]+itemprop="name"[^>]*>(.+?)</h1>',
114             webpage, 'anime name')
115         anime_description = self._html_search_regex(
116             r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
117             webpage, 'anime description', default=None)
118
119         entries = []
120
121         def extract_info(html, video_id, num=None):
122             title, description = [None] * 2
123             formats = []
124
125             for input_ in re.findall(
126                     r'<input[^>]+class=["\'].*?streamstarter[^>]+>', html):
127                 attributes = extract_attributes(input_)
128                 title = attributes.get('data-dialog-header')
129                 playlist_urls = []
130                 for playlist_key in ('data-playlist', 'data-otherplaylist', 'data-stream'):
131                     playlist_url = attributes.get(playlist_key)
132                     if isinstance(playlist_url, compat_str) and re.match(
133                             r'/?[\da-zA-Z]+', playlist_url):
134                         playlist_urls.append(attributes[playlist_key])
135                 if not playlist_urls:
136                     continue
137
138                 lang = attributes.get('data-lang')
139                 lang_note = attributes.get('value')
140
141                 for playlist_url in playlist_urls:
142                     kind = self._search_regex(
143                         r'videomaterialurl/\d+/([^/]+)/',
144                         playlist_url, 'media kind', default=None)
145                     format_id_list = []
146                     if lang:
147                         format_id_list.append(lang)
148                     if kind:
149                         format_id_list.append(kind)
150                     if not format_id_list and num is not None:
151                         format_id_list.append(compat_str(num))
152                     format_id = '-'.join(format_id_list)
153                     format_note = ', '.join(filter(None, (kind, lang_note)))
154                     item_id_list = []
155                     if format_id:
156                         item_id_list.append(format_id)
157                     item_id_list.append('videomaterial')
158                     playlist = self._download_json(
159                         urljoin(url, playlist_url), video_id,
160                         'Downloading %s JSON' % ' '.join(item_id_list),
161                         headers={
162                             'X-Requested-With': 'XMLHttpRequest',
163                             'X-CSRF-Token': csrf_token,
164                             'Referer': url,
165                             'Accept': 'application/json, text/javascript, */*; q=0.01',
166                         }, fatal=False)
167                     if not playlist:
168                         continue
169                     stream_url = url_or_none(playlist.get('streamurl'))
170                     if stream_url:
171                         rtmp = re.search(
172                             r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+/))(?P<playpath>mp[34]:.+)',
173                             stream_url)
174                         if rtmp:
175                             formats.append({
176                                 'url': rtmp.group('url'),
177                                 'app': rtmp.group('app'),
178                                 'play_path': rtmp.group('playpath'),
179                                 'page_url': url,
180                                 'player_url': 'https://www.anime-on-demand.de/assets/jwplayer.flash-55abfb34080700304d49125ce9ffb4a6.swf',
181                                 'rtmp_real_time': True,
182                                 'format_id': 'rtmp',
183                                 'ext': 'flv',
184                             })
185                             continue
186                     start_video = playlist.get('startvideo', 0)
187                     playlist = playlist.get('playlist')
188                     if not playlist or not isinstance(playlist, list):
189                         continue
190                     playlist = playlist[start_video]
191                     title = playlist.get('title')
192                     if not title:
193                         continue
194                     description = playlist.get('description')
195                     for source in playlist.get('sources', []):
196                         file_ = source.get('file')
197                         if not file_:
198                             continue
199                         ext = determine_ext(file_)
200                         format_id_list = [lang, kind]
201                         if ext == 'm3u8':
202                             format_id_list.append('hls')
203                         elif source.get('type') == 'video/dash' or ext == 'mpd':
204                             format_id_list.append('dash')
205                         format_id = '-'.join(filter(None, format_id_list))
206                         if ext == 'm3u8':
207                             file_formats = self._extract_m3u8_formats(
208                                 file_, video_id, 'mp4',
209                                 entry_protocol='m3u8_native', m3u8_id=format_id, fatal=False)
210                         elif source.get('type') == 'video/dash' or ext == 'mpd':
211                             continue
212                             file_formats = self._extract_mpd_formats(
213                                 file_, video_id, mpd_id=format_id, fatal=False)
214                         else:
215                             continue
216                         for f in file_formats:
217                             f.update({
218                                 'language': lang,
219                                 'format_note': format_note,
220                             })
221                         formats.extend(file_formats)
222
223             return {
224                 'title': title,
225                 'description': description,
226                 'formats': formats,
227             }
228
229         def extract_entries(html, video_id, common_info, num=None):
230             info = extract_info(html, video_id, num)
231
232             if info['formats']:
233                 self._sort_formats(info['formats'])
234                 f = common_info.copy()
235                 f.update(info)
236                 entries.append(f)
237
238             # Extract teaser/trailer only when full episode is not available
239             if not info['formats']:
240                 m = re.search(
241                     r'data-dialog-header=(["\'])(?P<title>.+?)\1[^>]+href=(["\'])(?P<href>.+?)\3[^>]*>(?P<kind>Teaser|Trailer)<',
242                     html)
243                 if m:
244                     f = common_info.copy()
245                     f.update({
246                         'id': '%s-%s' % (f['id'], m.group('kind').lower()),
247                         'title': m.group('title'),
248                         'url': urljoin(url, m.group('href')),
249                     })
250                     entries.append(f)
251
252         def extract_episodes(html):
253             for num, episode_html in enumerate(re.findall(
254                     r'(?s)<h3[^>]+class="episodebox-title".+?>Episodeninhalt<', html), 1):
255                 episodebox_title = self._search_regex(
256                     (r'class="episodebox-title"[^>]+title=(["\'])(?P<title>.+?)\1',
257                      r'class="episodebox-title"[^>]+>(?P<title>.+?)<'),
258                     episode_html, 'episodebox title', default=None, group='title')
259                 if not episodebox_title:
260                     continue
261
262                 episode_number = int(self._search_regex(
263                     r'(?:Episode|Film)\s*(\d+)',
264                     episodebox_title, 'episode number', default=num))
265                 episode_title = self._search_regex(
266                     r'(?:Episode|Film)\s*\d+\s*-\s*(.+)',
267                     episodebox_title, 'episode title', default=None)
268
269                 video_id = 'episode-%d' % episode_number
270
271                 common_info = {
272                     'id': video_id,
273                     'series': anime_title,
274                     'episode': episode_title,
275                     'episode_number': episode_number,
276                 }
277
278                 extract_entries(episode_html, video_id, common_info)
279
280         def extract_film(html, video_id):
281             common_info = {
282                 'id': anime_id,
283                 'title': anime_title,
284                 'description': anime_description,
285             }
286             extract_entries(html, video_id, common_info)
287
288         extract_episodes(webpage)
289
290         if not entries:
291             extract_film(webpage, anime_id)
292
293         return self.playlist_result(entries, anime_id, anime_title, anime_description)