_ Git - youtube-dl/blob - youtube_dl/extractor/polskieradio.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..compat import (
   8     compat_str,
   9     compat_urllib_parse_unquote,
  10     compat_urlparse
  11 )
  12 from ..utils import (
  13     int_or_none,
  14     strip_or_none,
  15     unified_timestamp,
  16 )
  17
  18
  19 class PolskieRadioProgrammeIE(InfoExtractor):
  20     _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+(,[^/]+)?/(?P<id>\d+)'
  21     _TESTS = [{
  22         'url': 'http://www.polskieradio.pl/7/5102,HISTORIA-ZYWA',
  23         'info_dict': {
  24             'id': '5102',
  25             'title': 'HISTORIA ŻYWA',
  26         },
  27         'playlist_mincount': 34,
  28     }, {
  29         'url': 'http://www.polskieradio.pl/7/4807',
  30         'info_dict': {
  31             'id': '4807',
  32             'title': 'Vademecum 1050. rocznicy Chrztu Polski'
  33         },
  34         'playlist_mincount': 5
  35     }, {
  36         'url': 'http://www.polskieradio.pl/7/129,Sygnaly-dnia?ref=source',
  37         'only_matching': True
  38     }, {
  39         'url': 'http://www.polskieradio.pl/37,RedakcjaKatolicka/4143,Kierunek-Krakow',
  40         'info_dict': {
  41             'id': '4143',
  42             'title': 'Kierunek Kraków',
  43         },
  44         'playlist_mincount': 61
  45     }, {
  46         'url': 'http://www.polskieradio.pl/7,Jedynka/5102,HISTORIA-ZYWA',
  47         'only_matching': True
  48     }]
  49
  50     def _get_entries_from_page_content(self, base_url, content):
  51         entries = []
  52
  53         articles = re.findall(
  54             r'<article class="ID-(\d+) article">\s+<a href="([^"]+)"( data-layer="[^"]*")? class="[^"]*" title="([^"]+)">',
  55             content)
  56         for article_id, article_url, _, article_title in articles:
  57             resolved_article_url = compat_urlparse.urljoin(base_url, article_url)
  58             entries.append(self.url_result(
  59                 resolved_article_url,
  60                 ie='PolskieRadio',
  61                 video_id=article_id,
  62                 video_title=article_title))
  63
  64         return entries
  65
  66     @classmethod
  67     def suitable(cls, url):
  68         return False if PolskieRadioIE.suitable(url) else super(PolskieRadioProgrammeIE, cls).suitable(url)
  69
  70     def _real_extract(self, url):
  71         programme_id = self._match_id(url)
  72         webpage = self._download_webpage(url, programme_id)
  73
  74         title = self._html_search_regex(
  75             r'<a href="[^"]+" id=".*_linkCategory" title="[^"]+">(.+?)</a>',
  76             webpage, 'title', fatal=False)
  77         description = None
  78
  79         entries = self._get_entries_from_page_content(url, webpage)
  80
  81         pages = re.findall(r'<a( href="([^"]+/Strona/)\d+")? id="[^"]+" title="strona&#32;(\d+)"', webpage)
  82         page_count = max(int(page_number) for _, _, page_number in pages) if pages else 1
  83
  84         if page_count > 1:
  85             page_url_root = next(url for _, url, _ in pages if len(url) > 0)
  86             for page_number in range(2, page_count + 1):
  87                 page_url = page_url_root + str(page_number)
  88                 resolved_page_url = compat_urlparse.urljoin(url, page_url)
  89                 page_content = self._download_webpage(
  90                     resolved_page_url, programme_id,
  91                     note="Downloading page number %d" % page_number)
  92                 entries.extend(self._get_entries_from_page_content(url, page_content))
  93
  94         return self.playlist_result(entries, programme_id, title, description)
  95
  96
  97 class PolskieRadioIE(InfoExtractor):
  98     _VALID_URL = r'https?://(?:www\.)?polskieradio\.pl/\d+/\d+/Artykul/(?P<id>[0-9]+)'
  99     _TESTS = [{
 100         'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943,Prof-Andrzej-Nowak-o-historii-nie-da-sie-myslec-beznamietnie',
 101         'info_dict': {
 102             'id': '1587943',
 103             'title': 'Prof. Andrzej Nowak: o historii nie da się myśleć beznamiętnie',
 104             'description': 'md5:12f954edbf3120c5e7075e17bf9fc5c5',
 105         },
 106         'playlist': [{
 107             'md5': '2984ee6ce9046d91fc233bc1a864a09a',
 108             'info_dict': {
 109                 'id': '1540576',
 110                 'ext': 'mp3',
 111                 'title': 'md5:d4623290d4ac983bf924061c75c23a0d',
 112                 'timestamp': 1456594200,
 113                 'upload_date': '20160227',
 114                 'duration': 2364,
 115                 'thumbnail': 're:^https?://static\.prsa\.pl/images/.*\.jpg$'
 116             },
 117         }],
 118     }, {
 119         'url': 'http://www.polskieradio.pl/265/5217/Artykul/1635803,Euro-2016-nie-ma-miejsca-na-blad-Polacy-graja-ze-Szwajcaria-o-cwiercfinal',
 120         'info_dict': {
 121             'id': '1635803',
 122             'title': 'Euro 2016: nie ma miejsca na błąd. Polacy grają ze Szwajcarią o ćwierćfinał',
 123             'description': 'md5:01cb7d0cad58664095d72b51a1ebada2',
 124         },
 125         'playlist_mincount': 12,
 126     }, {
 127         'url': 'http://polskieradio.pl/9/305/Artykul/1632955,Bardzo-popularne-slowo-remis',
 128         'only_matching': True,
 129     }, {
 130         'url': 'http://www.polskieradio.pl/7/5102/Artykul/1587943',
 131         'only_matching': True,
 132     }, {
 133         # with mp4 video
 134         'url': 'http://www.polskieradio.pl/9/299/Artykul/1634903,Brexit-Leszek-Miller-swiat-sie-nie-zawali-Europa-bedzie-trwac-dalej',
 135         'only_matching': True,
 136     }]
 137
 138     def _real_extract(self, url):
 139         playlist_id = self._match_id(url)
 140
 141         webpage = self._download_webpage(url, playlist_id)
 142
 143         content = self._search_regex(
 144             r'(?s)<div[^>]+class="audio atarticle"[^>]*>(.+?)<script>',
 145             webpage, 'content')
 146
 147         timestamp = unified_timestamp(self._html_search_regex(
 148             r'(?s)<span[^>]+id="datetime2"[^>]*>(.+?)</span>',
 149             webpage, 'timestamp', fatal=False))
 150
 151         thumbnail_url = self._og_search_thumbnail(webpage)
 152
 153         entries = []
 154
 155         media_urls = set()
 156
 157         for data_media in re.findall(r'<[^>]+data-media=({[^>]+})', content):
 158             media = self._parse_json(data_media, playlist_id, fatal=False)
 159             if not media.get('file') or not media.get('desc'):
 160                 continue
 161             media_url = self._proto_relative_url(media['file'], 'http:')
 162             if media_url in media_urls:
 163                 continue
 164             media_urls.add(media_url)
 165             entries.append({
 166                 'id': compat_str(media['id']),
 167                 'url': media_url,
 168                 'title': compat_urllib_parse_unquote(media['desc']),
 169                 'duration': int_or_none(media.get('length')),
 170                 'vcodec': 'none' if media.get('provider') == 'audio' else None,
 171                 'timestamp': timestamp,
 172                 'thumbnail': thumbnail_url
 173             })
 174
 175         title = self._og_search_title(webpage).strip()
 176         description = strip_or_none(self._og_search_description(webpage))
 177
 178         return self.playlist_result(entries, playlist_id, title, description)