_ Git - youtube-dl/blob - youtube_dl/extractor/tagesschau.py

   1 # -*- coding: utf-8 -*-
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..utils import (
   8     determine_ext,
   9     parse_filesize,
  10 )
  11
  12
  13 class TagesschauIE(InfoExtractor):
  14     _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/[^/]+/(?:[^/]+/)*?[^/#?]+?(?P<id>-?[0-9]+)(?:~_?[^/#?]+?)?\.html'
  15
  16     _TESTS = [{
  17         'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html',
  18         'md5': 'f7c27a0eff3bfe8c7727e65f8fe1b1e6',
  19         'info_dict': {
  20             'id': '102143',
  21             'ext': 'mp4',
  22             'title': 'Regierungsumbildung in Athen: Neue Minister in Griechenland vereidigt',
  23             'description': 'md5:171feccd9d9b3dd54d05d501568f6359',
  24             'thumbnail': 're:^https?:.*\.jpg$',
  25         },
  26     }, {
  27         'url': 'http://www.tagesschau.de/multimedia/sendung/ts-5727.html',
  28         'md5': '3c54c1f6243d279b706bde660ceec633',
  29         'info_dict': {
  30             'id': '5727',
  31             'ext': 'mp4',
  32             'description': 'md5:695c01bfd98b7e313c501386327aea59',
  33             'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
  34             'thumbnail': 're:^https?:.*\.jpg$',
  35         },
  36     }, {
  37         'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html',
  38         'md5': 'aef45de271c4bf0a5db834aa40bf774c',
  39         'info_dict': {
  40             'id': '18407',
  41             'ext': 'mp3',
  42             'title': 'Flüchtlingsdebatte: Hitzig, aber wenig hilfreich',
  43             'description': 'Flüchtlingsdebatte: Hitzig, aber wenig hilfreich',
  44             'thumbnail': 're:^https?:.*\.jpg$',
  45         },
  46     }, {
  47         'url': 'http://www.tagesschau.de/inland/afd-parteitag-135.html',
  48         'info_dict': {
  49             'id': '135',
  50             'title': 'Möchtegern-Underdog mit Machtanspruch',
  51         },
  52         'playlist_count': 2,
  53     }, {
  54         'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html',
  55         'only_matching': True,
  56     }, {
  57         'url': 'http://www.tagesschau.de/multimedia/sendung/tt-3827.html',
  58         'only_matching': True,
  59     }, {
  60         'url': 'http://www.tagesschau.de/multimedia/sendung/nm-3475.html',
  61         'only_matching': True,
  62     }, {
  63         'url': 'http://www.tagesschau.de/multimedia/sendung/weltspiegel-3167.html',
  64         'only_matching': True,
  65     }, {
  66         'url': 'http://www.tagesschau.de/multimedia/tsvorzwanzig-959.html',
  67         'only_matching': True,
  68     }, {
  69         'url': 'http://www.tagesschau.de/multimedia/sendung/bab/bab-3299~_bab-sendung-209.html',
  70         'only_matching': True,
  71     }, {
  72         'url': 'http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html',
  73         'only_matching': True,
  74     }, {
  75         'url': 'http://www.tagesschau.de/multimedia/video/video-179517~player.html',
  76         'only_matching': True,
  77     }]
  78
  79     _FORMATS = {
  80         'xs': {'quality': 0},
  81         's': {'width': 320, 'height': 180, 'quality': 1},
  82         'm': {'width': 512, 'height': 288, 'quality': 2},
  83         'l': {'width': 960, 'height': 540, 'quality': 3},
  84         'xl': {'width': 1280, 'height': 720, 'quality': 4},
  85         'xxl': {'quality': 5},
  86     }
  87
  88     def _extract_formats(self, download_text, media_kind):
  89         links = re.finditer(
  90             r'<div class="button" title="(?P<title>[^"]*)"><a href="(?P<url>[^"]+)">(?P<name>.+?)</a></div>',
  91             download_text)
  92         formats = []
  93         for l in links:
  94             link_url = l.group('url')
  95             if not link_url:
  96                 continue
  97             format_id = self._search_regex(
  98                 r'.*/[^/.]+\.([^/]+)\.[^/.]+$', link_url, 'format ID',
  99                 default=determine_ext(link_url))
 100             format = {
 101                 'format_id': format_id,
 102                 'url': l.group('url'),
 103                 'format_name': l.group('name'),
 104             }
 105             title = l.group('title')
 106             if title:
 107                 if media_kind.lower() == 'video':
 108                     m = re.match(
 109                         r'''(?x)
 110                             Video:\s*(?P<vcodec>[a-zA-Z0-9/._-]+)\s*&\#10;
 111                             (?P<width>[0-9]+)x(?P<height>[0-9]+)px&\#10;
 112                             (?P<vbr>[0-9]+)kbps&\#10;
 113                             Audio:\s*(?P<abr>[0-9]+)kbps,\s*(?P<audio_desc>[A-Za-z\.0-9]+)&\#10;
 114                             Gr&ouml;&szlig;e:\s*(?P<filesize_approx>[0-9.,]+\s+[a-zA-Z]*B)''',
 115                         title)
 116                     if m:
 117                         format.update({
 118                             'format_note': m.group('audio_desc'),
 119                             'vcodec': m.group('vcodec'),
 120                             'width': int(m.group('width')),
 121                             'height': int(m.group('height')),
 122                             'abr': int(m.group('abr')),
 123                             'vbr': int(m.group('vbr')),
 124                             'filesize_approx': parse_filesize(m.group('filesize_approx')),
 125                         })
 126                 else:
 127                     m = re.match(
 128                         r'(?P<format>.+?)-Format\s*:\s*(?P<abr>\d+)kbps\s*,\s*(?P<note>.+)',
 129                         title)
 130                     if m:
 131                         format.update({
 132                             'format_note': '%s, %s' % (m.group('format'), m.group('note')),
 133                             'vcodec': 'none',
 134                             'abr': int(m.group('abr')),
 135                         })
 136             formats.append(format)
 137         self._sort_formats(formats)
 138         return formats
 139
 140     def _real_extract(self, url):
 141         video_id = self._match_id(url)
 142         display_id = video_id.lstrip('-')
 143         webpage = self._download_webpage(url, display_id)
 144
 145         player_url = self._html_search_meta(
 146             'twitter:player', webpage, 'player URL', default=None)
 147         if player_url:
 148             playerpage = self._download_webpage(
 149                 player_url, display_id, 'Downloading player page')
 150
 151             formats = []
 152             for media in re.finditer(
 153                     r'''(?x)
 154                         (?P<q_url>["\'])(?P<url>http://media.+?)(?P=q_url)
 155                         ,\s*type:(?P<q_type>["\'])(?P<type>video|audio)/(?P<ext>.+?)(?P=q_type)
 156                         (?:,\s*quality:(?P<q_quality>["\'])(?P<quality>.+?)(?P=q_quality))?
 157                     ''', playerpage):
 158                 url = media.group('url')
 159                 webpage_type = media.group('type')
 160                 ext = media.group('ext')
 161                 res = media.group('quality')
 162                 f = {
 163                     'format_id': '%s_%s' % (res, ext) if res else ext,
 164                     'url': url,
 165                     'ext': ext,
 166                     'vcodec': 'none' if webpage_type == 'audio' else None,
 167                 }
 168                 f.update(self._FORMATS.get(res, {}))
 169                 formats.append(f)
 170             thumbnail = self._og_search_thumbnail(playerpage)
 171             title = self._og_search_title(webpage).strip()
 172             description = self._og_search_description(webpage).strip()
 173         else:
 174             title = self._html_search_regex(
 175                 r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 176
 177             DOWNLOAD_REGEX = r'(?s)<p>Wir bieten dieses (?P<kind>Video|Audio) in folgenden Formaten zum Download an:</p>\s*<div class="controls">(?P<links>.*?)</div>\s*<p>'
 178
 179             webpage_type = self._og_search_property('type', webpage, default=None)
 180             if webpage_type == 'website':  # Article
 181                 entries = []
 182                 for num, (entry_title, media_kind, download_text) in enumerate(re.findall(
 183                         r'(?s)<p[^>]+class="infotext"[^>]*>.*?<strong>(.+?)</strong>.*?</p>.*?%s' % DOWNLOAD_REGEX,
 184                         webpage)):
 185                     entries.append({
 186                         'id': display_id,
 187                         'title': '%s-%d' % (entry_title, num),
 188                         'formats': self._extract_formats(download_text, media_kind),
 189                     })
 190                 return self.playlist_result(entries, display_id, title)
 191             else:  # Assume single video
 192                 download_text = self._search_regex(
 193                     DOWNLOAD_REGEX, webpage, 'download links', group='links')
 194                 media_kind = self._search_regex(
 195                     DOWNLOAD_REGEX, webpage, 'media kind', default='Video', group='links')
 196                 formats = self._extract_formats(download_text, media_kind)
 197                 thumbnail = self._og_search_thumbnail(webpage)
 198                 description = self._html_search_regex(
 199                     r'(?s)<p class="teasertext">(.*?)</p>',
 200                     webpage, 'description', default=None)
 201
 202         self._sort_formats(formats)
 203
 204         return {
 205             'id': display_id,
 206             'title': title,
 207             'thumbnail': thumbnail,
 208             'formats': formats,
 209             'description': description,
 210         }