_ Git - youtube-dl/blob - youtube_dl/extractor/generic.py

   1 # encoding: utf-8
   2
   3 import os
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..utils import (
   8     compat_urllib_error,
   9     compat_urllib_parse,
  10     compat_urllib_request,
  11
  12     ExtractorError,
  13 )
  14 from .brightcove import BrightcoveIE
  15
  16 class GenericIE(InfoExtractor):
  17     IE_DESC = u'Generic downloader that works on some sites'
  18     _VALID_URL = r'.*'
  19     IE_NAME = u'generic'
  20     _TESTS = [
  21         {
  22             u'url': u'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
  23             u'file': u'13601338388002.mp4',
  24             u'md5': u'85b90ccc9d73b4acd9138d3af4c27f89',
  25             u'info_dict': {
  26                 u"uploader": u"www.hodiho.fr",
  27                 u"title": u"R\u00e9gis plante sa Jeep"
  28             }
  29         },
  30         {
  31             u'url': u'http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/',
  32             u'file': u'2371591881001.mp4',
  33             u'md5': u'9e80619e0a94663f0bdc849b4566af19',
  34             u'note': u'Test Brightcove downloads and detection in GenericIE',
  35             u'info_dict': {
  36                 u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”',
  37                 u'uploader': u'8TV',
  38                 u'description': u'md5:a950cc4285c43e44d763d036710cd9cd',
  39             }
  40         },
  41     ]
  42
  43     def report_download_webpage(self, video_id):
  44         """Report webpage download."""
  45         if not self._downloader.params.get('test', False):
  46             self._downloader.report_warning(u'Falling back on generic information extractor.')
  47         super(GenericIE, self).report_download_webpage(video_id)
  48
  49     def report_following_redirect(self, new_url):
  50         """Report information extraction."""
  51         self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
  52
  53     def _test_redirect(self, url):
  54         """Check if it is a redirect, like url shorteners, in case return the new url."""
  55         class HeadRequest(compat_urllib_request.Request):
  56             def get_method(self):
  57                 return "HEAD"
  58
  59         class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
  60             """
  61             Subclass the HTTPRedirectHandler to make it use our
  62             HeadRequest also on the redirected URL
  63             """
  64             def redirect_request(self, req, fp, code, msg, headers, newurl):
  65                 if code in (301, 302, 303, 307):
  66                     newurl = newurl.replace(' ', '%20')
  67                     newheaders = dict((k,v) for k,v in req.headers.items()
  68                                       if k.lower() not in ("content-length", "content-type"))
  69                     return HeadRequest(newurl,
  70                                        headers=newheaders,
  71                                        origin_req_host=req.get_origin_req_host(),
  72                                        unverifiable=True)
  73                 else:
  74                     raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp)
  75
  76         class HTTPMethodFallback(compat_urllib_request.BaseHandler):
  77             """
  78             Fallback to GET if HEAD is not allowed (405 HTTP error)
  79             """
  80             def http_error_405(self, req, fp, code, msg, headers):
  81                 fp.read()
  82                 fp.close()
  83
  84                 newheaders = dict((k,v) for k,v in req.headers.items()
  85                                   if k.lower() not in ("content-length", "content-type"))
  86                 return self.parent.open(compat_urllib_request.Request(req.get_full_url(),
  87                                                  headers=newheaders,
  88                                                  origin_req_host=req.get_origin_req_host(),
  89                                                  unverifiable=True))
  90
  91         # Build our opener
  92         opener = compat_urllib_request.OpenerDirector()
  93         for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler,
  94                         HTTPMethodFallback, HEADRedirectHandler,
  95                         compat_urllib_request.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]:
  96             opener.add_handler(handler())
  97
  98         response = opener.open(HeadRequest(url))
  99         if response is None:
 100             raise ExtractorError(u'Invalid URL protocol')
 101         new_url = response.geturl()
 102
 103         if url == new_url:
 104             return False
 105
 106         self.report_following_redirect(new_url)
 107         return new_url
 108
 109     def _real_extract(self, url):
 110         try:
 111             new_url = self._test_redirect(url)
 112             if new_url:
 113                 return [self.url_result(new_url)]
 114         except compat_urllib_error.HTTPError:
 115             # This may be a stupid server that doesn't like HEAD, our UA, or so
 116             pass
 117
 118         video_id = url.split('/')[-1]
 119         try:
 120             webpage = self._download_webpage(url, video_id)
 121         except ValueError:
 122             # since this is the last-resort InfoExtractor, if
 123             # this error is thrown, it'll be thrown here
 124             raise ExtractorError(u'Invalid URL: %s' % url)
 125
 126         self.report_extraction(video_id)
 127         # Look for BrigthCove:
 128         m_brightcove = re.search(r'<object.+?class=([\'"]).*?BrightcoveExperience.*?\1.+?</object>', webpage, re.DOTALL)
 129         if m_brightcove is not None:
 130             self.to_screen(u'Brightcove video detected.')
 131             bc_url = BrightcoveIE._build_brighcove_url(m_brightcove.group())
 132             return self.url_result(bc_url, 'Brightcove')
 133
 134         # Start with something easy: JW Player in SWFObject
 135         mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
 136         if mobj is None:
 137             # Broaden the search a little bit
 138             mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
 139         if mobj is None:
 140             # Broaden the search a little bit: JWPlayer JS loader
 141             mobj = re.search(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http[^\'"&]*)', webpage)
 142         if mobj is None:
 143             # Try to find twitter cards info
 144             mobj = re.search(r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage)
 145         if mobj is None:
 146             # We look for Open Graph info:
 147             # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
 148             m_video_type = re.search(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
 149             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
 150             if m_video_type is not None:
 151                 mobj = re.search(r'<meta.*?property="og:video".*?content="(.*?)"', webpage)
 152         if mobj is None:
 153             # HTML5 video
 154             mobj = re.search(r'<video[^<]*>.*?<source .*?src="([^"]+)"', webpage, flags=re.DOTALL)
 155         if mobj is None:
 156             raise ExtractorError(u'Invalid URL: %s' % url)
 157
 158         # It's possible that one of the regexes
 159         # matched, but returned an empty group:
 160         if mobj.group(1) is None:
 161             raise ExtractorError(u'Invalid URL: %s' % url)
 162
 163         video_url = compat_urllib_parse.unquote(mobj.group(1))
 164         video_id = os.path.basename(video_url)
 165
 166         # here's a fun little line of code for you:
 167         video_extension = os.path.splitext(video_id)[1][1:]
 168         video_id = os.path.splitext(video_id)[0]
 169
 170         # it's tempting to parse this further, but you would
 171         # have to take into account all the variations like
 172         #   Video Title - Site Name
 173         #   Site Name | Video Title
 174         #   Video Title - Tagline | Site Name
 175         # and so on and so forth; it's just not practical
 176         video_title = self._html_search_regex(r'<title>(.*)</title>',
 177             webpage, u'video title', default=u'video', flags=re.DOTALL)
 178
 179         # video uploader is domain name
 180         video_uploader = self._search_regex(r'(?:https?://)?([^/]*)/.*',
 181             url, u'video uploader')
 182
 183         return [{
 184             'id':       video_id,
 185             'url':      video_url,
 186             'uploader': video_uploader,
 187             'upload_date':  None,
 188             'title':    video_title,
 189             'ext':      video_extension,
 190         }]