11 compat_urllib_request,
20 class InfoExtractor(object):
21 """Information Extractor class.
23 Information extractors are the classes that, given a URL, extract
24 information about the video (or videos) the URL refers to. This
25 information includes the real video URL, the video title, author and
26 others. The information is stored in a dictionary which is then
27 passed to the FileDownloader. The FileDownloader processes this
28 information possibly downloading the video to the file system, among
29 other possible outcomes.
31 The dictionaries must include the following fields:
35 title: Video title, unescaped.
36 ext: Video filename extension.
38 The following fields are optional:
40 format: The video format, defaults to ext (used for --get-format)
41 thumbnails: A list of dictionaries (with the entries "resolution" and
42 "url") for the varying thumbnails
43 thumbnail: Full URL to a video thumbnail image.
44 description: One-line video description.
45 uploader: Full name of the video uploader.
46 upload_date: Video upload date (YYYYMMDD).
47 uploader_id: Nickname or id of the video uploader.
48 location: Physical location of the video.
49 player_url: SWF Player URL (used for rtmpdump).
50 subtitles: The subtitle file contents as a dictionary in the format
51 {language: subtitles}.
52 view_count: How many users have watched the video on the platform.
53 urlhandle: [internal] The urlHandle to be used to download the file,
54 like returned by urllib.request.urlopen
56 The fields should all be Unicode strings.
58 Subclasses of this one should re-define the _real_initialize() and
59 _real_extract() methods and define a _VALID_URL regexp.
60 Probably, they should also be added to the list of extractors.
62 _real_extract() must return a *list* of information dictionaries as
65 Finally, the _WORKING attribute should be set to False for broken IEs
66 in order to warn the users and skip the tests.
73 def __init__(self, downloader=None):
74 """Constructor. Receives an optional downloader."""
76 self.set_downloader(downloader)
79 def suitable(cls, url):
80 """Receives a URL and returns True if suitable for this IE."""
82 # This does not use has/getattr intentionally - we want to know whether
83 # we have cached the regexp for *this* class, whereas getattr would also
84 # match the superclass
85 if '_VALID_URL_RE' not in cls.__dict__:
86 cls._VALID_URL_RE = re.compile(cls._VALID_URL)
87 return cls._VALID_URL_RE.match(url) is not None
91 """Getter method for _WORKING."""
95 """Initializes an instance (authentication, etc)."""
97 self._real_initialize()
100 def extract(self, url):
101 """Extracts URL information and returns it in list of dicts."""
103 return self._real_extract(url)
105 def set_downloader(self, downloader):
106 """Sets the downloader for this IE."""
107 self._downloader = downloader
109 def _real_initialize(self):
110 """Real initialization process. Redefine in subclasses."""
113 def _real_extract(self, url):
114 """Real extraction process. Redefine in subclasses."""
119 """A string for getting the InfoExtractor with get_info_extractor"""
120 return cls.__name__[:-2]
124 return type(self).__name__[:-2]
126 def _request_webpage(self, url_or_request, video_id, note=None, errnote=None):
127 """ Returns the response handle """
129 self.report_download_webpage(video_id)
130 elif note is not False:
131 self.to_screen(u'%s: %s' % (video_id, note))
133 return compat_urllib_request.urlopen(url_or_request)
134 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
136 errnote = u'Unable to download webpage'
137 raise ExtractorError(u'%s: %s' % (errnote, compat_str(err)), sys.exc_info()[2], cause=err)
139 def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None):
140 """ Returns a tuple (page content as string, URL handle) """
142 # Strip hashes from the URL (#1038)
143 if isinstance(url_or_request, (compat_str, str)):
144 url_or_request = url_or_request.partition('#')[0]
146 urlh = self._request_webpage(url_or_request, video_id, note, errnote)
147 content_type = urlh.headers.get('Content-Type', '')
148 m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
150 encoding = m.group(1)
153 webpage_bytes = urlh.read()
154 if self._downloader.params.get('dump_intermediate_pages', False):
156 url = url_or_request.get_full_url()
157 except AttributeError:
159 self.to_screen(u'Dumping request to ' + url)
160 dump = base64.b64encode(webpage_bytes).decode('ascii')
161 self._downloader.to_screen(dump)
162 content = webpage_bytes.decode(encoding, 'replace')
163 return (content, urlh)
165 def _download_webpage(self, url_or_request, video_id, note=None, errnote=None):
166 """ Returns the data of the page as a string """
167 return self._download_webpage_handle(url_or_request, video_id, note, errnote)[0]
169 def to_screen(self, msg):
170 """Print msg to screen, prefixing it with '[ie_name]'"""
171 self._downloader.to_screen(u'[%s] %s' % (self.IE_NAME, msg))
173 def report_extraction(self, id_or_name):
174 """Report information extraction."""
175 self.to_screen(u'%s: Extracting information' % id_or_name)
177 def report_download_webpage(self, video_id):
178 """Report webpage download."""
179 self.to_screen(u'%s: Downloading webpage' % video_id)
181 def report_age_confirmation(self):
182 """Report attempt to confirm age."""
183 self.to_screen(u'Confirming age')
185 def report_login(self):
186 """Report attempt to log in."""
187 self.to_screen(u'Logging in')
189 #Methods for following #608
190 def url_result(self, url, ie=None):
191 """Returns a url that points to a page that should be processed"""
192 #TODO: ie should be the class used for getting the info
193 video_info = {'_type': 'url',
197 def playlist_result(self, entries, playlist_id=None, playlist_title=None):
198 """Returns a playlist"""
199 video_info = {'_type': 'playlist',
202 video_info['id'] = playlist_id
204 video_info['title'] = playlist_title
207 def _search_regex(self, pattern, string, name, default=None, fatal=True, flags=0):
209 Perform a regex search on the given string, using a single or a list of
210 patterns returning the first matching group.
211 In case of failure return a default value or raise a WARNING or a
212 ExtractorError, depending on fatal, specifying the field name.
214 if isinstance(pattern, (str, compat_str, compiled_regex_type)):
215 mobj = re.search(pattern, string, flags)
218 mobj = re.search(p, string, flags)
221 if sys.stderr.isatty() and os.name != 'nt':
222 _name = u'\033[0;34m%s\033[0m' % name
227 # return the first matching group
228 return next(g for g in mobj.groups() if g is not None)
229 elif default is not None:
232 raise ExtractorError(u'Unable to extract %s' % _name)
234 self._downloader.report_warning(u'unable to extract %s; '
235 u'please report this issue on http://yt-dl.org/bug' % _name)
238 def _html_search_regex(self, pattern, string, name, default=None, fatal=True, flags=0):
240 Like _search_regex, but strips HTML tags and unescapes entities.
242 res = self._search_regex(pattern, string, name, default, fatal, flags)
244 return clean_html(res).strip()
248 def _get_login_info(self):
250 Get the the login info as (username, password)
251 It will look in the netrc file using the _NETRC_MACHINE value
252 If there's no info available, return (None, None)
254 if self._downloader is None:
259 downloader_params = self._downloader.params
261 # Attempt to use provided username and password or .netrc data
262 if downloader_params.get('username', None) is not None:
263 username = downloader_params['username']
264 password = downloader_params['password']
265 elif downloader_params.get('usenetrc', False):
267 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
272 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
273 except (IOError, netrc.NetrcParseError) as err:
274 self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err))
276 return (username, password)
278 # Helper functions for extracting OpenGraph info
281 return r'<meta.+?property=[\'"]og:%s[\'"].+?content=(?:"(.+?)"|\'(.+?)\')' % re.escape(prop)
283 def _og_search_property(self, prop, html, name=None, **kargs):
285 name = 'OpenGraph %s' % prop
286 escaped = self._search_regex(self._og_regex(prop), html, name, flags=re.DOTALL, **kargs)
287 return unescapeHTML(escaped)
289 def _og_search_thumbnail(self, html, **kargs):
290 return self._og_search_property('image', html, u'thumbnail url', fatal=False, **kargs)
292 def _og_search_description(self, html, **kargs):
293 return self._og_search_property('description', html, fatal=False, **kargs)
295 def _og_search_title(self, html, **kargs):
296 return self._og_search_property('title', html, **kargs)
298 def _og_search_video_url(self, html, name='video url', **kargs):
299 return self._html_search_regex([self._og_regex('video:secure_url'),
300 self._og_regex('video')],
303 class SearchInfoExtractor(InfoExtractor):
305 Base class for paged search queries extractors.
306 They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
307 Instances should define _SEARCH_KEY and _MAX_RESULTS.
311 def _make_valid_url(cls):
312 return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
315 def suitable(cls, url):
316 return re.match(cls._make_valid_url(), url) is not None
318 def _real_extract(self, query):
319 mobj = re.match(self._make_valid_url(), query)
321 raise ExtractorError(u'Invalid search query "%s"' % query)
323 prefix = mobj.group('prefix')
324 query = mobj.group('query')
326 return self._get_n_results(query, 1)
327 elif prefix == 'all':
328 return self._get_n_results(query, self._MAX_RESULTS)
332 raise ExtractorError(u'invalid download number %s for query "%s"' % (n, query))
333 elif n > self._MAX_RESULTS:
334 self._downloader.report_warning(u'%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
335 n = self._MAX_RESULTS
336 return self._get_n_results(query, n)
338 def _get_n_results(self, query, n):
339 """Get a specified number of results for a query"""
340 raise NotImplementedError("This method must be implemented by sublclasses")
343 def SEARCH_KEY(self):
344 return self._SEARCH_KEY