clean_html,
compiled_regex_type,
ExtractorError,
+ RegexNotFoundError,
+ sanitize_filename,
unescapeHTML,
)
* ext Will be calculated from url if missing
* format A human-readable description of the format
("mp4 container with h264/opus").
- Calculated from width and height if missing.
+ Calculated from the format_id, width, height.
+ and format_note fields if missing.
* format_id A short description of the format
("mp4_h264_opus" or "19")
+ * format_note Additional info about the format
+ ("3D" or "DASH video")
* width Width of the video, if known
* height Height of the video, if known
+ * abr Average audio bitrate in KBit/s
+ * acodec Name of the audio codec in use
+ * vbr Average video bitrate in KBit/s
+ * vcodec Name of the video codec in use
+ webpage_url: The url to the video webpage, if given to youtube-dl it
+ should allow to get the same result again. (It will be set
+ by YoutubeDL if it's missing)
Unless mentioned otherwise, the fields should be Unicode strings.
self.to_screen(u'Dumping request to ' + url)
dump = base64.b64encode(webpage_bytes).decode('ascii')
self._downloader.to_screen(dump)
+ if self._downloader.params.get('write_pages', False):
+ try:
+ url = url_or_request.get_full_url()
+ except AttributeError:
+ url = url_or_request
+ raw_filename = ('%s_%s.dump' % (video_id, url))
+ filename = sanitize_filename(raw_filename, restricted=True)
+ self.to_screen(u'Saving request to ' + filename)
+ with open(filename, 'wb') as outf:
+ outf.write(webpage_bytes)
+
content = webpage_bytes.decode(encoding, 'replace')
return (content, urlh)
Perform a regex search on the given string, using a single or a list of
patterns returning the first matching group.
In case of failure return a default value or raise a WARNING or a
- ExtractorError, depending on fatal, specifying the field name.
+ RegexNotFoundError, depending on fatal, specifying the field name.
"""
if isinstance(pattern, (str, compat_str, compiled_regex_type)):
mobj = re.search(pattern, string, flags)
elif default is not None:
return default
elif fatal:
- raise ExtractorError(u'Unable to extract %s' % _name)
+ raise RegexNotFoundError(u'Unable to extract %s' % _name)
else:
self._downloader.report_warning(u'unable to extract %s; '
u'please report this issue on http://yt-dl.org/bug' % _name)
# Helper functions for extracting OpenGraph info
@staticmethod
- def _og_regex(prop):
- return r'<meta.+?property=[\'"]og:%s[\'"].+?content=(?:"(.+?)"|\'(.+?)\')' % re.escape(prop)
+ def _og_regexes(prop):
+ content_re = r'content=(?:"([^>]+?)"|\'(.+?)\')'
+ property_re = r'property=[\'"]og:%s[\'"]' % re.escape(prop)
+ template = r'<meta[^>]+?%s[^>]+?%s'
+ return [
+ template % (property_re, content_re),
+ template % (content_re, property_re),
+ ]
def _og_search_property(self, prop, html, name=None, **kargs):
if name is None:
name = 'OpenGraph %s' % prop
- escaped = self._search_regex(self._og_regex(prop), html, name, flags=re.DOTALL, **kargs)
+ escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
+ if escaped is None:
+ return None
return unescapeHTML(escaped)
def _og_search_thumbnail(self, html, **kargs):
def _og_search_title(self, html, **kargs):
return self._og_search_property('title', html, **kargs)
- def _og_search_video_url(self, html, name='video url', **kargs):
- return self._html_search_regex([self._og_regex('video:secure_url'),
- self._og_regex('video')],
- html, name, **kargs)
+ def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
+ regexes = self._og_regexes('video')
+ if secure: regexes = self._og_regexes('video:secure_url') + regexes
+ return self._html_search_regex(regexes, html, name, **kargs)
+
+ def _html_search_meta(self, name, html, display_name=None):
+ if display_name is None:
+ display_name = name
+ return self._html_search_regex(
+ r'''(?ix)<meta(?=[^>]+(?:name|property)=["\']%s["\'])
+ [^>]+content=["\']([^"\']+)["\']''' % re.escape(name),
+ html, display_name, fatal=False)
+
+ def _dc_search_uploader(self, html):
+ return self._html_search_meta('dc.creator', html, 'uploader')
def _rta_search(self, html):
# See http://www.rtalabel.org/index.php?content=howtofaq#single
return 18
return 0
+ def _media_rating_search(self, html):
+ # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
+ rating = self._html_search_meta('rating', html)
+
+ if not rating:
+ return None
+
+ RATING_TABLE = {
+ 'safe for kids': 0,
+ 'general': 8,
+ '14 years': 14,
+ 'mature': 17,
+ 'restricted': 19,
+ }
+ return RATING_TABLE.get(rating.lower(), None)
+
+
class SearchInfoExtractor(InfoExtractor):
"""
projects / youtube-dl / blobdiff