1 from __future__ import unicode_literals
13 import xml.etree.ElementTree
18 compat_urllib_parse_urlparse,
30 _NO_DEFAULT = object()
33 class InfoExtractor(object):
34 """Information Extractor class.
36 Information extractors are the classes that, given a URL, extract
37 information about the video (or videos) the URL refers to. This
38 information includes the real video URL, the video title, author and
39 others. The information is stored in a dictionary which is then
40 passed to the FileDownloader. The FileDownloader processes this
41 information possibly downloading the video to the file system, among
42 other possible outcomes.
44 The dictionaries must include the following fields:
47 title: Video title, unescaped.
49 Additionally, it must contain either a formats entry or a url one:
51 formats: A list of dictionaries for each format available, ordered
52 from worst to best quality.
55 * url Mandatory. The URL of the video file
56 * ext Will be calculated from url if missing
57 * format A human-readable description of the format
58 ("mp4 container with h264/opus").
59 Calculated from the format_id, width, height.
60 and format_note fields if missing.
61 * format_id A short description of the format
62 ("mp4_h264_opus" or "19").
63 Technically optional, but strongly recommended.
64 * format_note Additional info about the format
65 ("3D" or "DASH video")
66 * width Width of the video, if known
67 * height Height of the video, if known
68 * resolution Textual description of width and height
69 * tbr Average bitrate of audio and video in KBit/s
70 * abr Average audio bitrate in KBit/s
71 * acodec Name of the audio codec in use
72 * asr Audio sampling rate in Hertz
73 * vbr Average video bitrate in KBit/s
74 * vcodec Name of the video codec in use
75 * container Name of the container format
76 * filesize The number of bytes, if known in advance
77 * filesize_approx An estimate for the number of bytes
78 * player_url SWF Player URL (used for rtmpdump).
79 * protocol The protocol that will be used for the actual
81 "http", "https", "rtsp", "rtmp", "m3u8" or so.
82 * preference Order number of this format. If this field is
83 present and not None, the formats get sorted
84 by this field, regardless of all other values.
85 -1 for default (order by other properties),
86 -2 or smaller for less than default.
87 * quality Order number of the video quality of this
88 format, irrespective of the file format.
89 -1 for default (order by other properties),
90 -2 or smaller for less than default.
91 * http_referer HTTP Referer header value to set.
92 * http_method HTTP method to use for the download.
93 * http_headers A dictionary of additional HTTP headers
94 to add to the request.
95 * http_post_data Additional data to send with a POST
98 ext: Video filename extension.
99 format: The video format, defaults to ext (used for --get-format)
100 player_url: SWF Player URL (used for rtmpdump).
102 The following fields are optional:
104 display_id An alternative identifier for the video, not necessarily
105 unique, but available before title. Typically, id is
106 something like "4234987", title "Dancing naked mole rats",
107 and display_id "dancing-naked-mole-rats"
108 thumbnails: A list of dictionaries, with the following entries:
110 * "width" (optional, int)
111 * "height" (optional, int)
112 * "resolution" (optional, string "{width}x{height"},
114 thumbnail: Full URL to a video thumbnail image.
115 description: One-line video description.
116 uploader: Full name of the video uploader.
117 timestamp: UNIX timestamp of the moment the video became available.
118 upload_date: Video upload date (YYYYMMDD).
119 If not explicitly set, calculated from timestamp.
120 uploader_id: Nickname or id of the video uploader.
121 location: Physical location where the video was filmed.
122 subtitles: The subtitle file contents as a dictionary in the format
123 {language: subtitles}.
124 duration: Length of the video in seconds, as an integer.
125 view_count: How many users have watched the video on the platform.
126 like_count: Number of positive ratings of the video
127 dislike_count: Number of negative ratings of the video
128 comment_count: Number of comments on the video
129 age_limit: Age restriction for the video, as an integer (years)
130 webpage_url: The url to the video webpage, if given to youtube-dl it
131 should allow to get the same result again. (It will be set
132 by YoutubeDL if it's missing)
133 categories: A list of categories that the video falls in, for example
135 is_live: True, False, or None (=unknown). Whether this video is a
136 live stream that goes on instead of a fixed-length video.
138 Unless mentioned otherwise, the fields should be Unicode strings.
140 Subclasses of this one should re-define the _real_initialize() and
141 _real_extract() methods and define a _VALID_URL regexp.
142 Probably, they should also be added to the list of extractors.
144 Finally, the _WORKING attribute should be set to False for broken IEs
145 in order to warn the users and skip the tests.
152 def __init__(self, downloader=None):
153 """Constructor. Receives an optional downloader."""
155 self.set_downloader(downloader)
158 def suitable(cls, url):
159 """Receives a URL and returns True if suitable for this IE."""
161 # This does not use has/getattr intentionally - we want to know whether
162 # we have cached the regexp for *this* class, whereas getattr would also
163 # match the superclass
164 if '_VALID_URL_RE' not in cls.__dict__:
165 cls._VALID_URL_RE = re.compile(cls._VALID_URL)
166 return cls._VALID_URL_RE.match(url) is not None
169 def _match_id(cls, url):
170 if '_VALID_URL_RE' not in cls.__dict__:
171 cls._VALID_URL_RE = re.compile(cls._VALID_URL)
172 m = cls._VALID_URL_RE.match(url)
178 """Getter method for _WORKING."""
181 def initialize(self):
182 """Initializes an instance (authentication, etc)."""
184 self._real_initialize()
187 def extract(self, url):
188 """Extracts URL information and returns it in list of dicts."""
190 return self._real_extract(url)
192 def set_downloader(self, downloader):
193 """Sets the downloader for this IE."""
194 self._downloader = downloader
196 def _real_initialize(self):
197 """Real initialization process. Redefine in subclasses."""
200 def _real_extract(self, url):
201 """Real extraction process. Redefine in subclasses."""
206 """A string for getting the InfoExtractor with get_info_extractor"""
207 return cls.__name__[:-2]
211 return type(self).__name__[:-2]
213 def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
214 """ Returns the response handle """
216 self.report_download_webpage(video_id)
217 elif note is not False:
219 self.to_screen('%s' % (note,))
221 self.to_screen('%s: %s' % (video_id, note))
223 return self._downloader.urlopen(url_or_request)
224 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
228 errnote = 'Unable to download webpage'
229 errmsg = '%s: %s' % (errnote, compat_str(err))
231 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
233 self._downloader.report_warning(errmsg)
236 def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
237 """ Returns a tuple (page content as string, URL handle) """
239 # Strip hashes from the URL (#1038)
240 if isinstance(url_or_request, (compat_str, str)):
241 url_or_request = url_or_request.partition('#')[0]
243 urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
247 content_type = urlh.headers.get('Content-Type', '')
248 webpage_bytes = urlh.read()
249 m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
251 encoding = m.group(1)
253 m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
254 webpage_bytes[:1024])
256 encoding = m.group(1).decode('ascii')
257 elif webpage_bytes.startswith(b'\xff\xfe'):
261 if self._downloader.params.get('dump_intermediate_pages', False):
263 url = url_or_request.get_full_url()
264 except AttributeError:
266 self.to_screen('Dumping request to ' + url)
267 dump = base64.b64encode(webpage_bytes).decode('ascii')
268 self._downloader.to_screen(dump)
269 if self._downloader.params.get('write_pages', False):
271 url = url_or_request.get_full_url()
272 except AttributeError:
274 basen = '%s_%s' % (video_id, url)
276 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
277 basen = basen[:240 - len(h)] + h
278 raw_filename = basen + '.dump'
279 filename = sanitize_filename(raw_filename, restricted=True)
280 self.to_screen('Saving request to ' + filename)
281 with open(filename, 'wb') as outf:
282 outf.write(webpage_bytes)
285 content = webpage_bytes.decode(encoding, 'replace')
287 content = webpage_bytes.decode('utf-8', 'replace')
289 if ('<title>Access to this site is blocked</title>' in content and
290 'Websense' in content[:512]):
291 msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
292 blocked_iframe = self._html_search_regex(
293 r'<iframe src="([^"]+)"', content,
294 'Websense information URL', default=None)
296 msg += ' Visit %s for more details' % blocked_iframe
297 raise ExtractorError(msg, expected=True)
299 return (content, urlh)
301 def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
302 """ Returns the data of the page as a string """
303 res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
310 def _download_xml(self, url_or_request, video_id,
311 note='Downloading XML', errnote='Unable to download XML',
312 transform_source=None, fatal=True):
313 """Return the xml as an xml.etree.ElementTree.Element"""
314 xml_string = self._download_webpage(
315 url_or_request, video_id, note, errnote, fatal=fatal)
316 if xml_string is False:
319 xml_string = transform_source(xml_string)
320 return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
322 def _download_json(self, url_or_request, video_id,
323 note='Downloading JSON metadata',
324 errnote='Unable to download JSON metadata',
325 transform_source=None,
327 json_string = self._download_webpage(
328 url_or_request, video_id, note, errnote, fatal=fatal)
329 if (not fatal) and json_string is False:
332 json_string = transform_source(json_string)
334 return json.loads(json_string)
335 except ValueError as ve:
336 raise ExtractorError('Failed to download JSON', cause=ve)
338 def report_warning(self, msg, video_id=None):
339 idstr = '' if video_id is None else '%s: ' % video_id
340 self._downloader.report_warning(
341 '[%s] %s%s' % (self.IE_NAME, idstr, msg))
343 def to_screen(self, msg):
344 """Print msg to screen, prefixing it with '[ie_name]'"""
345 self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
347 def report_extraction(self, id_or_name):
348 """Report information extraction."""
349 self.to_screen('%s: Extracting information' % id_or_name)
351 def report_download_webpage(self, video_id):
352 """Report webpage download."""
353 self.to_screen('%s: Downloading webpage' % video_id)
355 def report_age_confirmation(self):
356 """Report attempt to confirm age."""
357 self.to_screen('Confirming age')
359 def report_login(self):
360 """Report attempt to log in."""
361 self.to_screen('Logging in')
363 #Methods for following #608
365 def url_result(url, ie=None, video_id=None):
366 """Returns a url that points to a page that should be processed"""
367 #TODO: ie should be the class used for getting the info
368 video_info = {'_type': 'url',
371 if video_id is not None:
372 video_info['id'] = video_id
375 def playlist_result(entries, playlist_id=None, playlist_title=None):
376 """Returns a playlist"""
377 video_info = {'_type': 'playlist',
380 video_info['id'] = playlist_id
382 video_info['title'] = playlist_title
385 def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0):
387 Perform a regex search on the given string, using a single or a list of
388 patterns returning the first matching group.
389 In case of failure return a default value or raise a WARNING or a
390 RegexNotFoundError, depending on fatal, specifying the field name.
392 if isinstance(pattern, (str, compat_str, compiled_regex_type)):
393 mobj = re.search(pattern, string, flags)
396 mobj = re.search(p, string, flags)
400 if os.name != 'nt' and sys.stderr.isatty():
401 _name = '\033[0;34m%s\033[0m' % name
406 # return the first matching group
407 return next(g for g in mobj.groups() if g is not None)
408 elif default is not _NO_DEFAULT:
411 raise RegexNotFoundError('Unable to extract %s' % _name)
413 self._downloader.report_warning('unable to extract %s; '
414 'please report this issue on http://yt-dl.org/bug' % _name)
417 def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0):
419 Like _search_regex, but strips HTML tags and unescapes entities.
421 res = self._search_regex(pattern, string, name, default, fatal, flags)
423 return clean_html(res).strip()
427 def _get_login_info(self):
429 Get the the login info as (username, password)
430 It will look in the netrc file using the _NETRC_MACHINE value
431 If there's no info available, return (None, None)
433 if self._downloader is None:
438 downloader_params = self._downloader.params
440 # Attempt to use provided username and password or .netrc data
441 if downloader_params.get('username', None) is not None:
442 username = downloader_params['username']
443 password = downloader_params['password']
444 elif downloader_params.get('usenetrc', False):
446 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
451 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
452 except (IOError, netrc.NetrcParseError) as err:
453 self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
455 return (username, password)
457 def _get_tfa_info(self):
459 Get the two-factor authentication info
460 TODO - asking the user will be required for sms/phone verify
461 currently just uses the command line option
462 If there's no info available, return None
464 if self._downloader is None:
466 downloader_params = self._downloader.params
468 if downloader_params.get('twofactor', None) is not None:
469 return downloader_params['twofactor']
473 # Helper functions for extracting OpenGraph info
475 def _og_regexes(prop):
476 content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
477 property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
478 template = r'<meta[^>]+?%s[^>]+?%s'
480 template % (property_re, content_re),
481 template % (content_re, property_re),
484 def _og_search_property(self, prop, html, name=None, **kargs):
486 name = 'OpenGraph %s' % prop
487 escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
490 return unescapeHTML(escaped)
492 def _og_search_thumbnail(self, html, **kargs):
493 return self._og_search_property('image', html, 'thumbnail url', fatal=False, **kargs)
495 def _og_search_description(self, html, **kargs):
496 return self._og_search_property('description', html, fatal=False, **kargs)
498 def _og_search_title(self, html, **kargs):
499 return self._og_search_property('title', html, **kargs)
501 def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
502 regexes = self._og_regexes('video') + self._og_regexes('video:url')
504 regexes = self._og_regexes('video:secure_url') + regexes
505 return self._html_search_regex(regexes, html, name, **kargs)
507 def _og_search_url(self, html, **kargs):
508 return self._og_search_property('url', html, **kargs)
510 def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
511 if display_name is None:
513 return self._html_search_regex(
515 (?=[^>]+(?:itemprop|name|property)=["\']?%s["\']?)
516 [^>]+content=["\']([^"\']+)["\']''' % re.escape(name),
517 html, display_name, fatal=fatal, **kwargs)
519 def _dc_search_uploader(self, html):
520 return self._html_search_meta('dc.creator', html, 'uploader')
522 def _rta_search(self, html):
523 # See http://www.rtalabel.org/index.php?content=howtofaq#single
524 if re.search(r'(?ix)<meta\s+name="rating"\s+'
525 r' content="RTA-5042-1996-1400-1577-RTA"',
530 def _media_rating_search(self, html):
531 # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
532 rating = self._html_search_meta('rating', html)
544 return RATING_TABLE.get(rating.lower(), None)
546 def _twitter_search_player(self, html):
547 return self._html_search_meta('twitter:player', html,
548 'twitter card player')
550 def _sort_formats(self, formats):
552 raise ExtractorError('No video formats found')
555 # TODO remove the following workaround
556 from ..utils import determine_ext
557 if not f.get('ext') and 'url' in f:
558 f['ext'] = determine_ext(f['url'])
560 preference = f.get('preference')
561 if preference is None:
562 proto = f.get('protocol')
564 proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
566 preference = 0 if proto in ['http', 'https'] else -0.1
567 if f.get('ext') in ['f4f', 'f4m']: # Not yet supported
570 if f.get('vcodec') == 'none': # audio only
571 if self._downloader.params.get('prefer_free_formats'):
572 ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
574 ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
577 audio_ext_preference = ORDER.index(f['ext'])
579 audio_ext_preference = -1
581 if self._downloader.params.get('prefer_free_formats'):
582 ORDER = ['flv', 'mp4', 'webm']
584 ORDER = ['webm', 'flv', 'mp4']
586 ext_preference = ORDER.index(f['ext'])
589 audio_ext_preference = 0
593 f.get('quality') if f.get('quality') is not None else -1,
594 f.get('height') if f.get('height') is not None else -1,
595 f.get('width') if f.get('width') is not None else -1,
597 f.get('tbr') if f.get('tbr') is not None else -1,
598 f.get('vbr') if f.get('vbr') is not None else -1,
599 f.get('abr') if f.get('abr') is not None else -1,
600 audio_ext_preference,
601 f.get('filesize') if f.get('filesize') is not None else -1,
602 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
605 formats.sort(key=_formats_key)
607 def http_scheme(self):
608 """ Either "https:" or "https:", depending on the user's preferences """
611 if self._downloader.params.get('prefer_insecure', False)
614 def _proto_relative_url(self, url, scheme=None):
617 if url.startswith('//'):
619 scheme = self.http_scheme()
624 def _sleep(self, timeout, video_id, msg_template=None):
625 if msg_template is None:
626 msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
627 msg = msg_template % {'video_id': video_id, 'timeout': timeout}
631 def _extract_f4m_formats(self, manifest_url, video_id):
632 manifest = self._download_xml(
633 manifest_url, video_id, 'Downloading f4m manifest',
634 'Unable to download f4m manifest')
637 media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
638 for i, media_el in enumerate(media_nodes):
639 tbr = int_or_none(media_el.attrib.get('bitrate'))
640 format_id = 'f4m-%d' % (i if tbr is None else tbr)
642 'format_id': format_id,
646 'width': int_or_none(media_el.attrib.get('width')),
647 'height': int_or_none(media_el.attrib.get('height')),
649 self._sort_formats(formats)
653 def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
654 entry_protocol='m3u8', preference=None):
657 'format_id': 'm3u8-meta',
662 'resolution': 'multiple',
663 'format_note': 'Quality selection URL',
666 format_url = lambda u: (
668 if re.match(r'^https?://', u)
669 else compat_urlparse.urljoin(m3u8_url, u))
671 m3u8_doc = self._download_webpage(m3u8_url, video_id)
674 r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
675 for line in m3u8_doc.splitlines():
676 if line.startswith('#EXT-X-STREAM-INF:'):
678 for m in kv_rex.finditer(line):
680 if v.startswith('"'):
682 last_info[m.group('key')] = v
683 elif line.startswith('#') or not line.strip():
686 if last_info is None:
687 formats.append({'url': format_url(line)})
689 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
692 'format_id': 'm3u8-%d' % (tbr if tbr else len(formats)),
693 'url': format_url(line.strip()),
696 'protocol': entry_protocol,
697 'preference': preference,
699 codecs = last_info.get('CODECS')
701 # TODO: looks like video codec is not always necessarily goes first
702 va_codecs = codecs.split(',')
704 f['vcodec'] = va_codecs[0].partition('.')[0]
705 if len(va_codecs) > 1 and va_codecs[1]:
706 f['acodec'] = va_codecs[1].partition('.')[0]
707 resolution = last_info.get('RESOLUTION')
709 width_str, height_str = resolution.split('x')
710 f['width'] = int(width_str)
711 f['height'] = int(height_str)
714 self._sort_formats(formats)
717 def _live_title(self, name):
718 """ Generate the title for a live video """
719 now = datetime.datetime.now()
720 now_str = now.strftime("%Y-%m-%d %H:%M")
721 return name + ' ' + now_str
724 class SearchInfoExtractor(InfoExtractor):
726 Base class for paged search queries extractors.
727 They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
728 Instances should define _SEARCH_KEY and _MAX_RESULTS.
732 def _make_valid_url(cls):
733 return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
736 def suitable(cls, url):
737 return re.match(cls._make_valid_url(), url) is not None
739 def _real_extract(self, query):
740 mobj = re.match(self._make_valid_url(), query)
742 raise ExtractorError('Invalid search query "%s"' % query)
744 prefix = mobj.group('prefix')
745 query = mobj.group('query')
747 return self._get_n_results(query, 1)
748 elif prefix == 'all':
749 return self._get_n_results(query, self._MAX_RESULTS)
753 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
754 elif n > self._MAX_RESULTS:
755 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
756 n = self._MAX_RESULTS
757 return self._get_n_results(query, n)
759 def _get_n_results(self, query, n):
760 """Get a specified number of results for a query"""
761 raise NotImplementedError("This method must be implemented by subclasses")
764 def SEARCH_KEY(self):
765 return self._SEARCH_KEY