1 from __future__ import unicode_literals
13 import xml.etree.ElementTree
15 from ..compat import (
19 compat_urllib_parse_urlparse,
33 _NO_DEFAULT = object()
36 class InfoExtractor(object):
37 """Information Extractor class.
39 Information extractors are the classes that, given a URL, extract
40 information about the video (or videos) the URL refers to. This
41 information includes the real video URL, the video title, author and
42 others. The information is stored in a dictionary which is then
43 passed to the FileDownloader. The FileDownloader processes this
44 information possibly downloading the video to the file system, among
45 other possible outcomes.
47 The type field determines the the type of the result.
48 By far the most common value (and the default if _type is missing) is
49 "video", which indicates a single video.
51 For a video, the dictionaries must include the following fields:
54 title: Video title, unescaped.
56 Additionally, it must contain either a formats entry or a url one:
58 formats: A list of dictionaries for each format available, ordered
59 from worst to best quality.
62 * url Mandatory. The URL of the video file
63 * ext Will be calculated from url if missing
64 * format A human-readable description of the format
65 ("mp4 container with h264/opus").
66 Calculated from the format_id, width, height.
67 and format_note fields if missing.
68 * format_id A short description of the format
69 ("mp4_h264_opus" or "19").
70 Technically optional, but strongly recommended.
71 * format_note Additional info about the format
72 ("3D" or "DASH video")
73 * width Width of the video, if known
74 * height Height of the video, if known
75 * resolution Textual description of width and height
76 * tbr Average bitrate of audio and video in KBit/s
77 * abr Average audio bitrate in KBit/s
78 * acodec Name of the audio codec in use
79 * asr Audio sampling rate in Hertz
80 * vbr Average video bitrate in KBit/s
82 * vcodec Name of the video codec in use
83 * container Name of the container format
84 * filesize The number of bytes, if known in advance
85 * filesize_approx An estimate for the number of bytes
86 * player_url SWF Player URL (used for rtmpdump).
87 * protocol The protocol that will be used for the actual
89 "http", "https", "rtsp", "rtmp", "m3u8" or so.
90 * preference Order number of this format. If this field is
91 present and not None, the formats get sorted
92 by this field, regardless of all other values.
93 -1 for default (order by other properties),
94 -2 or smaller for less than default.
95 * language_preference Is this in the correct requested
97 10 if it's what the URL is about,
98 -1 for default (don't know),
99 -10 otherwise, other values reserved for now.
100 * quality Order number of the video quality of this
101 format, irrespective of the file format.
102 -1 for default (order by other properties),
103 -2 or smaller for less than default.
104 * source_preference Order number for this video source
105 (quality takes higher priority)
106 -1 for default (order by other properties),
107 -2 or smaller for less than default.
108 * http_referer HTTP Referer header value to set.
109 * http_method HTTP method to use for the download.
110 * http_headers A dictionary of additional HTTP headers
111 to add to the request.
112 * http_post_data Additional data to send with a POST
114 url: Final video URL.
115 ext: Video filename extension.
116 format: The video format, defaults to ext (used for --get-format)
117 player_url: SWF Player URL (used for rtmpdump).
119 The following fields are optional:
121 alt_title: A secondary title of the video.
122 display_id An alternative identifier for the video, not necessarily
123 unique, but available before title. Typically, id is
124 something like "4234987", title "Dancing naked mole rats",
125 and display_id "dancing-naked-mole-rats"
126 thumbnails: A list of dictionaries, with the following entries:
128 * "width" (optional, int)
129 * "height" (optional, int)
130 * "resolution" (optional, string "{width}x{height"},
132 thumbnail: Full URL to a video thumbnail image.
133 description: Full video description.
134 uploader: Full name of the video uploader.
135 timestamp: UNIX timestamp of the moment the video became available.
136 upload_date: Video upload date (YYYYMMDD).
137 If not explicitly set, calculated from timestamp.
138 uploader_id: Nickname or id of the video uploader.
139 location: Physical location where the video was filmed.
140 subtitles: The subtitle file contents as a dictionary in the format
141 {language: subtitles}.
142 duration: Length of the video in seconds, as an integer.
143 view_count: How many users have watched the video on the platform.
144 like_count: Number of positive ratings of the video
145 dislike_count: Number of negative ratings of the video
146 comment_count: Number of comments on the video
147 age_limit: Age restriction for the video, as an integer (years)
148 webpage_url: The url to the video webpage, if given to youtube-dl it
149 should allow to get the same result again. (It will be set
150 by YoutubeDL if it's missing)
151 categories: A list of categories that the video falls in, for example
153 is_live: True, False, or None (=unknown). Whether this video is a
154 live stream that goes on instead of a fixed-length video.
156 Unless mentioned otherwise, the fields should be Unicode strings.
158 Unless mentioned otherwise, None is equivalent to absence of information.
161 _type "playlist" indicates multiple videos.
162 There must be a key "entries", which is a list, an iterable, or a PagedList
163 object, each element of which is a valid dictionary by this specification.
165 Additionally, playlists can have "title" and "id" attributes with the same
166 semantics as videos (see above).
169 _type "multi_video" indicates that there are multiple videos that
170 form a single show, for examples multiple acts of an opera or TV episode.
171 It must have an entries key like a playlist and contain all the keys
172 required for a video at the same time.
175 _type "url" indicates that the video must be extracted from another
176 location, possibly by a different extractor. Its only required key is:
177 "url" - the next URL to extract.
178 The key "ie_key" can be set to the class name (minus the trailing "IE",
179 e.g. "Youtube") if the extractor class is known in advance.
180 Additionally, the dictionary may have any properties of the resolved entity
181 known in advance, for example "title" if the title of the referred video is
185 _type "url_transparent" entities have the same specification as "url", but
186 indicate that the given additional information is more precise than the one
187 associated with the resolved URL.
188 This is useful when a site employs a video service that hosts the video and
189 its technical metadata, but that video service does not embed a useful
190 title, description etc.
193 Subclasses of this one should re-define the _real_initialize() and
194 _real_extract() methods and define a _VALID_URL regexp.
195 Probably, they should also be added to the list of extractors.
197 Finally, the _WORKING attribute should be set to False for broken IEs
198 in order to warn the users and skip the tests.
205 def __init__(self, downloader=None):
206 """Constructor. Receives an optional downloader."""
208 self.set_downloader(downloader)
211 def suitable(cls, url):
212 """Receives a URL and returns True if suitable for this IE."""
214 # This does not use has/getattr intentionally - we want to know whether
215 # we have cached the regexp for *this* class, whereas getattr would also
216 # match the superclass
217 if '_VALID_URL_RE' not in cls.__dict__:
218 cls._VALID_URL_RE = re.compile(cls._VALID_URL)
219 return cls._VALID_URL_RE.match(url) is not None
222 def _match_id(cls, url):
223 if '_VALID_URL_RE' not in cls.__dict__:
224 cls._VALID_URL_RE = re.compile(cls._VALID_URL)
225 m = cls._VALID_URL_RE.match(url)
231 """Getter method for _WORKING."""
234 def initialize(self):
235 """Initializes an instance (authentication, etc)."""
237 self._real_initialize()
240 def extract(self, url):
241 """Extracts URL information and returns it in list of dicts."""
243 return self._real_extract(url)
245 def set_downloader(self, downloader):
246 """Sets the downloader for this IE."""
247 self._downloader = downloader
249 def _real_initialize(self):
250 """Real initialization process. Redefine in subclasses."""
253 def _real_extract(self, url):
254 """Real extraction process. Redefine in subclasses."""
259 """A string for getting the InfoExtractor with get_info_extractor"""
260 return cls.__name__[:-2]
264 return type(self).__name__[:-2]
266 def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
267 """ Returns the response handle """
269 self.report_download_webpage(video_id)
270 elif note is not False:
272 self.to_screen('%s' % (note,))
274 self.to_screen('%s: %s' % (video_id, note))
276 return self._downloader.urlopen(url_or_request)
277 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
281 errnote = 'Unable to download webpage'
282 errmsg = '%s: %s' % (errnote, compat_str(err))
284 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
286 self._downloader.report_warning(errmsg)
289 def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
290 """ Returns a tuple (page content as string, URL handle) """
291 # Strip hashes from the URL (#1038)
292 if isinstance(url_or_request, (compat_str, str)):
293 url_or_request = url_or_request.partition('#')[0]
295 urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
299 content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal)
300 return (content, urlh)
302 def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None):
303 content_type = urlh.headers.get('Content-Type', '')
304 webpage_bytes = urlh.read()
305 if prefix is not None:
306 webpage_bytes = prefix + webpage_bytes
307 m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
309 encoding = m.group(1)
311 m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
312 webpage_bytes[:1024])
314 encoding = m.group(1).decode('ascii')
315 elif webpage_bytes.startswith(b'\xff\xfe'):
319 if self._downloader.params.get('dump_intermediate_pages', False):
321 url = url_or_request.get_full_url()
322 except AttributeError:
324 self.to_screen('Dumping request to ' + url)
325 dump = base64.b64encode(webpage_bytes).decode('ascii')
326 self._downloader.to_screen(dump)
327 if self._downloader.params.get('write_pages', False):
329 url = url_or_request.get_full_url()
330 except AttributeError:
332 basen = '%s_%s' % (video_id, url)
334 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
335 basen = basen[:240 - len(h)] + h
336 raw_filename = basen + '.dump'
337 filename = sanitize_filename(raw_filename, restricted=True)
338 self.to_screen('Saving request to ' + filename)
339 # Working around MAX_PATH limitation on Windows (see
340 # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
342 absfilepath = os.path.abspath(filename)
343 if len(absfilepath) > 259:
344 filename = '\\\\?\\' + absfilepath
345 with open(filename, 'wb') as outf:
346 outf.write(webpage_bytes)
349 content = webpage_bytes.decode(encoding, 'replace')
351 content = webpage_bytes.decode('utf-8', 'replace')
353 if ('<title>Access to this site is blocked</title>' in content and
354 'Websense' in content[:512]):
355 msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
356 blocked_iframe = self._html_search_regex(
357 r'<iframe src="([^"]+)"', content,
358 'Websense information URL', default=None)
360 msg += ' Visit %s for more details' % blocked_iframe
361 raise ExtractorError(msg, expected=True)
365 def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
366 """ Returns the data of the page as a string """
367 res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
374 def _download_xml(self, url_or_request, video_id,
375 note='Downloading XML', errnote='Unable to download XML',
376 transform_source=None, fatal=True):
377 """Return the xml as an xml.etree.ElementTree.Element"""
378 xml_string = self._download_webpage(
379 url_or_request, video_id, note, errnote, fatal=fatal)
380 if xml_string is False:
383 xml_string = transform_source(xml_string)
384 return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
386 def _download_json(self, url_or_request, video_id,
387 note='Downloading JSON metadata',
388 errnote='Unable to download JSON metadata',
389 transform_source=None,
391 json_string = self._download_webpage(
392 url_or_request, video_id, note, errnote, fatal=fatal)
393 if (not fatal) and json_string is False:
396 json_string = transform_source(json_string)
398 return json.loads(json_string)
399 except ValueError as ve:
400 errmsg = '%s: Failed to parse JSON ' % video_id
402 raise ExtractorError(errmsg, cause=ve)
404 self.report_warning(errmsg + str(ve))
406 def report_warning(self, msg, video_id=None):
407 idstr = '' if video_id is None else '%s: ' % video_id
408 self._downloader.report_warning(
409 '[%s] %s%s' % (self.IE_NAME, idstr, msg))
411 def to_screen(self, msg):
412 """Print msg to screen, prefixing it with '[ie_name]'"""
413 self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
415 def report_extraction(self, id_or_name):
416 """Report information extraction."""
417 self.to_screen('%s: Extracting information' % id_or_name)
419 def report_download_webpage(self, video_id):
420 """Report webpage download."""
421 self.to_screen('%s: Downloading webpage' % video_id)
423 def report_age_confirmation(self):
424 """Report attempt to confirm age."""
425 self.to_screen('Confirming age')
427 def report_login(self):
428 """Report attempt to log in."""
429 self.to_screen('Logging in')
431 # Methods for following #608
433 def url_result(url, ie=None, video_id=None):
434 """Returns a url that points to a page that should be processed"""
435 # TODO: ie should be the class used for getting the info
436 video_info = {'_type': 'url',
439 if video_id is not None:
440 video_info['id'] = video_id
444 def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
445 """Returns a playlist"""
446 video_info = {'_type': 'playlist',
449 video_info['id'] = playlist_id
451 video_info['title'] = playlist_title
452 if playlist_description:
453 video_info['description'] = playlist_description
456 def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
458 Perform a regex search on the given string, using a single or a list of
459 patterns returning the first matching group.
460 In case of failure return a default value or raise a WARNING or a
461 RegexNotFoundError, depending on fatal, specifying the field name.
463 if isinstance(pattern, (str, compat_str, compiled_regex_type)):
464 mobj = re.search(pattern, string, flags)
467 mobj = re.search(p, string, flags)
471 if os.name != 'nt' and sys.stderr.isatty():
472 _name = '\033[0;34m%s\033[0m' % name
478 # return the first matching group
479 return next(g for g in mobj.groups() if g is not None)
481 return mobj.group(group)
482 elif default is not _NO_DEFAULT:
485 raise RegexNotFoundError('Unable to extract %s' % _name)
487 self._downloader.report_warning('unable to extract %s; '
488 'please report this issue on http://yt-dl.org/bug' % _name)
491 def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
493 Like _search_regex, but strips HTML tags and unescapes entities.
495 res = self._search_regex(pattern, string, name, default, fatal, flags, group)
497 return clean_html(res).strip()
501 def _get_login_info(self):
503 Get the the login info as (username, password)
504 It will look in the netrc file using the _NETRC_MACHINE value
505 If there's no info available, return (None, None)
507 if self._downloader is None:
512 downloader_params = self._downloader.params
514 # Attempt to use provided username and password or .netrc data
515 if downloader_params.get('username', None) is not None:
516 username = downloader_params['username']
517 password = downloader_params['password']
518 elif downloader_params.get('usenetrc', False):
520 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
525 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
526 except (IOError, netrc.NetrcParseError) as err:
527 self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
529 return (username, password)
531 def _get_tfa_info(self):
533 Get the two-factor authentication info
534 TODO - asking the user will be required for sms/phone verify
535 currently just uses the command line option
536 If there's no info available, return None
538 if self._downloader is None:
540 downloader_params = self._downloader.params
542 if downloader_params.get('twofactor', None) is not None:
543 return downloader_params['twofactor']
547 # Helper functions for extracting OpenGraph info
549 def _og_regexes(prop):
550 content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
551 property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
552 template = r'<meta[^>]+?%s[^>]+?%s'
554 template % (property_re, content_re),
555 template % (content_re, property_re),
558 def _og_search_property(self, prop, html, name=None, **kargs):
560 name = 'OpenGraph %s' % prop
561 escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
564 return unescapeHTML(escaped)
566 def _og_search_thumbnail(self, html, **kargs):
567 return self._og_search_property('image', html, 'thumbnail url', fatal=False, **kargs)
569 def _og_search_description(self, html, **kargs):
570 return self._og_search_property('description', html, fatal=False, **kargs)
572 def _og_search_title(self, html, **kargs):
573 return self._og_search_property('title', html, **kargs)
575 def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
576 regexes = self._og_regexes('video') + self._og_regexes('video:url')
578 regexes = self._og_regexes('video:secure_url') + regexes
579 return self._html_search_regex(regexes, html, name, **kargs)
581 def _og_search_url(self, html, **kargs):
582 return self._og_search_property('url', html, **kargs)
584 def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
585 if display_name is None:
587 return self._html_search_regex(
589 (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
590 [^>]+content=(["\'])(?P<content>.*?)\1''' % re.escape(name),
591 html, display_name, fatal=fatal, group='content', **kwargs)
593 def _dc_search_uploader(self, html):
594 return self._html_search_meta('dc.creator', html, 'uploader')
596 def _rta_search(self, html):
597 # See http://www.rtalabel.org/index.php?content=howtofaq#single
598 if re.search(r'(?ix)<meta\s+name="rating"\s+'
599 r' content="RTA-5042-1996-1400-1577-RTA"',
604 def _media_rating_search(self, html):
605 # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
606 rating = self._html_search_meta('rating', html)
618 return RATING_TABLE.get(rating.lower(), None)
620 def _twitter_search_player(self, html):
621 return self._html_search_meta('twitter:player', html,
622 'twitter card player')
624 def _sort_formats(self, formats):
626 raise ExtractorError('No video formats found')
629 # TODO remove the following workaround
630 from ..utils import determine_ext
631 if not f.get('ext') and 'url' in f:
632 f['ext'] = determine_ext(f['url'])
634 preference = f.get('preference')
635 if preference is None:
636 proto = f.get('protocol')
638 proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
640 preference = 0 if proto in ['http', 'https'] else -0.1
641 if f.get('ext') in ['f4f', 'f4m']: # Not yet supported
644 if f.get('vcodec') == 'none': # audio only
645 if self._downloader.params.get('prefer_free_formats'):
646 ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
648 ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
651 audio_ext_preference = ORDER.index(f['ext'])
653 audio_ext_preference = -1
655 if self._downloader.params.get('prefer_free_formats'):
656 ORDER = ['flv', 'mp4', 'webm']
658 ORDER = ['webm', 'flv', 'mp4']
660 ext_preference = ORDER.index(f['ext'])
663 audio_ext_preference = 0
667 f.get('language_preference') if f.get('language_preference') is not None else -1,
668 f.get('quality') if f.get('quality') is not None else -1,
669 f.get('height') if f.get('height') is not None else -1,
670 f.get('width') if f.get('width') is not None else -1,
672 f.get('tbr') if f.get('tbr') is not None else -1,
673 f.get('vbr') if f.get('vbr') is not None else -1,
674 f.get('abr') if f.get('abr') is not None else -1,
675 audio_ext_preference,
676 f.get('fps') if f.get('fps') is not None else -1,
677 f.get('filesize') if f.get('filesize') is not None else -1,
678 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
679 f.get('source_preference') if f.get('source_preference') is not None else -1,
682 formats.sort(key=_formats_key)
684 def http_scheme(self):
685 """ Either "http:" or "https:", depending on the user's preferences """
688 if self._downloader.params.get('prefer_insecure', False)
691 def _proto_relative_url(self, url, scheme=None):
694 if url.startswith('//'):
696 scheme = self.http_scheme()
701 def _sleep(self, timeout, video_id, msg_template=None):
702 if msg_template is None:
703 msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
704 msg = msg_template % {'video_id': video_id, 'timeout': timeout}
708 def _extract_f4m_formats(self, manifest_url, video_id):
709 manifest = self._download_xml(
710 manifest_url, video_id, 'Downloading f4m manifest',
711 'Unable to download f4m manifest')
714 media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
715 for i, media_el in enumerate(media_nodes):
716 tbr = int_or_none(media_el.attrib.get('bitrate'))
717 format_id = 'f4m-%d' % (i if tbr is None else tbr)
719 'format_id': format_id,
723 'width': int_or_none(media_el.attrib.get('width')),
724 'height': int_or_none(media_el.attrib.get('height')),
726 self._sort_formats(formats)
730 def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
731 entry_protocol='m3u8', preference=None):
734 'format_id': 'm3u8-meta',
739 'resolution': 'multiple',
740 'format_note': 'Quality selection URL',
743 format_url = lambda u: (
745 if re.match(r'^https?://', u)
746 else compat_urlparse.urljoin(m3u8_url, u))
748 m3u8_doc = self._download_webpage(
750 note='Downloading m3u8 information',
751 errnote='Failed to download m3u8 information')
754 r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
755 for line in m3u8_doc.splitlines():
756 if line.startswith('#EXT-X-STREAM-INF:'):
758 for m in kv_rex.finditer(line):
760 if v.startswith('"'):
762 last_info[m.group('key')] = v
763 elif line.startswith('#') or not line.strip():
766 if last_info is None:
767 formats.append({'url': format_url(line)})
769 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
772 'format_id': 'm3u8-%d' % (tbr if tbr else len(formats)),
773 'url': format_url(line.strip()),
776 'protocol': entry_protocol,
777 'preference': preference,
779 codecs = last_info.get('CODECS')
781 # TODO: looks like video codec is not always necessarily goes first
782 va_codecs = codecs.split(',')
784 f['vcodec'] = va_codecs[0].partition('.')[0]
785 if len(va_codecs) > 1 and va_codecs[1]:
786 f['acodec'] = va_codecs[1].partition('.')[0]
787 resolution = last_info.get('RESOLUTION')
789 width_str, height_str = resolution.split('x')
790 f['width'] = int(width_str)
791 f['height'] = int(height_str)
794 self._sort_formats(formats)
797 # TODO: improve extraction
798 def _extract_smil_formats(self, smil_url, video_id):
799 smil = self._download_xml(
800 smil_url, video_id, 'Downloading SMIL file',
801 'Unable to download SMIL file')
803 base = smil.find('./head/meta').get('base')
807 for video in smil.findall('./body/switch/video'):
808 src = video.get('src')
811 bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
812 width = int_or_none(video.get('width'))
813 height = int_or_none(video.get('height'))
814 proto = video.get('proto')
817 if base.startswith('rtmp'):
819 elif base.startswith('http'):
821 ext = video.get('ext')
823 formats.extend(self._extract_m3u8_formats(src, video_id, ext))
824 elif proto == 'rtmp':
826 streamer = video.get('streamer') or base
831 'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
836 self._sort_formats(formats)
840 def _live_title(self, name):
841 """ Generate the title for a live video """
842 now = datetime.datetime.now()
843 now_str = now.strftime("%Y-%m-%d %H:%M")
844 return name + ' ' + now_str
846 def _int(self, v, name, fatal=False, **kwargs):
847 res = int_or_none(v, **kwargs)
848 if 'get_attr' in kwargs:
849 print(getattr(v, kwargs['get_attr']))
851 msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
853 raise ExtractorError(msg)
855 self._downloader.report_warning(msg)
858 def _float(self, v, name, fatal=False, **kwargs):
859 res = float_or_none(v, **kwargs)
861 msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
863 raise ExtractorError(msg)
865 self._downloader.report_warning(msg)
868 def _set_cookie(self, domain, name, value, expire_time=None):
869 cookie = compat_cookiejar.Cookie(
870 0, name, value, None, None, domain, None,
871 None, '/', True, False, expire_time, '', None, None, None)
872 self._downloader.cookiejar.set_cookie(cookie)
875 class SearchInfoExtractor(InfoExtractor):
877 Base class for paged search queries extractors.
878 They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
879 Instances should define _SEARCH_KEY and _MAX_RESULTS.
883 def _make_valid_url(cls):
884 return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
887 def suitable(cls, url):
888 return re.match(cls._make_valid_url(), url) is not None
890 def _real_extract(self, query):
891 mobj = re.match(self._make_valid_url(), query)
893 raise ExtractorError('Invalid search query "%s"' % query)
895 prefix = mobj.group('prefix')
896 query = mobj.group('query')
898 return self._get_n_results(query, 1)
899 elif prefix == 'all':
900 return self._get_n_results(query, self._MAX_RESULTS)
904 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
905 elif n > self._MAX_RESULTS:
906 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
907 n = self._MAX_RESULTS
908 return self._get_n_results(query, n)
910 def _get_n_results(self, query, n):
911 """Get a specified number of results for a query"""
912 raise NotImplementedError("This method must be implemented by subclasses")
915 def SEARCH_KEY(self):
916 return self._SEARCH_KEY