X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=src%2Flookup.rs;h=751ddf228145595332a42f884f5fe6424ac2dd1f;hb=74dd28d4e2c75a4dbb15b3c5045bddfeb2bd9c53;hp=fbc16729bb96cccccd7e6fec7d7ec72d0f349fcf;hpb=2488d9dfb30511675e192e08517fe517a6a96388;p=rapid-gossip-sync-server diff --git a/src/lookup.rs b/src/lookup.rs index fbc1672..751ddf2 100644 --- a/src/lookup.rs +++ b/src/lookup.rs @@ -1,16 +1,17 @@ -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::io::Cursor; use std::ops::Deref; use std::sync::Arc; use std::time::{Instant, SystemTime, UNIX_EPOCH}; -use lightning::ln::msgs::{ChannelAnnouncement, ChannelUpdate, UnsignedChannelAnnouncement, UnsignedChannelUpdate}; -use lightning::routing::gossip::NetworkGraph; +use lightning::ln::msgs::{ChannelAnnouncement, ChannelUpdate, NodeAnnouncement, SocketAddress, UnsignedChannelAnnouncement, UnsignedChannelUpdate}; +use lightning::routing::gossip::{NetworkGraph, NodeId}; use lightning::util::ser::Readable; use tokio_postgres::Client; use futures::StreamExt; -use lightning::{log_gossip, log_info}; +use lightning::{log_debug, log_gossip, log_info}; +use lightning::ln::features::NodeFeatures; use lightning::util::logger::Logger; use crate::config; @@ -19,6 +20,7 @@ use crate::serialization::MutatedProperties; /// The delta set needs to be a BTreeMap so the keys are sorted. /// That way, the scids in the response automatically grow monotonically pub(super) type DeltaSet = BTreeMap; +pub(super) type NodeDeltaSet = HashMap; pub(super) struct AnnouncementDelta { pub(super) seen: u32, @@ -50,6 +52,31 @@ pub(super) struct ChannelDelta { pub(super) requires_reminder: bool, } +pub(super) struct NodeDelta { + /// The most recently received, but new-to-the-client, node details + pub(super) latest_details_after_seen: Option, + + /// Between last_details_before_seen and latest_details_after_seen, including any potential + /// intermediate updates that are not kept track of here, has the set of features this node + /// supports changed? + pub(super) has_feature_set_changed: bool, + + /// Between last_details_before_seen and latest_details_after_seen, including any potential + /// intermediate updates that are not kept track of here, has the set of socket addresses this + /// node listens on changed? + pub(super) has_address_set_changed: bool, + + /// The most recent node details that the client would have seen already + pub(super) last_details_before_seen: Option +} + +pub(super) struct NodeDetails { + #[allow(unused)] + pub(super) seen: u32, + pub(super) features: NodeFeatures, + pub(super) addresses: HashSet +} + impl Default for ChannelDelta { fn default() -> Self { Self { @@ -61,6 +88,17 @@ impl Default for ChannelDelta { } } +impl Default for NodeDelta { + fn default() -> Self { + Self { + latest_details_after_seen: None, + has_feature_set_changed: false, + has_address_set_changed: false, + last_details_before_seen: None, + } + } +} + impl Default for DirectedUpdateDelta { fn default() -> Self { Self { @@ -76,14 +114,14 @@ impl Default for DirectedUpdateDelta { /// whether they had been seen before. /// Also include all announcements for which the first update was announced /// after `last_sync_timestamp` -pub(super) async fn fetch_channel_announcements(delta_set: &mut DeltaSet, network_graph: Arc>, client: &Client, last_sync_timestamp: u32, logger: L) where L::Target: Logger { +pub(super) async fn fetch_channel_announcements(delta_set: &mut DeltaSet, network_graph: Arc>, client: &Client, last_sync_timestamp: u32, snapshot_reference_timestamp: Option, logger: L) where L::Target: Logger { log_info!(logger, "Obtaining channel ids from network graph"); let channel_ids = { let read_only_graph = network_graph.read_only(); log_info!(logger, "Retrieved read-only network graph copy"); let channel_iterator = read_only_graph.channels().unordered_iter(); channel_iterator - .filter(|c| c.1.announcement_message.is_some()) + .filter(|c| c.1.announcement_message.is_some() && c.1.one_to_two.is_some() && c.1.two_to_one.is_some()) .map(|c| c.1.announcement_message.as_ref().unwrap().contents.short_channel_id as i64) .collect::>() }; @@ -92,6 +130,27 @@ pub(super) async fn fetch_channel_announcements(delta_set: &mut DeltaS log_info!(logger, "Last sync timestamp: {}", last_sync_timestamp); let last_sync_timestamp_float = last_sync_timestamp as f64; + let current_timestamp = snapshot_reference_timestamp.unwrap_or(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs()); + log_info!(logger, "Current timestamp: {}", current_timestamp); + + let include_reminders = { + let current_hour = current_timestamp / 3600; + let current_day = current_timestamp / (24 * 3600); + + log_debug!(logger, "Current day index: {}", current_day); + log_debug!(logger, "Current hour: {}", current_hour); + + // every 5th day at midnight + let is_reminder_hour = (current_hour % 24) == 0; + let is_reminder_day = (current_day % 5) == 0; + + let snapshot_scope = current_timestamp.saturating_sub(last_sync_timestamp as u64); + let is_reminder_scope = snapshot_scope > (50 * 3600); + log_debug!(logger, "Snapshot scope: {}s", snapshot_scope); + + (is_reminder_hour && is_reminder_day) || is_reminder_scope + }; + log_info!(logger, "Obtaining corresponding database entries"); // get all the channel announcements that are currently in the network graph let announcement_rows = client.query_raw("SELECT announcement_signed, CAST(EXTRACT('epoch' from seen) AS BIGINT) AS seen FROM channel_announcements WHERE short_channel_id = any($1) ORDER BY short_channel_id ASC", [&channel_ids]).await.unwrap(); @@ -162,17 +221,17 @@ pub(super) async fn fetch_channel_announcements(delta_set: &mut DeltaS log_info!(logger, "Fetched {} update rows of the first update in a new direction", newer_oldest_directional_update_count); } - { + if include_reminders { // THIS STEP IS USED TO DETERMINE IF A REMINDER UPDATE SHOULD BE SENT log_info!(logger, "Annotating channel announcements whose latest channel update in a given direction occurred more than six days ago"); // Steps: // — Obtain all updates, distinct by (scid, direction), ordered by seen DESC // — From those updates, select distinct by (scid), ordered by seen ASC (to obtain the older one per direction) - let reminder_threshold_timestamp = SystemTime::now().checked_sub(config::CHANNEL_REMINDER_AGE).unwrap().duration_since(UNIX_EPOCH).unwrap().as_secs() as f64; + let reminder_threshold_timestamp = current_timestamp.checked_sub(config::CHANNEL_REMINDER_AGE.as_secs()).unwrap() as f64; log_info!(logger, "Fetch first time we saw the current value combination for each direction (prior mutations excepted)"); - let reminder_lookup_threshold_timestamp = SystemTime::now().checked_sub(config::CHANNEL_REMINDER_AGE * 3).unwrap().duration_since(UNIX_EPOCH).unwrap().as_secs() as f64; + let reminder_lookup_threshold_timestamp = current_timestamp.checked_sub(config::CHANNEL_REMINDER_AGE.as_secs() * 3).unwrap() as f64; let params: [&(dyn tokio_postgres::types::ToSql + Sync); 2] = [&channel_ids, &reminder_lookup_threshold_timestamp]; /* @@ -344,7 +403,6 @@ pub(super) async fn fetch_channel_updates(delta_set: &mut DeltaSet, cl let mut previous_scid = u64::MAX; let mut previously_seen_directions = (false, false); - // let mut previously_seen_directions = (false, false); let mut intermediate_update_count = 0; while let Some(row_res) = pinned_updates.next().await { let intermediate_update = row_res.unwrap(); @@ -416,6 +474,108 @@ pub(super) async fn fetch_channel_updates(delta_set: &mut DeltaSet, cl log_info!(logger, "Processed intermediate rows ({}) (delta size: {}): {:?}", intermediate_update_count, delta_set.len(), start.elapsed()); } +pub(super) async fn fetch_node_updates(client: &Client, last_sync_timestamp: u32, logger: L) -> NodeDeltaSet where L::Target: Logger { + let start = Instant::now(); + let last_sync_timestamp_float = last_sync_timestamp as f64; + + let mut delta_set = NodeDeltaSet::new(); + + // get the latest node updates prior to last_sync_timestamp + let reference_rows = client.query_raw(" + SELECT DISTINCT ON (public_key) public_key, CAST(EXTRACT('epoch' from seen) AS BIGINT) AS seen, announcement_signed + FROM node_announcements + WHERE seen < TO_TIMESTAMP($1) + ORDER BY public_key ASC, seen DESC + ", [last_sync_timestamp_float]).await.unwrap(); + let mut pinned_rows = Box::pin(reference_rows); + + log_info!(logger, "Fetched node announcement reference rows in {:?}", start.elapsed()); + + let mut reference_row_count = 0; + + while let Some(row_res) = pinned_rows.next().await { + let current_reference = row_res.unwrap(); + + let seen = current_reference.get::<_, i64>("seen") as u32; + let blob: Vec = current_reference.get("announcement_signed"); + let mut readable = Cursor::new(blob); + let unsigned_node_announcement = NodeAnnouncement::read(&mut readable).unwrap().contents; + let node_id = unsigned_node_announcement.node_id; + + let current_node_delta = delta_set.entry(node_id).or_insert(NodeDelta::default()); + (*current_node_delta).last_details_before_seen.get_or_insert_with(|| { + let address_set: HashSet = unsigned_node_announcement.addresses.into_iter().collect(); + NodeDetails { + seen, + features: unsigned_node_announcement.features, + addresses: address_set, + } + }); + log_gossip!(logger, "Node {} last update before seen: {} (seen at {})", node_id, unsigned_node_announcement.timestamp, seen); + + reference_row_count += 1; + } + + + log_info!(logger, "Processed {} node announcement reference rows (delta size: {}) in {:?}", + reference_row_count, delta_set.len(), start.elapsed()); + + // get all the intermediate node updates + // (to calculate the set of mutated fields for snapshotting, where intermediate updates may + // have been omitted) + let intermediate_updates = client.query_raw(" + SELECT announcement_signed, CAST(EXTRACT('epoch' from seen) AS BIGINT) AS seen + FROM node_announcements + WHERE seen >= TO_TIMESTAMP($1) + ORDER BY public_key ASC, timestamp DESC + ", [last_sync_timestamp_float]).await.unwrap(); + let mut pinned_updates = Box::pin(intermediate_updates); + log_info!(logger, "Fetched intermediate node announcement rows in {:?}", start.elapsed()); + + let mut previous_node_id: Option = None; + + let mut intermediate_update_count = 0; + while let Some(row_res) = pinned_updates.next().await { + let intermediate_update = row_res.unwrap(); + intermediate_update_count += 1; + + let current_seen_timestamp = intermediate_update.get::<_, i64>("seen") as u32; + let blob: Vec = intermediate_update.get("announcement_signed"); + let mut readable = Cursor::new(blob); + let unsigned_node_announcement = NodeAnnouncement::read(&mut readable).unwrap().contents; + + let node_id = unsigned_node_announcement.node_id; + let is_previously_processed_node_id = Some(node_id) == previous_node_id; + + // get this node's address set + let current_node_delta = delta_set.entry(node_id).or_insert(NodeDelta::default()); + let address_set: HashSet = unsigned_node_announcement.addresses.into_iter().collect(); + + // determine mutations + if let Some(last_seen_update) = current_node_delta.last_details_before_seen.as_ref() { + if unsigned_node_announcement.features != last_seen_update.features { + current_node_delta.has_feature_set_changed = true; + } + if address_set != last_seen_update.addresses { + current_node_delta.has_address_set_changed = true; + } + } + + if !is_previously_processed_node_id { + (*current_node_delta).latest_details_after_seen.get_or_insert(NodeDetails { + seen: current_seen_timestamp, + features: unsigned_node_announcement.features, + addresses: address_set, + }); + } + + previous_node_id = Some(node_id); + } + log_info!(logger, "Processed intermediate node announcement rows ({}) (delta size: {}): {:?}", intermediate_update_count, delta_set.len(), start.elapsed()); + + delta_set +} + pub(super) fn filter_delta_set(delta_set: &mut DeltaSet, logger: L) where L::Target: Logger { let original_length = delta_set.len(); let keys: Vec = delta_set.keys().cloned().collect();