use std::sync::Arc;
use std::{fs, thread};
use std::time::{SystemTime, UNIX_EPOCH};
-use bitcoin::{BlockHash, Network};
+use bitcoin::blockdata::constants::ChainHash;
+use bitcoin::Network;
use bitcoin::secp256k1::ecdsa::Signature;
use bitcoin::secp256k1::{Secp256k1, SecretKey};
use bitcoin::hashes::Hash;
-use bitcoin::hashes::hex::ToHex;
use bitcoin::hashes::sha256d::Hash as Sha256dHash;
-use lightning::ln::features::ChannelFeatures;
-use lightning::ln::msgs::{ChannelAnnouncement, ChannelUpdate, UnsignedChannelAnnouncement, UnsignedChannelUpdate};
-use lightning::routing::gossip::{NetworkGraph, NodeId};
+use hex_conservative::DisplayHex;
+use lightning::ln::features::{ChannelFeatures, NodeFeatures};
+use lightning::ln::msgs::{ChannelAnnouncement, ChannelUpdate, NodeAnnouncement, SocketAddress, UnsignedChannelAnnouncement, UnsignedChannelUpdate, UnsignedNodeAnnouncement};
+use lightning::routing::gossip::{NetworkGraph, NodeAlias, NodeId};
use lightning::util::ser::Writeable;
use lightning_rapid_gossip_sync::RapidGossipSync;
use crate::{config, serialize_delta};
Signature::from_compact(&[0u8; 64]).unwrap()
}
-fn genesis_hash() -> BlockHash {
- bitcoin::blockdata::constants::genesis_block(Network::Bitcoin).block_hash()
+fn genesis_hash() -> ChainHash {
+ ChainHash::using_genesis_block(Network::Bitcoin)
}
fn current_time() -> u32 {
})
}
-fn generate_announcement(short_channel_id: u64) -> ChannelAnnouncement {
+fn generate_node_announcement() -> NodeAnnouncement {
+ let secp_context = Secp256k1::new();
+
+ let random_private_key = SecretKey::from_slice(&[1; 32]).unwrap();
+ let random_public_key = random_private_key.public_key(&secp_context);
+ let node_id = NodeId::from_pubkey(&random_public_key);
+
+ let announcement = UnsignedNodeAnnouncement {
+ features: NodeFeatures::empty(),
+ timestamp: 0,
+ node_id,
+ rgb: [0, 128, 255],
+ alias: NodeAlias([0; 32]),
+ addresses: vec![],
+ excess_data: vec![],
+ excess_address_data: vec![],
+ };
+
+ let msg_hash = bitcoin::secp256k1::Message::from_slice(&Sha256dHash::hash(&announcement.encode()[..])[..]).unwrap();
+ let signature = secp_context.sign_ecdsa(&msg_hash, &random_private_key);
+
+ NodeAnnouncement {
+ signature,
+ contents: announcement,
+ }
+}
+
+
+fn generate_channel_announcement(short_channel_id: u64) -> ChannelAnnouncement {
let secp_context = Secp256k1::new();
let random_private_key_1 = SecretKey::from_slice(&[1; 32]).unwrap();
let thread_id = thread::current().id();
let preimage = format!("{:?}-{}", thread_id, timestamp_nanos);
println!("test schema preimage: {}", preimage);
- let suffix = Sha256dHash::hash(preimage.as_bytes()).into_inner().to_hex();
+ let suffix = Sha256dHash::hash(preimage.as_bytes()).encode();
// the schema must start with a letter
- let schema = format!("test_{}_{}", timestamp_seconds, suffix);
+ let schema = format!("test_{}_{}", timestamp_seconds, suffix.as_hex());
*suffix_option = Some(schema);
});
IS_TEST_SCHEMA_CLEAN.with(|cleanliness_reference| {
let is_clean_option = cleanliness_reference.borrow();
if let Some(is_clean) = *is_clean_option {
+ if std::thread::panicking() {
+ return;
+ }
assert_eq!(is_clean, true);
}
});
});
}
+#[tokio::test]
+async fn test_persistence_runtime() {
+ let _sanitizer = SchemaSanitizer::new();
+ let logger = Arc::new(TestLogger::new());
+ let network_graph = NetworkGraph::new(Network::Bitcoin, logger.clone());
+ let network_graph_arc = Arc::new(network_graph);
+ let (_persister, _receiver) = GossipPersister::new(network_graph_arc.clone(), logger.clone());
+
+ tokio::task::spawn_blocking(move || {
+ drop(_persister);
+ }).await.unwrap();
+
+ clean_test_db().await;
+}
+
+
#[tokio::test]
async fn test_trivial_setup() {
let _sanitizer = SchemaSanitizer::new();
println!("timestamp: {}", timestamp);
{ // seed the db
- let announcement = generate_announcement(short_channel_id);
+ let announcement = generate_channel_announcement(short_channel_id);
let update_1 = generate_update(short_channel_id, false, timestamp, 0, 0, 0, 5, 0);
let update_2 = generate_update(short_channel_id, true, timestamp, 0, 0, 0, 10, 0);
persister.persist_gossip().await;
}
- let serialization = serialize_delta(network_graph_arc.clone(), 0, logger.clone()).await;
+ let serialization = serialize_delta(network_graph_arc.clone(), 0, None, logger.clone()).await;
logger.assert_log_contains("rapid_gossip_sync_server", "announcement channel count: 1", 1);
clean_test_db().await;
println!("last update b: {}", last_update_seen_b);
assert_eq!(last_update_seen_a, update_result - CLIENT_BACKDATE_INTERVAL);
assert_eq!(last_update_seen_b, update_result - CLIENT_BACKDATE_INTERVAL);
+
+ tokio::task::spawn_blocking(move || {
+ drop(persister);
+ }).await.unwrap();
}
+#[tokio::test]
+async fn test_node_announcement_persistence() {
+ let _sanitizer = SchemaSanitizer::new();
+ let logger = Arc::new(TestLogger::new());
+ let network_graph = NetworkGraph::new(Network::Bitcoin, logger.clone());
+ let network_graph_arc = Arc::new(network_graph);
+ let (mut persister, receiver) = GossipPersister::new(network_graph_arc.clone(), logger.clone());
+
+ { // seed the db
+ let mut announcement = generate_node_announcement();
+ receiver.send(GossipMessage::NodeAnnouncement(announcement.clone(), None)).await.unwrap();
+ receiver.send(GossipMessage::NodeAnnouncement(announcement.clone(), Some(12345))).await.unwrap();
+
+ {
+ // modify announcement to contain a bunch of addresses
+ announcement.contents.addresses.push(SocketAddress::Hostname {
+ hostname: "google.com".to_string().try_into().unwrap(),
+ port: 443,
+ });
+ announcement.contents.addresses.push(SocketAddress::TcpIpV4 { addr: [127, 0, 0, 1], port: 9635 });
+ announcement.contents.addresses.push(SocketAddress::TcpIpV6 { addr: [1; 16], port: 1337 });
+ announcement.contents.addresses.push(SocketAddress::OnionV2([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]));
+ announcement.contents.addresses.push(SocketAddress::OnionV3 {
+ ed25519_pubkey: [1; 32],
+ checksum: 2,
+ version: 3,
+ port: 4,
+ });
+ }
+ receiver.send(GossipMessage::NodeAnnouncement(announcement, Some(12345))).await.unwrap();
+
+ drop(receiver);
+ persister.persist_gossip().await;
+
+ tokio::task::spawn_blocking(move || {
+ drop(persister);
+ }).await.unwrap();
+ }
+ clean_test_db().await;
+}
+
+
/// If a channel has only seen updates in one direction, it should not be announced
#[tokio::test]
async fn test_unidirectional_intermediate_update_consideration() {
println!("timestamp: {}", timestamp);
{ // seed the db
- let announcement = generate_announcement(short_channel_id);
+ let announcement = generate_channel_announcement(short_channel_id);
let update_1 = generate_update(short_channel_id, false, timestamp, 0, 0, 0, 6, 0);
let update_2 = generate_update(short_channel_id, true, timestamp + 1, 0, 0, 0, 3, 0);
let update_3 = generate_update(short_channel_id, true, timestamp + 2, 0, 0, 0, 4, 0);
let client_graph_arc = Arc::new(client_graph);
let rgs = RapidGossipSync::new(client_graph_arc.clone(), logger.clone());
- let serialization = serialize_delta(network_graph_arc.clone(), timestamp + 1, logger.clone()).await;
+ let serialization = serialize_delta(network_graph_arc.clone(), timestamp + 1, None, logger.clone()).await;
logger.assert_log_contains("rapid_gossip_sync_server::lookup", "Fetched 1 update rows of the first update in a new direction", 1);
logger.assert_log_contains("rapid_gossip_sync_server::lookup", "Processed 1 reference rows", 1);
let client_channel_count = channels.len();
assert_eq!(client_channel_count, 1);
+ tokio::task::spawn_blocking(move || {
+ drop(persister);
+ }).await.unwrap();
+
clean_test_db().await;
}
println!("timestamp: {}", timestamp);
{ // seed the db
- let announcement = generate_announcement(short_channel_id);
+ let announcement = generate_channel_announcement(short_channel_id);
let update_1 = generate_update(short_channel_id, false, timestamp, 0, 0, 0, 5, 0);
let update_2 = generate_update(short_channel_id, false, timestamp + 1, 0, 0, 0, 4, 0);
let update_3 = generate_update(short_channel_id, false, timestamp + 2, 0, 0, 0, 3, 0);
let channel_count = network_graph_arc.read_only().channels().len();
assert_eq!(channel_count, 1);
- let serialization = serialize_delta(network_graph_arc.clone(), timestamp + 1, logger.clone()).await;
+ let serialization = serialize_delta(network_graph_arc.clone(), timestamp + 1, None, logger.clone()).await;
logger.assert_log_contains("rapid_gossip_sync_server::lookup", "Fetched 0 update rows of the first update in a new direction", 1);
logger.assert_log_contains("rapid_gossip_sync_server::lookup", "Processed 2 reference rows", 1);
assert_eq!(serialization.update_count_full, 0);
assert_eq!(serialization.update_count_incremental, 1);
+ tokio::task::spawn_blocking(move || {
+ drop(persister);
+ }).await.unwrap();
+
+ clean_test_db().await;
+}
+
+#[tokio::test]
+async fn test_channel_reminders() {
+ let _sanitizer = SchemaSanitizer::new();
+
+ let logger = Arc::new(TestLogger::new());
+ let network_graph = NetworkGraph::new(Network::Bitcoin, logger.clone());
+ let network_graph_arc = Arc::new(network_graph);
+ let (mut persister, receiver) = GossipPersister::new(network_graph_arc.clone(), logger.clone());
+
+ let timestamp = current_time();
+ println!("timestamp: {}", timestamp);
+ let channel_reminder_delta = config::CHANNEL_REMINDER_AGE.as_secs() as u32;
+
+ { // seed the db
+ { // unupdated channel
+ let short_channel_id = 1;
+ let announcement = generate_channel_announcement(short_channel_id);
+ let update_1 = generate_update(short_channel_id, false, timestamp - channel_reminder_delta - 1, 0, 0, 0, 5, 0);
+ let update_2 = generate_update(short_channel_id, true, timestamp - channel_reminder_delta - 1, 0, 0, 0, 3, 0);
+
+ network_graph_arc.update_channel_from_announcement_no_lookup(&announcement).unwrap();
+ network_graph_arc.update_channel_unsigned(&update_1.contents).unwrap();
+ network_graph_arc.update_channel_unsigned(&update_2.contents).unwrap();
+
+ receiver.send(GossipMessage::ChannelAnnouncement(announcement, Some(timestamp - channel_reminder_delta - 1))).await.unwrap();
+ receiver.send(GossipMessage::ChannelUpdate(update_1, Some(timestamp - channel_reminder_delta - 1))).await.unwrap();
+ receiver.send(GossipMessage::ChannelUpdate(update_2, Some(timestamp - channel_reminder_delta - 1))).await.unwrap();
+ }
+ { // unmodified but updated channel
+ let short_channel_id = 2;
+ let announcement = generate_channel_announcement(short_channel_id);
+ let update_1 = generate_update(short_channel_id, false, timestamp - channel_reminder_delta - 10, 0, 0, 0, 5, 0);
+ // in the false direction, we have one update that's different prior
+ let update_2 = generate_update(short_channel_id, false, timestamp - channel_reminder_delta - 5, 0, 1, 0, 5, 0);
+ let update_3 = generate_update(short_channel_id, false, timestamp - channel_reminder_delta - 1, 0, 0, 0, 5, 0);
+ let update_4 = generate_update(short_channel_id, true, timestamp - channel_reminder_delta - 1, 0, 0, 0, 3, 0);
+ let update_5 = generate_update(short_channel_id, false, timestamp - channel_reminder_delta + 10, 0, 0, 0, 5, 0);
+ let update_6 = generate_update(short_channel_id, true, timestamp - channel_reminder_delta + 10, 0, 0, 0, 3, 0);
+ let update_7 = generate_update(short_channel_id, false, timestamp - channel_reminder_delta + 20, 0, 0, 0, 5, 0);
+ let update_8 = generate_update(short_channel_id, true, timestamp - channel_reminder_delta + 20, 0, 0, 0, 3, 0);
+
+ network_graph_arc.update_channel_from_announcement_no_lookup(&announcement).unwrap();
+ network_graph_arc.update_channel_unsigned(&update_7.contents).unwrap();
+ network_graph_arc.update_channel_unsigned(&update_8.contents).unwrap();
+
+ receiver.send(GossipMessage::ChannelAnnouncement(announcement, Some(timestamp - channel_reminder_delta - 1))).await.unwrap();
+ receiver.send(GossipMessage::ChannelUpdate(update_1, Some(timestamp - channel_reminder_delta - 10))).await.unwrap();
+ receiver.send(GossipMessage::ChannelUpdate(update_2, Some(timestamp - channel_reminder_delta - 5))).await.unwrap();
+ receiver.send(GossipMessage::ChannelUpdate(update_3, Some(timestamp - channel_reminder_delta - 1))).await.unwrap();
+ receiver.send(GossipMessage::ChannelUpdate(update_4, Some(timestamp - channel_reminder_delta - 1))).await.unwrap();
+
+ receiver.send(GossipMessage::ChannelUpdate(update_5, Some(timestamp - channel_reminder_delta + 10))).await.unwrap();
+ receiver.send(GossipMessage::ChannelUpdate(update_6, Some(timestamp - channel_reminder_delta + 10))).await.unwrap();
+
+ receiver.send(GossipMessage::ChannelUpdate(update_7, Some(timestamp - channel_reminder_delta + 20))).await.unwrap();
+ receiver.send(GossipMessage::ChannelUpdate(update_8, Some(timestamp - channel_reminder_delta + 20))).await.unwrap();
+ }
+ drop(receiver);
+ persister.persist_gossip().await;
+ }
+
+ let channel_count = network_graph_arc.read_only().channels().len();
+ assert_eq!(channel_count, 2);
+
+ let serialization = serialize_delta(network_graph_arc.clone(), timestamp - channel_reminder_delta + 15, None, logger.clone()).await;
+
+ logger.assert_log_contains("rapid_gossip_sync_server::lookup", "Fetched 0 update rows of the first update in a new direction", 1);
+ logger.assert_log_contains("rapid_gossip_sync_server::lookup", "Fetched 4 update rows of the latest update in the less recently updated direction", 1);
+ logger.assert_log_contains("rapid_gossip_sync_server::lookup", "Processed 2 reference rows", 1);
+ logger.assert_log_contains("rapid_gossip_sync_server::lookup", "Processed intermediate rows (2)", 1);
+
+ assert_eq!(serialization.message_count, 4);
+ assert_eq!(serialization.announcement_count, 0);
+ assert_eq!(serialization.update_count, 4);
+ assert_eq!(serialization.update_count_full, 0);
+ assert_eq!(serialization.update_count_incremental, 4);
+
+ tokio::task::spawn_blocking(move || {
+ drop(persister);
+ }).await.unwrap();
+
clean_test_db().await;
}
{ // seed the db
let (mut persister, receiver) = GossipPersister::new(network_graph_arc.clone(), logger.clone());
- let announcement = generate_announcement(short_channel_id);
+ let announcement = generate_channel_announcement(short_channel_id);
network_graph_arc.update_channel_from_announcement_no_lookup(&announcement).unwrap();
receiver.send(GossipMessage::ChannelAnnouncement(announcement, None)).await.unwrap();
drop(receiver);
persister.persist_gossip().await;
+
+ tokio::task::spawn_blocking(move || {
+ drop(persister);
+ }).await.unwrap();
}
let client_graph = NetworkGraph::new(Network::Bitcoin, logger.clone());
let client_graph_arc = Arc::new(client_graph);
{ // sync after initial seed
- let serialization = serialize_delta(network_graph_arc.clone(), 0, logger.clone()).await;
+ let serialization = serialize_delta(network_graph_arc.clone(), 0, None, logger.clone()).await;
logger.assert_log_contains("rapid_gossip_sync_server", "announcement channel count: 1", 1);
let channel_count = network_graph_arc.read_only().channels().len();
{ // seed the db
let (mut persister, receiver) = GossipPersister::new(network_graph_arc.clone(), logger.clone());
- let announcement = generate_announcement(short_channel_id);
+ let announcement = generate_channel_announcement(short_channel_id);
network_graph_arc.update_channel_from_announcement_no_lookup(&announcement).unwrap();
receiver.send(GossipMessage::ChannelAnnouncement(announcement, None)).await.unwrap();
drop(receiver);
persister.persist_gossip().await;
+
+ tokio::task::spawn_blocking(move || {
+ drop(persister);
+ }).await.unwrap();
}
let client_graph = NetworkGraph::new(Network::Bitcoin, logger.clone());
let client_graph_arc = Arc::new(client_graph);
{ // sync after initial seed
- let serialization = serialize_delta(network_graph_arc.clone(), 0, logger.clone()).await;
+ let serialization = serialize_delta(network_graph_arc.clone(), 0, None, logger.clone()).await;
logger.assert_log_contains("rapid_gossip_sync_server", "announcement channel count: 1", 1);
let channel_count = network_graph_arc.read_only().channels().len();
{ // seed the db
let (mut persister, receiver) = GossipPersister::new(network_graph_arc.clone(), logger.clone());
- let announcement = generate_announcement(short_channel_id);
+ let announcement = generate_channel_announcement(short_channel_id);
network_graph_arc.update_channel_from_announcement_no_lookup(&announcement).unwrap();
receiver.send(GossipMessage::ChannelAnnouncement(announcement, None)).await.unwrap();
drop(receiver);
persister.persist_gossip().await;
+
+ tokio::task::spawn_blocking(move || {
+ drop(persister);
+ }).await.unwrap();
}
let client_graph = NetworkGraph::new(Network::Bitcoin, logger.clone());
let client_graph_arc = Arc::new(client_graph);
{ // sync after initial seed
- let serialization = serialize_delta(network_graph_arc.clone(), 0, logger.clone()).await;
+ let serialization = serialize_delta(network_graph_arc.clone(), 0, None, logger.clone()).await;
logger.assert_log_contains("rapid_gossip_sync_server", "announcement channel count: 1", 1);
let channel_count = network_graph_arc.read_only().channels().len();
{ // seed the db
let (mut persister, receiver) = GossipPersister::new(network_graph_arc.clone(), logger.clone());
- let announcement = generate_announcement(short_channel_id);
+ let announcement = generate_channel_announcement(short_channel_id);
network_graph_arc.update_channel_from_announcement_no_lookup(&announcement).unwrap();
receiver.send(GossipMessage::ChannelAnnouncement(announcement, None)).await.unwrap();
drop(receiver);
persister.persist_gossip().await;
+
+ tokio::task::spawn_blocking(move || {
+ drop(persister);
+ }).await.unwrap();
}
let client_graph = NetworkGraph::new(Network::Bitcoin, logger.clone());
let client_graph_arc = Arc::new(client_graph);
{ // sync after initial seed
- let serialization = serialize_delta(network_graph_arc.clone(), 0, logger.clone()).await;
+ let serialization = serialize_delta(network_graph_arc.clone(), 0, None, logger.clone()).await;
logger.assert_log_contains("rapid_gossip_sync_server", "announcement channel count: 1", 1);
let channel_count = network_graph_arc.read_only().channels().len();
let secondary_channel_id = main_channel_id + 1;
{ // main channel
- let announcement = generate_announcement(main_channel_id);
+ let announcement = generate_channel_announcement(main_channel_id);
network_graph_arc.update_channel_from_announcement_no_lookup(&announcement).unwrap();
receiver.send(GossipMessage::ChannelAnnouncement(announcement, None)).await.unwrap();
}
{ // secondary channel
- let announcement = generate_announcement(secondary_channel_id);
+ let announcement = generate_channel_announcement(secondary_channel_id);
network_graph_arc.update_channel_from_announcement_no_lookup(&announcement).unwrap();
receiver.send(GossipMessage::ChannelAnnouncement(announcement, None)).await.unwrap();
}
drop(receiver);
persister.persist_gossip().await;
+
+ tokio::task::spawn_blocking(move || {
+ drop(persister);
+ }).await.unwrap();
}
let client_graph = NetworkGraph::new(Network::Bitcoin, logger.clone());
let client_graph_arc = Arc::new(client_graph);
{ // sync after initial seed
- let serialization = serialize_delta(network_graph_arc.clone(), 0, logger.clone()).await;
+ let serialization = serialize_delta(network_graph_arc.clone(), 0, None, logger.clone()).await;
logger.assert_log_contains("rapid_gossip_sync_server", "announcement channel count: 2", 1);
let channel_count = network_graph_arc.read_only().channels().len();
{ // seed the db
let (mut persister, receiver) = GossipPersister::new(network_graph_arc.clone(), logger.clone());
- let announcement = generate_announcement(short_channel_id);
+ let announcement = generate_channel_announcement(short_channel_id);
network_graph_arc.update_channel_from_announcement_no_lookup(&announcement).unwrap();
receiver.send(GossipMessage::ChannelAnnouncement(announcement, None)).await.unwrap();
drop(receiver);
persister.persist_gossip().await;
+
+ tokio::task::spawn_blocking(move || {
+ drop(persister);
+ }).await.unwrap();
}
let cache_path = cache_sanitizer.cache_path();
drop(receiver);
persister.persist_gossip().await;
+
+ tokio::task::spawn_blocking(move || {
+ drop(persister);
+ }).await.unwrap();
}
// regenerate snapshots
projects / rapid-gossip-sync-server / blobdiff