1 // Prefix these with `rustdoc::` when we update our MSRV to be >= 1.52 to remove warnings.
2 #![deny(broken_intra_doc_links)]
3 #![deny(private_intra_doc_links)]
7 #![deny(non_upper_case_globals)]
8 #![deny(non_camel_case_types)]
9 #![deny(non_snake_case)]
11 #![deny(unused_variables)]
12 #![deny(unused_imports)]
13 //! This crate exposes functionality to rapidly sync gossip data, aimed primarily at mobile
16 //! The server sends a compressed response containing differential gossip data. The gossip data is
17 //! formatted compactly, omitting signatures and opportunistically incremental where previous
18 //! channel updates are known (a mechanism that is enabled when the timestamp of the last known
19 //! channel update is communicated). A reference server implementation can be found
20 //! [here](https://github.com/lightningdevkit/rapid-gossip-sync-server).
22 //! An example server request could look as simple as the following. Note that the first ever rapid
23 //! sync should use `0` for `last_sync_timestamp`:
26 //! curl -o rapid_sync.lngossip https://rapidsync.lightningdevkit.org/snapshot/<last_sync_timestamp>
29 //! Then, call the network processing function. In this example, we process the update by reading
30 //! its contents from disk, which we do by calling the `sync_network_graph_with_file_path` method:
33 //! use bitcoin::blockdata::constants::genesis_block;
34 //! use bitcoin::Network;
35 //! use lightning::routing::gossip::NetworkGraph;
36 //! use lightning_rapid_gossip_sync::RapidGossipSync;
38 //! # use lightning::util::logger::{Logger, Record};
39 //! # struct FakeLogger {}
40 //! # impl Logger for FakeLogger {
41 //! # fn log(&self, record: &Record) { unimplemented!() }
43 //! # let logger = FakeLogger {};
45 //! let block_hash = genesis_block(Network::Bitcoin).header.block_hash();
46 //! let network_graph = NetworkGraph::new(block_hash, &logger);
47 //! let rapid_sync = RapidGossipSync::new(&network_graph);
48 //! let new_last_sync_timestamp_result = rapid_sync.sync_network_graph_with_file_path("./rapid_sync.lngossip");
51 //! The primary benefit this syncing mechanism provides is that given a trusted server, a
52 //! low-powered client can offload the validation of gossip signatures. This enables a client to
53 //! privately calculate routes for payments, and do so much faster and earlier than requiring a full
54 //! peer-to-peer gossip sync to complete.
56 //! The reason the rapid sync server requires trust is that it could provide bogus data, though at
57 //! worst, all that would result in is a fake network topology, which wouldn't enable the server to
58 //! steal or siphon off funds. It could, however, reduce the client's privacy by forcing all
59 //! payments to be routed via channels the server controls.
61 //! The way a server is meant to calculate this rapid gossip sync data is by using a `latest_seen`
62 //! timestamp provided by the client. It's not included in either channel announcement or update,
63 //! (not least due to announcements not including any timestamps at all, but only a block height)
64 //! but rather, it's a timestamp of when the server saw a particular message.
66 // Allow and import test features for benching
67 #![cfg_attr(all(test, feature = "_bench_unstable"), feature(test))]
68 #[cfg(all(test, feature = "_bench_unstable"))]
73 use std::sync::atomic::{AtomicBool, Ordering};
75 use lightning::routing::gossip::NetworkGraph;
76 use lightning::util::logger::Logger;
78 use crate::error::GraphSyncError;
80 /// Error types that these functions can return
83 /// Core functionality of this crate
86 /// Rapid Gossip Sync struct
87 /// See [crate-level documentation] for usage.
89 /// [crate-level documentation]: crate
90 pub struct RapidGossipSync<NG: Deref<Target=NetworkGraph<L>>, L: Deref>
91 where L::Target: Logger {
93 is_initial_sync_complete: AtomicBool
96 impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L::Target: Logger {
97 /// Instantiate a new [`RapidGossipSync`] instance
98 pub fn new(network_graph: NG) -> Self {
101 is_initial_sync_complete: AtomicBool::new(false)
105 /// Sync gossip data from a file
106 /// Returns the last sync timestamp to be used the next time rapid sync data is queried.
108 /// `network_graph`: The network graph to apply the updates to
110 /// `sync_path`: Path to the file where the gossip update data is located
112 pub fn sync_network_graph_with_file_path(
115 ) -> Result<u32, GraphSyncError> {
116 let mut file = File::open(sync_path)?;
117 self.update_network_graph_from_byte_stream(&mut file)
120 /// Gets a reference to the underlying [`NetworkGraph`] which was provided in
121 /// [`RapidGossipSync::new`].
123 /// (C-not exported) as bindings don't support a reference-to-a-reference yet
124 pub fn network_graph(&self) -> &NG {
128 /// Returns whether a rapid gossip sync has completed at least once
129 pub fn is_initial_sync_complete(&self) -> bool {
130 self.is_initial_sync_complete.load(Ordering::Acquire)
138 use bitcoin::blockdata::constants::genesis_block;
139 use bitcoin::Network;
141 use lightning::ln::msgs::DecodeError;
142 use lightning::routing::gossip::NetworkGraph;
143 use lightning::util::test_utils::TestLogger;
144 use crate::RapidGossipSync;
147 fn test_sync_from_file() {
148 struct FileSyncTest {
153 fn new(tmp_directory: &str, valid_response: &[u8]) -> FileSyncTest {
154 let test = FileSyncTest { directory: tmp_directory.to_owned() };
156 let graph_sync_test_directory = test.get_test_directory();
157 fs::create_dir_all(graph_sync_test_directory).unwrap();
159 let graph_sync_test_file = test.get_test_file_path();
160 fs::write(&graph_sync_test_file, valid_response).unwrap();
164 fn get_test_directory(&self) -> String {
165 let graph_sync_test_directory = self.directory.clone() + "/graph-sync-tests";
166 graph_sync_test_directory
168 fn get_test_file_path(&self) -> String {
169 let graph_sync_test_directory = self.get_test_directory();
170 let graph_sync_test_file = graph_sync_test_directory.to_owned() + "/test_data.lngossip";
175 impl Drop for FileSyncTest {
177 fs::remove_dir_all(self.directory.clone()).unwrap();
181 // same as incremental_only_update_fails_without_prior_same_direction_updates
182 let valid_response = vec![
183 76, 68, 75, 1, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
184 79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0, 0, 0, 0, 97, 227, 98, 218,
185 0, 0, 0, 4, 2, 22, 7, 207, 206, 25, 164, 197, 231, 230, 231, 56, 102, 61, 250, 251,
186 187, 172, 38, 46, 79, 247, 108, 44, 155, 48, 219, 238, 252, 53, 192, 6, 67, 2, 36, 125,
187 157, 176, 223, 175, 234, 116, 94, 248, 201, 225, 97, 235, 50, 47, 115, 172, 63, 136,
188 88, 216, 115, 11, 111, 217, 114, 84, 116, 124, 231, 107, 2, 158, 1, 242, 121, 152, 106,
189 204, 131, 186, 35, 93, 70, 216, 10, 237, 224, 183, 89, 95, 65, 3, 83, 185, 58, 138,
190 181, 64, 187, 103, 127, 68, 50, 2, 201, 19, 17, 138, 136, 149, 185, 226, 156, 137, 175,
191 110, 32, 237, 0, 217, 90, 31, 100, 228, 149, 46, 219, 175, 168, 77, 4, 143, 38, 128,
192 76, 97, 0, 0, 0, 2, 0, 0, 255, 8, 153, 192, 0, 2, 27, 0, 0, 0, 1, 0, 0, 255, 2, 68,
193 226, 0, 6, 11, 0, 1, 2, 3, 0, 0, 0, 2, 0, 40, 0, 0, 0, 0, 0, 0, 3, 232, 0, 0, 3, 232,
194 0, 0, 0, 1, 0, 0, 0, 0, 58, 85, 116, 216, 255, 8, 153, 192, 0, 2, 27, 0, 0, 25, 0, 0,
195 0, 1, 0, 0, 0, 125, 255, 2, 68, 226, 0, 6, 11, 0, 1, 5, 0, 0, 0, 0, 29, 129, 25, 192,
198 let tmp_directory = "./rapid-gossip-sync-tests-tmp";
199 let sync_test = FileSyncTest::new(tmp_directory, &valid_response);
200 let graph_sync_test_file = sync_test.get_test_file_path();
202 let block_hash = genesis_block(Network::Bitcoin).block_hash();
203 let logger = TestLogger::new();
204 let network_graph = NetworkGraph::new(block_hash, &logger);
206 assert_eq!(network_graph.read_only().channels().len(), 0);
208 let rapid_sync = RapidGossipSync::new(&network_graph);
209 let sync_result = rapid_sync.sync_network_graph_with_file_path(&graph_sync_test_file);
211 if sync_result.is_err() {
212 panic!("Unexpected sync result: {:?}", sync_result)
215 assert_eq!(network_graph.read_only().channels().len(), 2);
216 let after = network_graph.to_string();
218 after.contains("021607cfce19a4c5e7e6e738663dfafbbbac262e4ff76c2c9b30dbeefc35c00643")
221 after.contains("02247d9db0dfafea745ef8c9e161eb322f73ac3f8858d8730b6fd97254747ce76b")
224 after.contains("029e01f279986acc83ba235d46d80aede0b7595f410353b93a8ab540bb677f4432")
227 after.contains("02c913118a8895b9e29c89af6e20ed00d95a1f64e4952edbafa84d048f26804c61")
229 assert!(after.contains("619737530008010752"));
230 assert!(after.contains("783241506229452801"));
234 fn measure_native_read_from_file() {
235 let block_hash = genesis_block(Network::Bitcoin).block_hash();
236 let logger = TestLogger::new();
237 let network_graph = NetworkGraph::new(block_hash, &logger);
239 assert_eq!(network_graph.read_only().channels().len(), 0);
241 let rapid_sync = RapidGossipSync::new(&network_graph);
242 let start = std::time::Instant::now();
243 let sync_result = rapid_sync
244 .sync_network_graph_with_file_path("./res/full_graph.lngossip");
245 if let Err(crate::error::GraphSyncError::DecodeError(DecodeError::Io(io_error))) = &sync_result {
246 let error_string = format!("Input file lightning-rapid-gossip-sync/res/full_graph.lngossip is missing! Download it from https://bitcoin.ninja/ldk-compressed_graph-285cb27df79-2022-07-21.bin\n\n{:?}", io_error);
247 #[cfg(not(require_route_graph_test))]
249 println!("{}", error_string);
252 #[cfg(require_route_graph_test)]
253 panic!("{}", error_string);
255 let elapsed = start.elapsed();
256 println!("initialization duration: {:?}", elapsed);
257 if sync_result.is_err() {
258 panic!("Unexpected sync result: {:?}", sync_result)
263 #[cfg(all(test, feature = "_bench_unstable"))]
267 use bitcoin::blockdata::constants::genesis_block;
268 use bitcoin::Network;
270 use lightning::ln::msgs::DecodeError;
271 use lightning::routing::gossip::NetworkGraph;
272 use lightning::util::test_utils::TestLogger;
274 use crate::RapidGossipSync;
277 fn bench_reading_full_graph_from_file(b: &mut Bencher) {
278 let block_hash = genesis_block(Network::Bitcoin).block_hash();
279 let logger = TestLogger::new();
281 let network_graph = NetworkGraph::new(block_hash, &logger);
282 let rapid_sync = RapidGossipSync::new(&network_graph);
283 let sync_result = rapid_sync.sync_network_graph_with_file_path("./res/full_graph.lngossip");
284 if let Err(crate::error::GraphSyncError::DecodeError(DecodeError::Io(io_error))) = &sync_result {
285 let error_string = format!("Input file lightning-rapid-gossip-sync/res/full_graph.lngossip is missing! Download it from https://bitcoin.ninja/ldk-compressed_graph-bc08df7542-2022-05-05.bin\n\n{:?}", io_error);
286 #[cfg(not(require_route_graph_test))]
288 println!("{}", error_string);
291 #[cfg(require_route_graph_test)]
292 panic!("{}", error_string);
294 assert!(sync_result.is_ok())