_ Git - rust-lightning/blob - lightning-rapid-gossip-sync/src/lib.rs

   1 // Prefix these with `rustdoc::` when we update our MSRV to be >= 1.52 to remove warnings.
   2 #![deny(broken_intra_doc_links)]
   3 #![deny(private_intra_doc_links)]
   4
   5 #![deny(missing_docs)]
   6 #![deny(unsafe_code)]
   7 #![deny(non_upper_case_globals)]
   8 #![deny(non_camel_case_types)]
   9 #![deny(non_snake_case)]
  10 #![deny(unused_mut)]
  11 #![deny(unused_variables)]
  12 #![deny(unused_imports)]
  13 //! This crate exposes client functionality to rapidly sync gossip data, aimed primarily at mobile
  14 //! devices.
  15 //!
  16 //! The rapid gossip sync server will provide a compressed response containing differential gossip
  17 //! data. The gossip data is formatted compactly, omitting signatures and opportunistically
  18 //! incremental where previous channel updates are known. This mechanism is enabled when the
  19 //! timestamp of the last known channel update is communicated. A reference server implementation
  20 //! can be found [on Github](https://github.com/lightningdevkit/rapid-gossip-sync-server).
  21 //!
  22 //! The primary benefit of this syncing mechanism is that it allows a low-powered client to offload
  23 //! the validation of gossip signatures to a semi-trusted server. This enables the client to
  24 //! privately calculate routes for payments, and to do so much faster than requiring a full
  25 //! peer-to-peer gossip sync to complete.
  26 //!
  27 //! The server calculates its response on the basis of a client-provided `latest_seen` timestamp,
  28 //! i.e., the server will return all rapid gossip sync data it has seen after the given timestamp.
  29 //!
  30 //! # Getting Started
  31 //! Firstly, the data needs to be retrieved from the server. For example, you could use the server
  32 //! at <https://rapidsync.lightningdevkit.org> with the following request format:
  33 //!
  34 //! ```shell
  35 //! curl -o rapid_sync.lngossip https://rapidsync.lightningdevkit.org/snapshot/<last_sync_timestamp>
  36 //! ```
  37 //! Note that the first ever rapid sync should use `0` for `last_sync_timestamp`.
  38 //!
  39 //! After the gossip data snapshot has been downloaded, one of the client's graph processing
  40 //! functions needs to be called. In this example, we process the update by reading its contents
  41 //! from disk, which we do by calling [sync_network_graph_with_file_path]:
  42 //!
  43 //! ```
  44 //! use bitcoin::blockdata::constants::genesis_block;
  45 //! use bitcoin::Network;
  46 //! use lightning::routing::gossip::NetworkGraph;
  47 //! use lightning_rapid_gossip_sync::RapidGossipSync;
  48 //!
  49 //! # use lightning::util::logger::{Logger, Record};
  50 //! # struct FakeLogger {}
  51 //! # impl Logger for FakeLogger {
  52 //! #     fn log(&self, record: &Record) { unimplemented!() }
  53 //! # }
  54 //! # let logger = FakeLogger {};
  55 //!
  56 //! let block_hash = genesis_block(Network::Bitcoin).header.block_hash();
  57 //! let network_graph = NetworkGraph::new(block_hash, &logger);
  58 //! let rapid_sync = RapidGossipSync::new(&network_graph);
  59 //! let new_last_sync_timestamp_result = rapid_sync.sync_network_graph_with_file_path("./rapid_sync.lngossip");
  60 //! ```
  61 //! [sync_network_graph_with_file_path]: RapidGossipSync::sync_network_graph_with_file_path
  62
  63 // Allow and import test features for benching
  64 #![cfg_attr(all(test, feature = "_bench_unstable"), feature(test))]
  65 #[cfg(all(test, feature = "_bench_unstable"))]
  66 extern crate test;
  67
  68 #[cfg(feature = "std")]
  69 use std::fs::File;
  70 use core::ops::Deref;
  71 use core::sync::atomic::{AtomicBool, Ordering};
  72
  73 use lightning::io;
  74 use lightning::routing::gossip::NetworkGraph;
  75 use lightning::util::logger::Logger;
  76
  77 pub use crate::error::GraphSyncError;
  78
  79 /// Error types that these functions can return
  80 mod error;
  81
  82 /// Core functionality of this crate
  83 mod processing;
  84
  85 /// The main Rapid Gossip Sync object.
  86 ///
  87 /// See [crate-level documentation] for usage.
  88 ///
  89 /// [crate-level documentation]: crate
  90 pub struct RapidGossipSync<NG: Deref<Target=NetworkGraph<L>>, L: Deref>
  91 where L::Target: Logger {
  92         network_graph: NG,
  93         is_initial_sync_complete: AtomicBool
  94 }
  95
  96 impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L::Target: Logger {
  97         /// Instantiate a new [`RapidGossipSync`] instance.
  98         pub fn new(network_graph: NG) -> Self {
  99                 Self {
 100                         network_graph,
 101                         is_initial_sync_complete: AtomicBool::new(false)
 102                 }
 103         }
 104
 105         /// Sync gossip data from a file.
 106         /// Returns the last sync timestamp to be used the next time rapid sync data is queried.
 107         ///
 108         /// `network_graph`: The network graph to apply the updates to
 109         ///
 110         /// `sync_path`: Path to the file where the gossip update data is located
 111         ///
 112         #[cfg(feature = "std")]
 113         pub fn sync_network_graph_with_file_path(
 114                 &self,
 115                 sync_path: &str,
 116         ) -> Result<u32, GraphSyncError> {
 117                 let mut file = File::open(sync_path)?;
 118                 self.update_network_graph_from_byte_stream(&mut file)
 119         }
 120
 121         /// Update network graph from binary data.
 122         /// Returns the last sync timestamp to be used the next time rapid sync data is queried.
 123         ///
 124         /// `network_graph`: network graph to be updated
 125         ///
 126         /// `update_data`: `&[u8]` binary stream that comprises the update data
 127         pub fn update_network_graph(&self, update_data: &[u8]) -> Result<u32, GraphSyncError> {
 128                 let mut read_cursor = io::Cursor::new(update_data);
 129                 self.update_network_graph_from_byte_stream(&mut read_cursor)
 130         }
 131
 132         /// Gets a reference to the underlying [`NetworkGraph`] which was provided in
 133         /// [`RapidGossipSync::new`].
 134         ///
 135         /// (C-not exported) as bindings don't support a reference-to-a-reference yet
 136         pub fn network_graph(&self) -> &NG {
 137                 &self.network_graph
 138         }
 139
 140         /// Returns whether a rapid gossip sync has completed at least once.
 141         pub fn is_initial_sync_complete(&self) -> bool {
 142                 self.is_initial_sync_complete.load(Ordering::Acquire)
 143         }
 144 }
 145
 146 #[cfg(test)]
 147 mod tests {
 148         use std::fs;
 149
 150         use bitcoin::blockdata::constants::genesis_block;
 151         use bitcoin::Network;
 152
 153         use lightning::ln::msgs::DecodeError;
 154         use lightning::routing::gossip::NetworkGraph;
 155         use lightning::util::test_utils::TestLogger;
 156         use crate::RapidGossipSync;
 157
 158         #[test]
 159         fn test_sync_from_file() {
 160                 struct FileSyncTest {
 161                         directory: String,
 162                 }
 163
 164                 impl FileSyncTest {
 165                         fn new(tmp_directory: &str, valid_response: &[u8]) -> FileSyncTest {
 166                                 let test = FileSyncTest { directory: tmp_directory.to_owned() };
 167
 168                                 let graph_sync_test_directory = test.get_test_directory();
 169                                 fs::create_dir_all(graph_sync_test_directory).unwrap();
 170
 171                                 let graph_sync_test_file = test.get_test_file_path();
 172                                 fs::write(&graph_sync_test_file, valid_response).unwrap();
 173
 174                                 test
 175                         }
 176                         fn get_test_directory(&self) -> String {
 177                                 let graph_sync_test_directory = self.directory.clone() + "/graph-sync-tests";
 178                                 graph_sync_test_directory
 179                         }
 180                         fn get_test_file_path(&self) -> String {
 181                                 let graph_sync_test_directory = self.get_test_directory();
 182                                 let graph_sync_test_file = graph_sync_test_directory.to_owned() + "/test_data.lngossip";
 183                                 graph_sync_test_file
 184                         }
 185                 }
 186
 187                 impl Drop for FileSyncTest {
 188                         fn drop(&mut self) {
 189                                 fs::remove_dir_all(self.directory.clone()).unwrap();
 190                         }
 191                 }
 192
 193                 // same as incremental_only_update_fails_without_prior_same_direction_updates
 194                 let valid_response = vec![
 195                         76, 68, 75, 1, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
 196                         79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0, 0, 0, 0, 97, 227, 98, 218,
 197                         0, 0, 0, 4, 2, 22, 7, 207, 206, 25, 164, 197, 231, 230, 231, 56, 102, 61, 250, 251,
 198                         187, 172, 38, 46, 79, 247, 108, 44, 155, 48, 219, 238, 252, 53, 192, 6, 67, 2, 36, 125,
 199                         157, 176, 223, 175, 234, 116, 94, 248, 201, 225, 97, 235, 50, 47, 115, 172, 63, 136,
 200                         88, 216, 115, 11, 111, 217, 114, 84, 116, 124, 231, 107, 2, 158, 1, 242, 121, 152, 106,
 201                         204, 131, 186, 35, 93, 70, 216, 10, 237, 224, 183, 89, 95, 65, 3, 83, 185, 58, 138,
 202                         181, 64, 187, 103, 127, 68, 50, 2, 201, 19, 17, 138, 136, 149, 185, 226, 156, 137, 175,
 203                         110, 32, 237, 0, 217, 90, 31, 100, 228, 149, 46, 219, 175, 168, 77, 4, 143, 38, 128,
 204                         76, 97, 0, 0, 0, 2, 0, 0, 255, 8, 153, 192, 0, 2, 27, 0, 0, 0, 1, 0, 0, 255, 2, 68,
 205                         226, 0, 6, 11, 0, 1, 2, 3, 0, 0, 0, 2, 0, 40, 0, 0, 0, 0, 0, 0, 3, 232, 0, 0, 3, 232,
 206                         0, 0, 0, 1, 0, 0, 0, 0, 58, 85, 116, 216, 255, 8, 153, 192, 0, 2, 27, 0, 0, 25, 0, 0,
 207                         0, 1, 0, 0, 0, 125, 255, 2, 68, 226, 0, 6, 11, 0, 1, 5, 0, 0, 0, 0, 29, 129, 25, 192,
 208                 ];
 209
 210                 let tmp_directory = "./rapid-gossip-sync-tests-tmp";
 211                 let sync_test = FileSyncTest::new(tmp_directory, &valid_response);
 212                 let graph_sync_test_file = sync_test.get_test_file_path();
 213
 214                 let block_hash = genesis_block(Network::Bitcoin).block_hash();
 215                 let logger = TestLogger::new();
 216                 let network_graph = NetworkGraph::new(block_hash, &logger);
 217
 218                 assert_eq!(network_graph.read_only().channels().len(), 0);
 219
 220                 let rapid_sync = RapidGossipSync::new(&network_graph);
 221                 let sync_result = rapid_sync.sync_network_graph_with_file_path(&graph_sync_test_file);
 222
 223                 if sync_result.is_err() {
 224                         panic!("Unexpected sync result: {:?}", sync_result)
 225                 }
 226
 227                 assert_eq!(network_graph.read_only().channels().len(), 2);
 228                 let after = network_graph.to_string();
 229                 assert!(
 230                         after.contains("021607cfce19a4c5e7e6e738663dfafbbbac262e4ff76c2c9b30dbeefc35c00643")
 231                 );
 232                 assert!(
 233                         after.contains("02247d9db0dfafea745ef8c9e161eb322f73ac3f8858d8730b6fd97254747ce76b")
 234                 );
 235                 assert!(
 236                         after.contains("029e01f279986acc83ba235d46d80aede0b7595f410353b93a8ab540bb677f4432")
 237                 );
 238                 assert!(
 239                         after.contains("02c913118a8895b9e29c89af6e20ed00d95a1f64e4952edbafa84d048f26804c61")
 240                 );
 241                 assert!(after.contains("619737530008010752"));
 242                 assert!(after.contains("783241506229452801"));
 243         }
 244
 245         #[test]
 246         fn measure_native_read_from_file() {
 247                 let block_hash = genesis_block(Network::Bitcoin).block_hash();
 248                 let logger = TestLogger::new();
 249                 let network_graph = NetworkGraph::new(block_hash, &logger);
 250
 251                 assert_eq!(network_graph.read_only().channels().len(), 0);
 252
 253                 let rapid_sync = RapidGossipSync::new(&network_graph);
 254                 let start = std::time::Instant::now();
 255                 let sync_result = rapid_sync
 256                         .sync_network_graph_with_file_path("./res/full_graph.lngossip");
 257                 if let Err(crate::error::GraphSyncError::DecodeError(DecodeError::Io(io_error))) = &sync_result {
 258                         let error_string = format!("Input file lightning-rapid-gossip-sync/res/full_graph.lngossip is missing! Download it from https://bitcoin.ninja/ldk-compressed_graph-285cb27df79-2022-07-21.bin\n\n{:?}", io_error);
 259                         #[cfg(not(require_route_graph_test))]
 260                         {
 261                                 println!("{}", error_string);
 262                                 return;
 263                         }
 264                         #[cfg(require_route_graph_test)]
 265                         panic!("{}", error_string);
 266                 }
 267                 let elapsed = start.elapsed();
 268                 println!("initialization duration: {:?}", elapsed);
 269                 if sync_result.is_err() {
 270                         panic!("Unexpected sync result: {:?}", sync_result)
 271                 }
 272         }
 273 }
 274
 275 #[cfg(all(test, feature = "_bench_unstable"))]
 276 pub mod bench {
 277         use test::Bencher;
 278
 279         use bitcoin::blockdata::constants::genesis_block;
 280         use bitcoin::Network;
 281
 282         use lightning::ln::msgs::DecodeError;
 283         use lightning::routing::gossip::NetworkGraph;
 284         use lightning::util::test_utils::TestLogger;
 285
 286         use crate::RapidGossipSync;
 287
 288         #[bench]
 289         fn bench_reading_full_graph_from_file(b: &mut Bencher) {
 290                 let block_hash = genesis_block(Network::Bitcoin).block_hash();
 291                 let logger = TestLogger::new();
 292                 b.iter(|| {
 293                         let network_graph = NetworkGraph::new(block_hash, &logger);
 294                         let rapid_sync = RapidGossipSync::new(&network_graph);
 295                         let sync_result = rapid_sync.sync_network_graph_with_file_path("./res/full_graph.lngossip");
 296                         if let Err(crate::error::GraphSyncError::DecodeError(DecodeError::Io(io_error))) = &sync_result {
 297                                 let error_string = format!("Input file lightning-rapid-gossip-sync/res/full_graph.lngossip is missing! Download it from https://bitcoin.ninja/ldk-compressed_graph-bc08df7542-2022-05-05.bin\n\n{:?}", io_error);
 298                                 #[cfg(not(require_route_graph_test))]
 299                                 {
 300                                         println!("{}", error_string);
 301                                         return;
 302                                 }
 303                                 #[cfg(require_route_graph_test)]
 304                                 panic!("{}", error_string);
 305                         }
 306                         assert!(sync_result.is_ok())
 307                 });
 308         }
 309 }