1 use std::collections::HashMap;
3 use std::os::unix::fs::symlink;
5 use std::time::{Duration, SystemTime, UNIX_EPOCH};
7 use lightning::routing::gossip::NetworkGraph;
9 use crate::{config, TestLogger};
11 pub(crate) struct Snapshotter {
12 network_graph: Arc<NetworkGraph<Arc<TestLogger>>>,
16 pub fn new(network_graph: Arc<NetworkGraph<Arc<TestLogger>>>) -> Self {
17 Self { network_graph }
20 pub(crate) async fn snapshot_gossip(&self) {
21 println!("Initiating snapshotting service");
23 let snapshot_sync_day_factors = [1, 2, 3, 4, 5, 6, 7, 14, 21, u64::MAX];
24 let round_day_seconds = config::SNAPSHOT_CALCULATION_INTERVAL as u64;
26 let pending_snapshot_directory = "./res/snapshots_pending";
27 let pending_symlink_directory = "./res/symlinks_pending";
28 let finalized_snapshot_directory = "./res/snapshots";
29 let finalized_symlink_directory = "./res/symlinks";
30 let relative_symlink_to_snapshot_path = "../snapshots";
32 // this is gonna be a never-ending background job
34 // 1. get the current timestamp
35 let timestamp_seen = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
36 let filename_timestamp = Self::round_down_to_nearest_multiple(timestamp_seen, round_day_seconds);
37 println!("Capturing snapshots at {} for: {}", timestamp_seen, filename_timestamp);
39 // 2. sleep until the next round 24 hours
40 // 3. refresh all snapshots
42 // the stored snapshots should adhere to the following format
48 // from three weeks ago
50 // That means that at any given moment, there should only ever be
51 // 6 (daily) + 3 (weekly) + 1 (total) = 10 cached snapshots
52 // The snapshots, unlike dynamic updates, should account for all intermediate
57 // purge and recreate the pending directories
58 if fs::metadata(&pending_snapshot_directory).is_ok(){
59 fs::remove_dir_all(&pending_snapshot_directory).expect("Failed to remove pending snapshot directory.");
61 if fs::metadata(&pending_symlink_directory).is_ok(){
62 fs::remove_dir_all(&pending_symlink_directory).expect("Failed to remove pending symlink directory.");
64 fs::create_dir_all(&pending_snapshot_directory).expect("Failed to create pending snapshot directory");
65 fs::create_dir_all(&pending_symlink_directory).expect("Failed to create pending symlink directory");
67 let mut snapshot_sync_timestamps: Vec<(u64, u64)> = Vec::new();
68 for factor in &snapshot_sync_day_factors {
69 // basically timestamp - day_seconds * factor
70 let timestamp = timestamp_seen.saturating_sub(round_day_seconds.saturating_mul(factor.clone()));
71 snapshot_sync_timestamps.push((factor.clone(), timestamp));
74 let mut snapshot_filenames_by_day_range: HashMap<u64, String> = HashMap::with_capacity(10);
76 for (day_range, current_last_sync_timestamp) in &snapshot_sync_timestamps {
77 let network_graph_clone = self.network_graph.clone();
79 println!("Calculating {}-day snapshot", day_range);
80 // calculate the snapshot
81 let snapshot = super::serialize_delta(network_graph_clone, current_last_sync_timestamp.clone() as u32, true).await;
83 // persist the snapshot and update the symlink
84 let snapshot_filename = format!("snapshot__calculated-at:{}__range:{}-days__previous-sync:{}.lngossip", filename_timestamp, day_range, current_last_sync_timestamp);
85 let snapshot_path = format!("{}/{}", pending_snapshot_directory, snapshot_filename);
86 println!("Persisting {}-day snapshot: {} ({} messages, {} announcements, {} updates ({} full, {} incremental))", day_range, snapshot_filename, snapshot.message_count, snapshot.announcement_count, snapshot.update_count, snapshot.update_count_full, snapshot.update_count_incremental);
87 fs::write(&snapshot_path, snapshot.data).unwrap();
88 snapshot_filenames_by_day_range.insert(day_range.clone(), snapshot_filename);
92 for i in 0..10_001u64 {
93 // let's create symlinks
95 // first, determine which snapshot range should be referenced
96 let referenced_day_range = if i == 0 {
97 // special-case 0 to always refer to a full/initial sync
100 // find min(x) in snapshot_sync_day_factors where x >= i
101 snapshot_sync_day_factors.iter().find(|x| {
106 let snapshot_filename = snapshot_filenames_by_day_range.get(&referenced_day_range).unwrap();
107 let relative_snapshot_path = format!("{}/{}", relative_symlink_to_snapshot_path, snapshot_filename);
109 let canonical_last_sync_timestamp = if i == 0 {
110 // special-case 0 to always refer to a full/initial sync
113 let simulated_last_sync_timestamp = timestamp_seen.saturating_sub(round_day_seconds.saturating_mul(i));
114 Self::round_down_to_nearest_multiple(simulated_last_sync_timestamp, round_day_seconds)
116 let symlink_path = format!("{}/{}.bin", pending_symlink_directory, canonical_last_sync_timestamp);
118 println!("Symlinking: {} -> {} ({} -> {}", i, referenced_day_range, symlink_path, relative_snapshot_path);
119 symlink(&relative_snapshot_path, &symlink_path).unwrap();
122 if fs::metadata(&finalized_snapshot_directory).is_ok(){
123 fs::remove_dir_all(&finalized_snapshot_directory).expect("Failed to remove finalized snapshot directory.");
125 if fs::metadata(&finalized_symlink_directory).is_ok(){
126 fs::remove_dir_all(&finalized_symlink_directory).expect("Failed to remove pending symlink directory.");
128 fs::rename(&pending_snapshot_directory, &finalized_snapshot_directory).expect("Failed to finalize snapshot directory.");
129 fs::rename(&pending_symlink_directory, &finalized_symlink_directory).expect("Failed to finalize symlink directory.");
132 let remainder = timestamp_seen % round_day_seconds;
133 let time_until_next_day = round_day_seconds - remainder;
135 println!("Sleeping until next snapshot capture: {}s", time_until_next_day);
136 // add in an extra five seconds to assure the rounding down works correctly
137 let sleep = tokio::time::sleep(Duration::from_secs(time_until_next_day + 5));
142 fn round_down_to_nearest_multiple(number: u64, multiple: u64) -> u64 {
143 let round_multiple_delta = number % multiple;
144 number - round_multiple_delta