Dynamically adjust snapshot scopes.

Previously, we had hard-coded factors for the default snapshot
generation interval, which also served as the minimum snapshot
scope. In this commit, we substitute that with a doubling
mechanism that stops once it reaches or exceeds the
21-day-mark, which can be configured using an additional flag.

diff --git a/src/config.rs b/src/config.rs
index b1c7ba283e9f73fe7ac732cd76680891fb20a14d..b905c53f3b877f27b12e21d8fd816f9864a6a1a0 100644 (file)
--- a/src/config.rs
+++ b/src/config.rs
@@ -16,7 +16,8 @@ use lightning_block_sync::http::HttpEndpoint;
 use tokio_postgres::Config;
 
 pub(crate) const SCHEMA_VERSION: i32 = 12;
-pub(crate) const SYMLINK_GRANULARITY_INTERVAL: u32 = 3600 * 3;
+pub(crate) const SYMLINK_GRANULARITY_INTERVAL: u32 = 3600 * 3; // three hours
+pub(crate) const MAX_SNAPSHOT_SCOPE: u32 = 3600 * 24 * 21; // three weeks
 // generate symlinks based on a 3-hour-granularity
 /// If the last update in either direction was more than six days ago, we send a reminder
 /// That reminder may be either in the form of a channel announcement, or in the form of empty

diff --git a/src/snapshot.rs b/src/snapshot.rs
index dd3bb7604ed04ee2ba371a9537d8feb47c3c6929..218285652868b56dac11dacedf3cf3c843d10510 100644 (file)
--- a/src/snapshot.rs
+++ b/src/snapshot.rs
@@ -14,7 +14,7 @@ use crate::config::cache_path;
 
 pub(crate) struct Snapshotter<L: Deref + Clone> where L::Target: Logger {
        network_graph: Arc<NetworkGraph<L>>,
-       logger: L
+       logger: L,
 }
 
 impl<L: Deref + Clone> Snapshotter<L> where L::Target: Logger {
@@ -26,7 +26,20 @@ impl<L: Deref + Clone> Snapshotter<L> where L::Target: Logger {
                log_info!(self.logger, "Initiating snapshotting service");
 
                let snapshot_interval = config::snapshot_generation_interval() as u64;
-               let snapshot_sync_factors = [1, 2, 3, 4, 5, 6, 7, 14, 21, u64::MAX];
+               let mut snapshot_scopes = vec![];
+               { // double the coefficient until it reaches the maximum (limited) snapshot scope
+                       let mut current_scope = snapshot_interval;
+                       loop {
+                               snapshot_scopes.push(current_scope);
+                               if current_scope >= config::MAX_SNAPSHOT_SCOPE as u64 {
+                                       snapshot_scopes.push(u64::MAX);
+                                       break;
+                               }
+
+                               // double the current factor
+                               current_scope <<= 1;
+                       }
+               }
 
                let pending_snapshot_directory = format!("{}/snapshots_pending", cache_path());
                let pending_symlink_directory = format!("{}/symlinks_pending", cache_path());
@@ -58,37 +71,36 @@ impl<L: Deref + Clone> Snapshotter<L> where L::Target: Logger {
                        // channel updates
 
                        // purge and recreate the pending directories
-                       if fs::metadata(&pending_snapshot_directory).is_ok(){
+                       if fs::metadata(&pending_snapshot_directory).is_ok() {
                                fs::remove_dir_all(&pending_snapshot_directory).expect("Failed to remove pending snapshot directory.");
                        }
-                       if fs::metadata(&pending_symlink_directory).is_ok(){
+                       if fs::metadata(&pending_symlink_directory).is_ok() {
                                fs::remove_dir_all(&pending_symlink_directory).expect("Failed to remove pending symlink directory.");
                        }
                        fs::create_dir_all(&pending_snapshot_directory).expect("Failed to create pending snapshot directory");
                        fs::create_dir_all(&pending_symlink_directory).expect("Failed to create pending symlink directory");
 
                        let mut snapshot_sync_timestamps: Vec<(u64, u64)> = Vec::new();
-                       for factor in &snapshot_sync_factors {
-                               // basically timestamp - day_seconds * factor
-                               let timestamp = reference_timestamp.saturating_sub(snapshot_interval.saturating_mul(factor.clone()));
-                               snapshot_sync_timestamps.push((factor.clone(), timestamp));
+                       for current_scope in &snapshot_scopes {
+                               let timestamp = reference_timestamp.saturating_sub(current_scope.clone());
+                               snapshot_sync_timestamps.push((current_scope.clone(), timestamp));
                        };
 
-                       let mut snapshot_filenames_by_day_range: HashMap<u64, String> = HashMap::with_capacity(10);
+                       let mut snapshot_filenames_by_scope: HashMap<u64, String> = HashMap::with_capacity(10);
 
-                       for (day_range, current_last_sync_timestamp) in &snapshot_sync_timestamps {
+                       for (current_scope, current_last_sync_timestamp) in &snapshot_sync_timestamps {
                                let network_graph_clone = self.network_graph.clone();
                                {
-                                       log_info!(self.logger, "Calculating {}-day snapshot", day_range);
+                                       log_info!(self.logger, "Calculating {}-second snapshot", current_scope);
                                        // calculate the snapshot
                                        let snapshot = super::serialize_delta(network_graph_clone, current_last_sync_timestamp.clone() as u32, self.logger.clone()).await;
 
                                        // persist the snapshot and update the symlink
-                                       let snapshot_filename = format!("snapshot__calculated-at:{}__range:{}-days__previous-sync:{}.lngossip", reference_timestamp, day_range, current_last_sync_timestamp);
+                                       let snapshot_filename = format!("snapshot__calculated-at:{}__range:{}-scope__previous-sync:{}.lngossip", reference_timestamp, current_scope, current_last_sync_timestamp);
                                        let snapshot_path = format!("{}/{}", pending_snapshot_directory, snapshot_filename);
-                                       log_info!(self.logger, "Persisting {}-day snapshot: {} ({} messages, {} announcements, {} updates ({} full, {} incremental))", day_range, snapshot_filename, snapshot.message_count, snapshot.announcement_count, snapshot.update_count, snapshot.update_count_full, snapshot.update_count_incremental);
+                                       log_info!(self.logger, "Persisting {}-second snapshot: {} ({} messages, {} announcements, {} updates ({} full, {} incremental))", current_scope, snapshot_filename, snapshot.message_count, snapshot.announcement_count, snapshot.update_count, snapshot.update_count_full, snapshot.update_count_incremental);
                                        fs::write(&snapshot_path, snapshot.data).unwrap();
-                                       snapshot_filenames_by_day_range.insert(day_range.clone(), snapshot_filename);
+                                       snapshot_filenames_by_scope.insert(current_scope.clone(), snapshot_filename);
                                }
                        }
 
@@ -111,7 +123,7 @@ impl<L: Deref + Clone> Snapshotter<L> where L::Target: Logger {
                                // let's create non-dummy-symlinks
 
                                // first, determine which snapshot range should be referenced
-                               let referenced_day_range = if i == 0 {
+                               let referenced_scope = if i == 0 {
                                        // special-case 0 to always refer to a full/initial sync
                                        u64::MAX
                                } else {
@@ -130,14 +142,14 @@ impl<L: Deref + Clone> Snapshotter<L> where L::Target: Logger {
                                        saturating_mul.
                                         */
 
-                                       // find min(x) in snapshot_sync_day_factors where x >= i
-                                       snapshot_sync_factors.iter().find(|x| {
-                                               snapshot_interval.saturating_mul(**x) >= i * config::SYMLINK_GRANULARITY_INTERVAL as u64
+                                       // find min(x) in snapshot_scopes where i * granularity <= x (the current scope)
+                                       snapshot_scopes.iter().find(|current_scope| {
+                                               i * config::SYMLINK_GRANULARITY_INTERVAL as u64 <= **current_scope
                                        }).unwrap().clone()
                                };
-                               log_info!(self.logger, "i: {}, referenced day range: {}", i, referenced_day_range);
+                               log_info!(self.logger, "i: {}, referenced scope: {}", i, referenced_scope);
 
-                               let snapshot_filename = snapshot_filenames_by_day_range.get(&referenced_day_range).unwrap();
+                               let snapshot_filename = snapshot_filenames_by_scope.get(&referenced_scope).unwrap();
                                let relative_snapshot_path = format!("{}/{}", relative_symlink_to_snapshot_path, snapshot_filename);
 
                                let canonical_last_sync_timestamp = if i == 0 {
@@ -148,7 +160,7 @@ impl<L: Deref + Clone> Snapshotter<L> where L::Target: Logger {
                                };
                                let symlink_path = format!("{}/{}.bin", pending_symlink_directory, canonical_last_sync_timestamp);
 
-                               log_info!(self.logger, "Symlinking: {} -> {} ({} -> {}", i, referenced_day_range, symlink_path, relative_snapshot_path);
+                               log_info!(self.logger, "Symlinking: {} -> {} ({} -> {}", i, referenced_scope, symlink_path, relative_snapshot_path);
                                symlink(&relative_snapshot_path, &symlink_path).unwrap();
                        }
 
@@ -156,10 +168,10 @@ impl<L: Deref + Clone> Snapshotter<L> where L::Target: Logger {
                        let update_time = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
                        fs::write(&update_time_path, format!("{}", update_time)).unwrap();
 
-                       if fs::metadata(&finalized_snapshot_directory).is_ok(){
+                       if fs::metadata(&finalized_snapshot_directory).is_ok() {
                                fs::remove_dir_all(&finalized_snapshot_directory).expect("Failed to remove finalized snapshot directory.");
                        }
-                       if fs::metadata(&finalized_symlink_directory).is_ok(){
+                       if fs::metadata(&finalized_symlink_directory).is_ok() {
                                fs::remove_dir_all(&finalized_symlink_directory).expect("Failed to remove pending symlink directory.");
                        }
                        fs::rename(&pending_snapshot_directory, &finalized_snapshot_directory).expect("Failed to finalize snapshot directory.");
@@ -173,11 +185,11 @@ impl<L: Deref + Clone> Snapshotter<L> where L::Target: Logger {
                        // symlinks will be generated. That should be ok, because any timestamps previously
                        // returned would already have generated symlinks, but this does have bug potential
                        let remainder = current_time % snapshot_interval;
-                       let time_until_next_day = snapshot_interval - remainder;
+                       let time_until_next_generation = snapshot_interval - remainder;
 
-                       log_info!(self.logger, "Sleeping until next snapshot capture: {}s", time_until_next_day);
+                       log_info!(self.logger, "Sleeping until next snapshot capture: {}s", time_until_next_generation);
                        // add in an extra five seconds to assure the rounding down works correctly
-                       let sleep = tokio::time::sleep(Duration::from_secs(time_until_next_day + 5));
+                       let sleep = tokio::time::sleep(Duration::from_secs(time_until_next_generation + 5));
                        sleep.await;
                }
        }