Dynamically adjust snapshot scopes.

[rapid-gossip-sync-server] / src / config.rs

diff --git a/src/config.rs b/src/config.rs
index 1f968d6418760b880cd1db6862ecf892b9446dc7..b905c53f3b877f27b12e21d8fd816f9864a6a1a0 100644 (file)
--- a/src/config.rs
+++ b/src/config.rs
@@ -4,6 +4,7 @@ use std::convert::TryInto;
 use std::env;
 use std::io::Cursor;
 use std::net::{SocketAddr, ToSocketAddrs};
+use std::time::Duration;
 
 use bitcoin::Network;
 use bitcoin::hashes::hex::FromHex;
@@ -14,10 +15,25 @@ use lightning::util::ser::Readable;
 use lightning_block_sync::http::HttpEndpoint;
 use tokio_postgres::Config;
 
-pub(crate) const SCHEMA_VERSION: i32 = 8;
-pub(crate) const SNAPSHOT_CALCULATION_INTERVAL: u32 = 3600 * 24; // every 24 hours, in seconds
+pub(crate) const SCHEMA_VERSION: i32 = 12;
+pub(crate) const SYMLINK_GRANULARITY_INTERVAL: u32 = 3600 * 3; // three hours
+pub(crate) const MAX_SNAPSHOT_SCOPE: u32 = 3600 * 24 * 21; // three weeks
+// generate symlinks based on a 3-hour-granularity
+/// If the last update in either direction was more than six days ago, we send a reminder
+/// That reminder may be either in the form of a channel announcement, or in the form of empty
+/// updates in both directions.
+pub(crate) const CHANNEL_REMINDER_AGE: Duration = Duration::from_secs(6 * 24 * 60 * 60);
 pub(crate) const DOWNLOAD_NEW_GOSSIP: bool = true;
 
+pub(crate) fn snapshot_generation_interval() -> u32 {
+       let interval = env::var("RAPID_GOSSIP_SYNC_SERVER_SNAPSHOT_INTERVAL").unwrap_or(SYMLINK_GRANULARITY_INTERVAL.to_string())
+               .parse::<u32>()
+               .expect("RAPID_GOSSIP_SYNC_SERVER_SNAPSHOT_INTERVAL env variable must be a u32.");
+       assert!(interval > 0, "RAPID_GOSSIP_SYNC_SERVER_SNAPSHOT_INTERVAL must be positive");
+       assert_eq!(interval % SYMLINK_GRANULARITY_INTERVAL, 0, "RAPID_GOSSIP_SYNC_SERVER_SNAPSHOT_INTERVAL must be a multiple of {} (seconds)", SYMLINK_GRANULARITY_INTERVAL);
+       interval
+}
+
 pub(crate) fn network() -> Network {
        let network = env::var("RAPID_GOSSIP_SYNC_SERVER_NETWORK").unwrap_or("bitcoin".to_string()).to_lowercase();
        match network.as_str() {
@@ -30,8 +46,26 @@ pub(crate) fn network() -> Network {
        }
 }
 
-pub(crate) fn network_graph_cache_path() -> &'static str {
-       "./res/network_graph.bin"
+pub(crate) fn log_level() -> lightning::util::logger::Level {
+       let level = env::var("RAPID_GOSSIP_SYNC_SERVER_LOG_LEVEL").unwrap_or("info".to_string()).to_lowercase();
+       match level.as_str() {
+               "gossip" => lightning::util::logger::Level::Gossip,
+               "trace" => lightning::util::logger::Level::Trace,
+               "debug" => lightning::util::logger::Level::Debug,
+               "info" => lightning::util::logger::Level::Info,
+               "warn" => lightning::util::logger::Level::Warn,
+               "error" => lightning::util::logger::Level::Error,
+               _ => panic!("Invalid log level"),
+       }
+}
+
+pub(crate) fn network_graph_cache_path() -> String {
+       format!("{}/network_graph.bin", cache_path())
+}
+
+pub(crate) fn cache_path() -> String {
+       let path = env::var("RAPID_GOSSIP_SYNC_SERVER_CACHES_PATH").unwrap_or("./res".to_string()).to_lowercase();
+       path
 }
 
 pub(crate) fn db_connection_config() -> Config {
@@ -94,11 +128,12 @@ pub(crate) fn db_channel_update_table_creation_query() -> &'static str {
 
 pub(crate) fn db_index_creation_query() -> &'static str {
        "
-       CREATE INDEX IF NOT EXISTS channel_updates_seen ON channel_updates(seen, short_channel_id, direction) INCLUDE (id, blob_signed);
-       CREATE INDEX IF NOT EXISTS channel_updates_scid_seen ON channel_updates(short_channel_id, seen) INCLUDE (blob_signed);
        CREATE INDEX IF NOT EXISTS channel_updates_seen_scid ON channel_updates(seen, short_channel_id);
-       CREATE INDEX IF NOT EXISTS channel_updates_scid_dir_seen ON channel_updates(short_channel_id ASC, direction ASC, seen DESC) INCLUDE (id, blob_signed);
+       CREATE INDEX IF NOT EXISTS channel_updates_scid_dir_seen_asc ON channel_updates(short_channel_id, direction, seen);
+       CREATE INDEX IF NOT EXISTS channel_updates_scid_dir_seen_desc_with_id ON channel_updates(short_channel_id ASC, direction ASC, seen DESC) INCLUDE (id);
        CREATE UNIQUE INDEX IF NOT EXISTS channel_updates_key ON channel_updates (short_channel_id, direction, timestamp);
+       CREATE INDEX IF NOT EXISTS channel_updates_seen ON channel_updates(seen);
+       CREATE INDEX IF NOT EXISTS channel_updates_timestamp_desc ON channel_updates(timestamp DESC);
        "
 }
 
@@ -137,7 +172,7 @@ pub(crate) async fn upgrade_db(schema: i32, client: &mut tokio_postgres::Client)
                                        tx_ref.execute("UPDATE channel_updates SET short_channel_id = $1, direction = $2 WHERE id = $3", &[&scid, &direction, &id]).await.unwrap();
                                });
                        }
-                       while let Some(_) = updates.next().await { }
+                       while let Some(_) = updates.next().await {}
                }
                tx.execute("ALTER TABLE channel_updates ALTER short_channel_id DROP DEFAULT", &[]).await.unwrap();
                tx.execute("ALTER TABLE channel_updates ALTER short_channel_id SET NOT NULL", &[]).await.unwrap();
@@ -164,7 +199,7 @@ pub(crate) async fn upgrade_db(schema: i32, client: &mut tokio_postgres::Client)
                                        tx_ref.execute("UPDATE channel_announcements SET short_channel_id = $1 WHERE id = $2", &[&scid, &id]).await.unwrap();
                                });
                        }
-                       while let Some(_) = updates.next().await { }
+                       while let Some(_) = updates.next().await {}
                }
                tx.execute("ALTER TABLE channel_announcements ADD CONSTRAINT channel_announcements_short_channel_id_key UNIQUE (short_channel_id)", &[]).await.unwrap();
                tx.execute("ALTER TABLE channel_announcements ALTER short_channel_id DROP DEFAULT", &[]).await.unwrap();
@@ -203,15 +238,40 @@ pub(crate) async fn upgrade_db(schema: i32, client: &mut tokio_postgres::Client)
        }
        if schema >= 1 && schema <= 7 {
                let tx = client.transaction().await.unwrap();
-               tx.execute("DROP INDEX channels_seen", &[]).await.unwrap();
-               tx.execute("DROP INDEX channel_updates_scid", &[]).await.unwrap();
-               tx.execute("DROP INDEX channel_updates_direction", &[]).await.unwrap();
-               tx.execute("DROP INDEX channel_updates_seen", &[]).await.unwrap();
-               tx.execute("DROP INDEX channel_updates_scid_seen", &[]).await.unwrap();
-               tx.execute("DROP INDEX channel_updates_scid_dir_seen", &[]).await.unwrap();
+               tx.execute("DROP INDEX IF EXISTS channels_seen", &[]).await.unwrap();
+               tx.execute("DROP INDEX IF EXISTS channel_updates_scid", &[]).await.unwrap();
+               tx.execute("DROP INDEX IF EXISTS channel_updates_direction", &[]).await.unwrap();
+               tx.execute("DROP INDEX IF EXISTS channel_updates_seen", &[]).await.unwrap();
+               tx.execute("DROP INDEX IF EXISTS channel_updates_scid_seen", &[]).await.unwrap();
+               tx.execute("DROP INDEX IF EXISTS channel_updates_scid_dir_seen", &[]).await.unwrap();
                tx.execute("UPDATE config SET db_schema = 8 WHERE id = 1", &[]).await.unwrap();
                tx.commit().await.unwrap();
        }
+       if schema >= 1 && schema <= 8 {
+               let tx = client.transaction().await.unwrap();
+               tx.execute("DROP INDEX IF EXISTS channel_updates_seen", &[]).await.unwrap();
+               tx.execute("DROP INDEX IF EXISTS channel_updates_scid_seen", &[]).await.unwrap();
+               tx.execute("UPDATE config SET db_schema = 9 WHERE id = 1", &[]).await.unwrap();
+               tx.commit().await.unwrap();
+       }
+       if schema >= 1 && schema <= 9 {
+               let tx = client.transaction().await.unwrap();
+               tx.execute("DROP INDEX IF EXISTS channel_updates_scid_dir_seen", &[]).await.unwrap();
+               tx.execute("UPDATE config SET db_schema = 10 WHERE id = 1", &[]).await.unwrap();
+               tx.commit().await.unwrap();
+       }
+       if schema >= 1 && schema <= 10 {
+               let tx = client.transaction().await.unwrap();
+               tx.execute("DROP INDEX IF EXISTS channel_updates_id_with_scid_dir_blob", &[]).await.unwrap();
+               tx.execute("UPDATE config SET db_schema = 11 WHERE id = 1", &[]).await.unwrap();
+               tx.commit().await.unwrap();
+       }
+       if schema >= 1 && schema <= 11 {
+               let tx = client.transaction().await.unwrap();
+               tx.execute("DROP INDEX IF EXISTS channel_updates_seen_with_id_direction_blob", &[]).await.unwrap();
+               tx.execute("UPDATE config SET db_schema = 12 WHERE id = 1", &[]).await.unwrap();
+               tx.commit().await.unwrap();
+       }
        if schema <= 1 || schema > SCHEMA_VERSION {
                panic!("Unknown schema in db: {}, we support up to {}", schema, SCHEMA_VERSION);
        }