5 use std::net::{SocketAddr, ToSocketAddrs};
6 use std::time::Duration;
9 use bitcoin::hashes::hex::FromHex;
10 use bitcoin::secp256k1::PublicKey;
11 use futures::stream::{FuturesUnordered, StreamExt};
12 use lightning::ln::msgs::ChannelAnnouncement;
13 use lightning::util::ser::Readable;
14 use lightning_block_sync::http::HttpEndpoint;
15 use tokio_postgres::Config;
17 pub(crate) const SCHEMA_VERSION: i32 = 14;
18 pub(crate) const SYMLINK_GRANULARITY_INTERVAL: u32 = 3600 * 3; // three hours
19 pub(crate) const MAX_SNAPSHOT_SCOPE: u32 = 3600 * 24 * 21; // three weeks
20 // generate symlinks based on a 3-hour-granularity
21 /// If the last update in either direction was more than six days ago, we send a reminder
22 /// That reminder may be either in the form of a channel announcement, or in the form of empty
23 /// updates in both directions.
24 pub(crate) const CHANNEL_REMINDER_AGE: Duration = Duration::from_secs(6 * 24 * 60 * 60);
26 /// Maximum number of default features to calculate for node announcements
27 pub(crate) const NODE_DEFAULT_FEATURE_COUNT: u8 = 6;
29 /// The number of successful peer connections to await prior to continuing to gossip storage.
30 /// The application will still work if the number of specified peers is lower, as long as there is
31 /// at least one successful peer connection, but it may result in long startup times.
32 pub(crate) const CONNECTED_PEER_ASSERTION_LIMIT: usize = 5;
33 pub(crate) const DOWNLOAD_NEW_GOSSIP: bool = true;
35 pub(crate) fn snapshot_generation_interval() -> u32 {
36 let interval = env::var("RAPID_GOSSIP_SYNC_SERVER_SNAPSHOT_INTERVAL").unwrap_or(SYMLINK_GRANULARITY_INTERVAL.to_string())
38 .expect("RAPID_GOSSIP_SYNC_SERVER_SNAPSHOT_INTERVAL env variable must be a u32.");
39 assert!(interval > 0, "RAPID_GOSSIP_SYNC_SERVER_SNAPSHOT_INTERVAL must be positive");
40 assert_eq!(interval % SYMLINK_GRANULARITY_INTERVAL, 0, "RAPID_GOSSIP_SYNC_SERVER_SNAPSHOT_INTERVAL must be a multiple of {} (seconds)", SYMLINK_GRANULARITY_INTERVAL);
44 pub(crate) fn network() -> Network {
45 let network = env::var("RAPID_GOSSIP_SYNC_SERVER_NETWORK").unwrap_or("bitcoin".to_string()).to_lowercase();
46 match network.as_str() {
47 "mainnet" => Network::Bitcoin,
48 "bitcoin" => Network::Bitcoin,
49 "testnet" => Network::Testnet,
50 "signet" => Network::Signet,
51 "regtest" => Network::Regtest,
52 _ => panic!("Invalid network"),
56 pub(crate) fn log_level() -> lightning::util::logger::Level {
57 let level = env::var("RAPID_GOSSIP_SYNC_SERVER_LOG_LEVEL").unwrap_or("info".to_string()).to_lowercase();
58 match level.as_str() {
59 "gossip" => lightning::util::logger::Level::Gossip,
60 "trace" => lightning::util::logger::Level::Trace,
61 "debug" => lightning::util::logger::Level::Debug,
62 "info" => lightning::util::logger::Level::Info,
63 "warn" => lightning::util::logger::Level::Warn,
64 "error" => lightning::util::logger::Level::Error,
65 _ => panic!("Invalid log level"),
69 pub(crate) fn network_graph_cache_path() -> String {
70 format!("{}/network_graph.bin", cache_path())
73 pub(crate) fn cache_path() -> String {
74 let path = env::var("RAPID_GOSSIP_SYNC_SERVER_CACHES_PATH").unwrap_or("./res".to_string()).to_lowercase();
78 pub(crate) fn db_connection_config() -> Config {
79 let mut config = Config::new();
80 let env_name_prefix = if cfg!(test) {
81 "RAPID_GOSSIP_TEST_DB"
83 "RAPID_GOSSIP_SYNC_SERVER_DB"
86 let host = env::var(format!("{}{}", env_name_prefix, "_HOST")).unwrap_or("localhost".to_string());
87 let user = env::var(format!("{}{}", env_name_prefix, "_USER")).unwrap_or("alice".to_string());
88 let db = env::var(format!("{}{}", env_name_prefix, "_NAME")).unwrap_or("ln_graph_sync".to_string());
92 if let Ok(password) = env::var(format!("{}{}", env_name_prefix, "_PASSWORD")) {
93 config.password(&password);
98 pub(crate) fn bitcoin_rest_endpoint() -> HttpEndpoint {
99 let host = env::var("BITCOIN_REST_DOMAIN").unwrap_or("127.0.0.1".to_string());
100 let port = env::var("BITCOIN_REST_PORT")
101 .unwrap_or("8332".to_string())
103 .expect("BITCOIN_REST_PORT env variable must be a u16.");
104 let path = env::var("BITCOIN_REST_PATH").unwrap_or("/rest/".to_string());
105 HttpEndpoint::for_host(host).with_port(port).with_path(path)
108 pub(crate) fn db_config_table_creation_query() -> &'static str {
109 "CREATE TABLE IF NOT EXISTS config (
110 id SERIAL PRIMARY KEY,
115 pub(crate) fn db_announcement_table_creation_query() -> &'static str {
116 "CREATE TABLE IF NOT EXISTS channel_announcements (
117 id SERIAL PRIMARY KEY,
118 short_channel_id bigint NOT NULL UNIQUE,
119 announcement_signed BYTEA,
120 seen timestamp NOT NULL DEFAULT NOW()
124 pub(crate) fn db_channel_update_table_creation_query() -> &'static str {
125 "CREATE TABLE IF NOT EXISTS channel_updates (
126 id SERIAL PRIMARY KEY,
127 short_channel_id bigint NOT NULL,
128 timestamp bigint NOT NULL,
129 channel_flags smallint NOT NULL,
130 direction boolean NOT NULL,
131 disable boolean NOT NULL,
132 cltv_expiry_delta integer NOT NULL,
133 htlc_minimum_msat bigint NOT NULL,
134 fee_base_msat integer NOT NULL,
135 fee_proportional_millionths integer NOT NULL,
136 htlc_maximum_msat bigint NOT NULL,
137 blob_signed BYTEA NOT NULL,
138 seen timestamp NOT NULL DEFAULT NOW()
142 pub(crate) fn db_node_announcement_table_creation_query() -> &'static str {
143 "CREATE TABLE IF NOT EXISTS node_announcements (
144 id SERIAL PRIMARY KEY,
145 public_key varchar(66) NOT NULL,
146 features BYTEA NOT NULL,
147 socket_addresses BYTEA NOT NULL,
148 timestamp bigint NOT NULL,
149 announcement_signed BYTEA,
150 seen timestamp NOT NULL DEFAULT NOW()
154 pub(crate) fn db_index_creation_query() -> &'static str {
156 CREATE INDEX IF NOT EXISTS channel_updates_seen_scid ON channel_updates(seen, short_channel_id);
157 CREATE INDEX IF NOT EXISTS channel_updates_scid_dir_seen_asc ON channel_updates(short_channel_id, direction, seen);
158 CREATE INDEX IF NOT EXISTS channel_updates_scid_dir_seen_desc_with_id ON channel_updates(short_channel_id ASC, direction ASC, seen DESC) INCLUDE (id);
159 CREATE UNIQUE INDEX IF NOT EXISTS channel_updates_key ON channel_updates (short_channel_id, direction, timestamp);
160 CREATE INDEX IF NOT EXISTS channel_updates_seen ON channel_updates(seen);
161 CREATE INDEX IF NOT EXISTS channel_updates_scid_asc_timestamp_desc ON channel_updates(short_channel_id ASC, timestamp DESC);
165 pub(crate) async fn upgrade_db(schema: i32, client: &mut tokio_postgres::Client) {
167 let tx = client.transaction().await.unwrap();
168 tx.execute("ALTER TABLE channel_updates DROP COLUMN chain_hash", &[]).await.unwrap();
169 tx.execute("ALTER TABLE channel_announcements DROP COLUMN chain_hash", &[]).await.unwrap();
170 tx.execute("UPDATE config SET db_schema = 2 WHERE id = 1", &[]).await.unwrap();
171 tx.commit().await.unwrap();
173 if schema == 1 || schema == 2 {
174 let tx = client.transaction().await.unwrap();
175 tx.execute("ALTER TABLE channel_updates DROP COLUMN short_channel_id", &[]).await.unwrap();
176 tx.execute("ALTER TABLE channel_updates ADD COLUMN short_channel_id bigint DEFAULT null", &[]).await.unwrap();
177 tx.execute("ALTER TABLE channel_updates DROP COLUMN direction", &[]).await.unwrap();
178 tx.execute("ALTER TABLE channel_updates ADD COLUMN direction boolean DEFAULT null", &[]).await.unwrap();
180 let rows = tx.query("SELECT id, composite_index FROM channel_updates WHERE short_channel_id IS NULL LIMIT 50000", &[]).await.unwrap();
181 if rows.is_empty() { break; }
182 let mut updates = FuturesUnordered::new();
184 let id: i32 = row.get("id");
185 let index: String = row.get("composite_index");
187 updates.push(async move {
188 let mut index_iter = index.split(":");
189 let scid_hex = index_iter.next().unwrap();
190 index_iter.next().unwrap();
191 let direction_str = index_iter.next().unwrap();
192 assert!(direction_str == "1" || direction_str == "0");
193 let direction = direction_str == "1";
194 let scid_be_bytes = hex_utils::to_vec(scid_hex).unwrap();
195 let scid = i64::from_be_bytes(scid_be_bytes.try_into().unwrap());
196 assert!(scid > 0); // Will roll over in some 150 years or so
197 tx_ref.execute("UPDATE channel_updates SET short_channel_id = $1, direction = $2 WHERE id = $3", &[&scid, &direction, &id]).await.unwrap();
200 while let Some(_) = updates.next().await {}
202 tx.execute("ALTER TABLE channel_updates ALTER short_channel_id DROP DEFAULT", &[]).await.unwrap();
203 tx.execute("ALTER TABLE channel_updates ALTER short_channel_id SET NOT NULL", &[]).await.unwrap();
204 tx.execute("ALTER TABLE channel_updates ALTER direction DROP DEFAULT", &[]).await.unwrap();
205 tx.execute("ALTER TABLE channel_updates ALTER direction SET NOT NULL", &[]).await.unwrap();
206 tx.execute("UPDATE config SET db_schema = 3 WHERE id = 1", &[]).await.unwrap();
207 tx.commit().await.unwrap();
209 if schema >= 1 && schema <= 3 {
210 let tx = client.transaction().await.unwrap();
211 tx.execute("ALTER TABLE channel_announcements DROP COLUMN short_channel_id", &[]).await.unwrap();
212 tx.execute("ALTER TABLE channel_announcements ADD COLUMN short_channel_id bigint DEFAULT null", &[]).await.unwrap();
214 let rows = tx.query("SELECT id, announcement_signed FROM channel_announcements WHERE short_channel_id IS NULL LIMIT 10000", &[]).await.unwrap();
215 if rows.is_empty() { break; }
216 let mut updates = FuturesUnordered::new();
218 let id: i32 = row.get("id");
219 let announcement: Vec<u8> = row.get("announcement_signed");
221 updates.push(async move {
222 let scid = ChannelAnnouncement::read(&mut Cursor::new(announcement)).unwrap().contents.short_channel_id as i64;
223 assert!(scid > 0); // Will roll over in some 150 years or so
224 tx_ref.execute("UPDATE channel_announcements SET short_channel_id = $1 WHERE id = $2", &[&scid, &id]).await.unwrap();
227 while let Some(_) = updates.next().await {}
229 tx.execute("ALTER TABLE channel_announcements ADD CONSTRAINT channel_announcements_short_channel_id_key UNIQUE (short_channel_id)", &[]).await.unwrap();
230 tx.execute("ALTER TABLE channel_announcements ALTER short_channel_id DROP DEFAULT", &[]).await.unwrap();
231 tx.execute("ALTER TABLE channel_announcements ALTER short_channel_id SET NOT NULL", &[]).await.unwrap();
232 tx.execute("UPDATE config SET db_schema = 4 WHERE id = 1", &[]).await.unwrap();
233 tx.commit().await.unwrap();
235 if schema >= 1 && schema <= 4 {
236 let tx = client.transaction().await.unwrap();
237 tx.execute("ALTER TABLE channel_updates ALTER composite_index SET DATA TYPE character(29)", &[]).await.unwrap();
238 tx.execute("UPDATE config SET db_schema = 5 WHERE id = 1", &[]).await.unwrap();
239 tx.commit().await.unwrap();
241 if schema >= 1 && schema <= 5 {
242 let tx = client.transaction().await.unwrap();
243 tx.execute("ALTER TABLE channel_updates ALTER channel_flags SET DATA TYPE smallint", &[]).await.unwrap();
244 tx.execute("ALTER TABLE channel_announcements DROP COLUMN block_height", &[]).await.unwrap();
245 tx.execute("UPDATE config SET db_schema = 6 WHERE id = 1", &[]).await.unwrap();
246 tx.commit().await.unwrap();
248 if schema >= 1 && schema <= 6 {
249 let tx = client.transaction().await.unwrap();
250 tx.execute("ALTER TABLE channel_updates DROP COLUMN composite_index", &[]).await.unwrap();
251 tx.execute("ALTER TABLE channel_updates ALTER timestamp SET NOT NULL", &[]).await.unwrap();
252 tx.execute("ALTER TABLE channel_updates ALTER channel_flags SET NOT NULL", &[]).await.unwrap();
253 tx.execute("ALTER TABLE channel_updates ALTER disable SET NOT NULL", &[]).await.unwrap();
254 tx.execute("ALTER TABLE channel_updates ALTER cltv_expiry_delta SET NOT NULL", &[]).await.unwrap();
255 tx.execute("ALTER TABLE channel_updates ALTER htlc_minimum_msat SET NOT NULL", &[]).await.unwrap();
256 tx.execute("ALTER TABLE channel_updates ALTER fee_base_msat SET NOT NULL", &[]).await.unwrap();
257 tx.execute("ALTER TABLE channel_updates ALTER fee_proportional_millionths SET NOT NULL", &[]).await.unwrap();
258 tx.execute("ALTER TABLE channel_updates ALTER htlc_maximum_msat SET NOT NULL", &[]).await.unwrap();
259 tx.execute("ALTER TABLE channel_updates ALTER blob_signed SET NOT NULL", &[]).await.unwrap();
260 tx.execute("CREATE UNIQUE INDEX channel_updates_key ON channel_updates (short_channel_id, direction, timestamp)", &[]).await.unwrap();
261 tx.execute("UPDATE config SET db_schema = 7 WHERE id = 1", &[]).await.unwrap();
262 tx.commit().await.unwrap();
264 if schema >= 1 && schema <= 7 {
265 let tx = client.transaction().await.unwrap();
266 tx.execute("DROP INDEX IF EXISTS channels_seen", &[]).await.unwrap();
267 tx.execute("DROP INDEX IF EXISTS channel_updates_scid", &[]).await.unwrap();
268 tx.execute("DROP INDEX IF EXISTS channel_updates_direction", &[]).await.unwrap();
269 tx.execute("DROP INDEX IF EXISTS channel_updates_seen", &[]).await.unwrap();
270 tx.execute("DROP INDEX IF EXISTS channel_updates_scid_seen", &[]).await.unwrap();
271 tx.execute("DROP INDEX IF EXISTS channel_updates_scid_dir_seen", &[]).await.unwrap();
272 tx.execute("UPDATE config SET db_schema = 8 WHERE id = 1", &[]).await.unwrap();
273 tx.commit().await.unwrap();
275 if schema >= 1 && schema <= 8 {
276 let tx = client.transaction().await.unwrap();
277 tx.execute("DROP INDEX IF EXISTS channel_updates_seen", &[]).await.unwrap();
278 tx.execute("DROP INDEX IF EXISTS channel_updates_scid_seen", &[]).await.unwrap();
279 tx.execute("UPDATE config SET db_schema = 9 WHERE id = 1", &[]).await.unwrap();
280 tx.commit().await.unwrap();
282 if schema >= 1 && schema <= 9 {
283 let tx = client.transaction().await.unwrap();
284 tx.execute("DROP INDEX IF EXISTS channel_updates_scid_dir_seen", &[]).await.unwrap();
285 tx.execute("UPDATE config SET db_schema = 10 WHERE id = 1", &[]).await.unwrap();
286 tx.commit().await.unwrap();
288 if schema >= 1 && schema <= 10 {
289 let tx = client.transaction().await.unwrap();
290 tx.execute("DROP INDEX IF EXISTS channel_updates_id_with_scid_dir_blob", &[]).await.unwrap();
291 tx.execute("UPDATE config SET db_schema = 11 WHERE id = 1", &[]).await.unwrap();
292 tx.commit().await.unwrap();
294 if schema >= 1 && schema <= 11 {
295 let tx = client.transaction().await.unwrap();
296 tx.execute("DROP INDEX IF EXISTS channel_updates_seen_with_id_direction_blob", &[]).await.unwrap();
297 tx.execute("UPDATE config SET db_schema = 12 WHERE id = 1", &[]).await.unwrap();
298 tx.commit().await.unwrap();
300 if schema >= 1 && schema <= 12 {
301 let tx = client.transaction().await.unwrap();
302 tx.execute("DROP INDEX IF EXISTS channel_updates_timestamp_desc", &[]).await.unwrap();
303 tx.execute("UPDATE config SET db_schema = 13 WHERE id = 1", &[]).await.unwrap();
304 tx.commit().await.unwrap();
306 if schema >= 1 && schema <= 13 {
307 let tx = client.transaction().await.unwrap();
308 tx.execute("UPDATE config SET db_schema = 14 WHERE id = 1", &[]).await.unwrap();
309 tx.commit().await.unwrap();
311 if schema <= 1 || schema > SCHEMA_VERSION {
312 panic!("Unknown schema in db: {}, we support up to {}", schema, SCHEMA_VERSION);
314 // PostgreSQL (at least v13, but likely later versions as well) handles insert-only tables
315 // *very* poorly. After some number of inserts, it refuses to rely on indexes, assuming them to
316 // be possibly-stale, until a VACUUM happens. Thus, we set the vacuum factor really low here,
317 // pushing PostgreSQL to vacuum often.
318 // See https://www.cybertec-postgresql.com/en/postgresql-autovacuum-insert-only-tables/
319 let _ = client.execute("ALTER TABLE channel_updates SET ( autovacuum_vacuum_insert_scale_factor = 0.005 );", &[]).await;
320 let _ = client.execute("ALTER TABLE channel_announcements SET ( autovacuum_vacuum_insert_scale_factor = 0.005 );", &[]).await;
323 pub(crate) fn ln_peers() -> Vec<(PublicKey, SocketAddr)> {
324 const WALLET_OF_SATOSHI: &str = "035e4ff418fc8b5554c5d9eea66396c227bd429a3251c8cbc711002ba215bfc226@170.75.163.209:9735";
325 let list = env::var("LN_PEERS").unwrap_or(WALLET_OF_SATOSHI.to_string());
326 let mut peers = Vec::new();
327 for (item, peer_info) in list.split(',').enumerate() {
328 // Ignore leading or trailing whitespace
329 let trimmed_peer_info = peer_info.trim();
330 // Ignore trailing or repeated commas
331 if !trimmed_peer_info.is_empty() {
332 peers.push(resolve_peer_info(trimmed_peer_info).unwrap_or_else(|_| {
333 panic!("Invalid peer info in LN_PEERS at item {}: {}", item, peer_info)
340 fn resolve_peer_info(peer_info: &str) -> Result<(PublicKey, SocketAddr), &str> {
341 let mut peer_info = peer_info.splitn(2, '@');
343 let pubkey = peer_info.next().ok_or("Invalid peer info. Should be formatted as: `pubkey@host:port`")?;
344 let pubkey = Vec::from_hex(pubkey).map_err(|_| "Invalid node pubkey")?;
345 let pubkey = PublicKey::from_slice(&pubkey).map_err(|_| "Invalid node pubkey")?;
347 let socket_address = peer_info.next().ok_or("Invalid peer info. Should be formatted as: `pubkey@host:port`")?;
348 let socket_address = socket_address
350 .map_err(|_| "Cannot resolve node address")?
352 .ok_or("Cannot resolve node address")?;
354 Ok((pubkey, socket_address))
360 use hex_conservative::DisplayHex;
361 use std::str::FromStr;
364 fn test_resolve_peer_info() {
365 let wallet_of_satoshi = "035e4ff418fc8b5554c5d9eea66396c227bd429a3251c8cbc711002ba215bfc226@170.75.163.209:9735";
366 let (pubkey, socket_address) = resolve_peer_info(wallet_of_satoshi).unwrap();
368 pubkey.serialize().to_lower_hex_string(),
369 "035e4ff418fc8b5554c5d9eea66396c227bd429a3251c8cbc711002ba215bfc226"
371 assert_eq!(socket_address.to_string(), "170.75.163.209:9735");
373 let ipv6 = "033d8656219478701227199cbd6f670335c8d408a92ae88b962c49d4dc0e83e025@[2001:db8::1]:80";
374 let (pubkey, socket_address) = resolve_peer_info(ipv6).unwrap();
376 pubkey.serialize().to_lower_hex_string(),
377 "033d8656219478701227199cbd6f670335c8d408a92ae88b962c49d4dc0e83e025"
379 assert_eq!(socket_address.to_string(), "[2001:db8::1]:80");
381 let localhost = "033d8656219478701227199cbd6f670335c8d408a92ae88b962c49d4dc0e83e025@localhost:9735";
382 let (pubkey, socket_address) = resolve_peer_info(localhost).unwrap();
384 pubkey.serialize().to_lower_hex_string(),
385 "033d8656219478701227199cbd6f670335c8d408a92ae88b962c49d4dc0e83e025"
387 let socket_address = socket_address.to_string();
388 assert!(socket_address == "127.0.0.1:9735" || socket_address == "[::1]:9735");
393 // Set the environment variable, including a repeated comma, leading space, and trailing comma.
394 std::env::set_var("LN_PEERS", "035e4ff418fc8b5554c5d9eea66396c227bd429a3251c8cbc711002ba215bfc226@170.75.163.209:9735,, 035e4ff418fc8b5554c5d9eea66396c227bd429a3251c8cbc711002ba215bfc227@170.75.163.210:9735,");
395 let peers = ln_peers();
397 // Assert output is as expected
402 PublicKey::from_str("035e4ff418fc8b5554c5d9eea66396c227bd429a3251c8cbc711002ba215bfc226").unwrap(),
403 SocketAddr::from_str("170.75.163.209:9735").unwrap()
406 PublicKey::from_str("035e4ff418fc8b5554c5d9eea66396c227bd429a3251c8cbc711002ba215bfc227").unwrap(),
407 SocketAddr::from_str("170.75.163.210:9735").unwrap()