3 use std::convert::TryInto;
6 use std::net::{SocketAddr, ToSocketAddrs};
9 use bitcoin::hashes::hex::FromHex;
10 use bitcoin::secp256k1::PublicKey;
11 use futures::stream::{FuturesUnordered, StreamExt};
12 use lightning::ln::msgs::ChannelAnnouncement;
13 use lightning::util::ser::Readable;
14 use lightning_block_sync::http::HttpEndpoint;
15 use tokio_postgres::Config;
17 pub(crate) const SCHEMA_VERSION: i32 = 8;
18 pub(crate) const SNAPSHOT_CALCULATION_INTERVAL: u32 = 3600 * 24; // every 24 hours, in seconds
19 pub(crate) const DOWNLOAD_NEW_GOSSIP: bool = true;
21 pub(crate) fn network() -> Network {
22 let network = env::var("RAPID_GOSSIP_SYNC_SERVER_NETWORK").unwrap_or("bitcoin".to_string()).to_lowercase();
23 match network.as_str() {
24 "mainnet" => Network::Bitcoin,
25 "bitcoin" => Network::Bitcoin,
26 "testnet" => Network::Testnet,
27 "signet" => Network::Signet,
28 "regtest" => Network::Regtest,
29 _ => panic!("Invalid network"),
33 pub(crate) fn network_graph_cache_path() -> String {
34 format!("{}/network_graph.bin", cache_path())
37 pub(crate) fn cache_path() -> String {
38 let path = env::var("RAPID_GOSSIP_SYNC_SERVER_CACHES_PATH").unwrap_or("./res".to_string()).to_lowercase();
42 pub(crate) fn db_connection_config() -> Config {
43 let mut config = Config::new();
44 let host = env::var("RAPID_GOSSIP_SYNC_SERVER_DB_HOST").unwrap_or("localhost".to_string());
45 let user = env::var("RAPID_GOSSIP_SYNC_SERVER_DB_USER").unwrap_or("alice".to_string());
46 let db = env::var("RAPID_GOSSIP_SYNC_SERVER_DB_NAME").unwrap_or("ln_graph_sync".to_string());
50 if let Ok(password) = env::var("RAPID_GOSSIP_SYNC_SERVER_DB_PASSWORD") {
51 config.password(&password);
56 pub(crate) fn bitcoin_rest_endpoint() -> HttpEndpoint {
57 let host = env::var("BITCOIN_REST_DOMAIN").unwrap_or("127.0.0.1".to_string());
58 let port = env::var("BITCOIN_REST_PORT")
59 .unwrap_or("8332".to_string())
61 .expect("BITCOIN_REST_PORT env variable must be a u16.");
62 let path = env::var("BITCOIN_REST_PATH").unwrap_or("/rest/".to_string());
63 HttpEndpoint::for_host(host).with_port(port).with_path(path)
66 pub(crate) fn db_config_table_creation_query() -> &'static str {
67 "CREATE TABLE IF NOT EXISTS config (
68 id SERIAL PRIMARY KEY,
73 pub(crate) fn db_announcement_table_creation_query() -> &'static str {
74 "CREATE TABLE IF NOT EXISTS channel_announcements (
75 id SERIAL PRIMARY KEY,
76 short_channel_id bigint NOT NULL UNIQUE,
77 announcement_signed BYTEA,
78 seen timestamp NOT NULL DEFAULT NOW()
82 pub(crate) fn db_channel_update_table_creation_query() -> &'static str {
83 "CREATE TABLE IF NOT EXISTS channel_updates (
84 id SERIAL PRIMARY KEY,
85 short_channel_id bigint NOT NULL,
86 timestamp bigint NOT NULL,
87 channel_flags smallint NOT NULL,
88 direction boolean NOT NULL,
89 disable boolean NOT NULL,
90 cltv_expiry_delta integer NOT NULL,
91 htlc_minimum_msat bigint NOT NULL,
92 fee_base_msat integer NOT NULL,
93 fee_proportional_millionths integer NOT NULL,
94 htlc_maximum_msat bigint NOT NULL,
95 blob_signed BYTEA NOT NULL,
96 seen timestamp NOT NULL DEFAULT NOW()
100 pub(crate) fn db_index_creation_query() -> &'static str {
102 CREATE INDEX IF NOT EXISTS channel_updates_seen ON channel_updates(seen, short_channel_id, direction) INCLUDE (id, blob_signed);
103 CREATE INDEX IF NOT EXISTS channel_updates_scid_seen ON channel_updates(short_channel_id, seen) INCLUDE (blob_signed);
104 CREATE INDEX IF NOT EXISTS channel_updates_seen_scid ON channel_updates(seen, short_channel_id);
105 CREATE INDEX IF NOT EXISTS channel_updates_scid_dir_seen ON channel_updates(short_channel_id ASC, direction ASC, seen DESC) INCLUDE (id, blob_signed);
106 CREATE UNIQUE INDEX IF NOT EXISTS channel_updates_key ON channel_updates (short_channel_id, direction, timestamp);
110 pub(crate) async fn upgrade_db(schema: i32, client: &mut tokio_postgres::Client) {
112 let tx = client.transaction().await.unwrap();
113 tx.execute("ALTER TABLE channel_updates DROP COLUMN chain_hash", &[]).await.unwrap();
114 tx.execute("ALTER TABLE channel_announcements DROP COLUMN chain_hash", &[]).await.unwrap();
115 tx.execute("UPDATE config SET db_schema = 2 WHERE id = 1", &[]).await.unwrap();
116 tx.commit().await.unwrap();
118 if schema == 1 || schema == 2 {
119 let tx = client.transaction().await.unwrap();
120 tx.execute("ALTER TABLE channel_updates DROP COLUMN short_channel_id", &[]).await.unwrap();
121 tx.execute("ALTER TABLE channel_updates ADD COLUMN short_channel_id bigint DEFAULT null", &[]).await.unwrap();
122 tx.execute("ALTER TABLE channel_updates DROP COLUMN direction", &[]).await.unwrap();
123 tx.execute("ALTER TABLE channel_updates ADD COLUMN direction boolean DEFAULT null", &[]).await.unwrap();
125 let rows = tx.query("SELECT id, composite_index FROM channel_updates WHERE short_channel_id IS NULL LIMIT 50000", &[]).await.unwrap();
126 if rows.is_empty() { break; }
127 let mut updates = FuturesUnordered::new();
129 let id: i32 = row.get("id");
130 let index: String = row.get("composite_index");
132 updates.push(async move {
133 let mut index_iter = index.split(":");
134 let scid_hex = index_iter.next().unwrap();
135 index_iter.next().unwrap();
136 let direction_str = index_iter.next().unwrap();
137 assert!(direction_str == "1" || direction_str == "0");
138 let direction = direction_str == "1";
139 let scid_be_bytes = hex_utils::to_vec(scid_hex).unwrap();
140 let scid = i64::from_be_bytes(scid_be_bytes.try_into().unwrap());
141 assert!(scid > 0); // Will roll over in some 150 years or so
142 tx_ref.execute("UPDATE channel_updates SET short_channel_id = $1, direction = $2 WHERE id = $3", &[&scid, &direction, &id]).await.unwrap();
145 while let Some(_) = updates.next().await { }
147 tx.execute("ALTER TABLE channel_updates ALTER short_channel_id DROP DEFAULT", &[]).await.unwrap();
148 tx.execute("ALTER TABLE channel_updates ALTER short_channel_id SET NOT NULL", &[]).await.unwrap();
149 tx.execute("ALTER TABLE channel_updates ALTER direction DROP DEFAULT", &[]).await.unwrap();
150 tx.execute("ALTER TABLE channel_updates ALTER direction SET NOT NULL", &[]).await.unwrap();
151 tx.execute("UPDATE config SET db_schema = 3 WHERE id = 1", &[]).await.unwrap();
152 tx.commit().await.unwrap();
154 if schema >= 1 && schema <= 3 {
155 let tx = client.transaction().await.unwrap();
156 tx.execute("ALTER TABLE channel_announcements DROP COLUMN short_channel_id", &[]).await.unwrap();
157 tx.execute("ALTER TABLE channel_announcements ADD COLUMN short_channel_id bigint DEFAULT null", &[]).await.unwrap();
159 let rows = tx.query("SELECT id, announcement_signed FROM channel_announcements WHERE short_channel_id IS NULL LIMIT 10000", &[]).await.unwrap();
160 if rows.is_empty() { break; }
161 let mut updates = FuturesUnordered::new();
163 let id: i32 = row.get("id");
164 let announcement: Vec<u8> = row.get("announcement_signed");
166 updates.push(async move {
167 let scid = ChannelAnnouncement::read(&mut Cursor::new(announcement)).unwrap().contents.short_channel_id as i64;
168 assert!(scid > 0); // Will roll over in some 150 years or so
169 tx_ref.execute("UPDATE channel_announcements SET short_channel_id = $1 WHERE id = $2", &[&scid, &id]).await.unwrap();
172 while let Some(_) = updates.next().await { }
174 tx.execute("ALTER TABLE channel_announcements ADD CONSTRAINT channel_announcements_short_channel_id_key UNIQUE (short_channel_id)", &[]).await.unwrap();
175 tx.execute("ALTER TABLE channel_announcements ALTER short_channel_id DROP DEFAULT", &[]).await.unwrap();
176 tx.execute("ALTER TABLE channel_announcements ALTER short_channel_id SET NOT NULL", &[]).await.unwrap();
177 tx.execute("UPDATE config SET db_schema = 4 WHERE id = 1", &[]).await.unwrap();
178 tx.commit().await.unwrap();
180 if schema >= 1 && schema <= 4 {
181 let tx = client.transaction().await.unwrap();
182 tx.execute("ALTER TABLE channel_updates ALTER composite_index SET DATA TYPE character(29)", &[]).await.unwrap();
183 tx.execute("UPDATE config SET db_schema = 5 WHERE id = 1", &[]).await.unwrap();
184 tx.commit().await.unwrap();
186 if schema >= 1 && schema <= 5 {
187 let tx = client.transaction().await.unwrap();
188 tx.execute("ALTER TABLE channel_updates ALTER channel_flags SET DATA TYPE smallint", &[]).await.unwrap();
189 tx.execute("ALTER TABLE channel_announcements DROP COLUMN block_height", &[]).await.unwrap();
190 tx.execute("UPDATE config SET db_schema = 6 WHERE id = 1", &[]).await.unwrap();
191 tx.commit().await.unwrap();
193 if schema >= 1 && schema <= 6 {
194 let tx = client.transaction().await.unwrap();
195 tx.execute("ALTER TABLE channel_updates DROP COLUMN composite_index", &[]).await.unwrap();
196 tx.execute("ALTER TABLE channel_updates ALTER timestamp SET NOT NULL", &[]).await.unwrap();
197 tx.execute("ALTER TABLE channel_updates ALTER channel_flags SET NOT NULL", &[]).await.unwrap();
198 tx.execute("ALTER TABLE channel_updates ALTER disable SET NOT NULL", &[]).await.unwrap();
199 tx.execute("ALTER TABLE channel_updates ALTER cltv_expiry_delta SET NOT NULL", &[]).await.unwrap();
200 tx.execute("ALTER TABLE channel_updates ALTER htlc_minimum_msat SET NOT NULL", &[]).await.unwrap();
201 tx.execute("ALTER TABLE channel_updates ALTER fee_base_msat SET NOT NULL", &[]).await.unwrap();
202 tx.execute("ALTER TABLE channel_updates ALTER fee_proportional_millionths SET NOT NULL", &[]).await.unwrap();
203 tx.execute("ALTER TABLE channel_updates ALTER htlc_maximum_msat SET NOT NULL", &[]).await.unwrap();
204 tx.execute("ALTER TABLE channel_updates ALTER blob_signed SET NOT NULL", &[]).await.unwrap();
205 tx.execute("CREATE UNIQUE INDEX channel_updates_key ON channel_updates (short_channel_id, direction, timestamp)", &[]).await.unwrap();
206 tx.execute("UPDATE config SET db_schema = 7 WHERE id = 1", &[]).await.unwrap();
207 tx.commit().await.unwrap();
209 if schema >= 1 && schema <= 7 {
210 let tx = client.transaction().await.unwrap();
211 tx.execute("DROP INDEX channels_seen", &[]).await.unwrap();
212 tx.execute("DROP INDEX channel_updates_scid", &[]).await.unwrap();
213 tx.execute("DROP INDEX channel_updates_direction", &[]).await.unwrap();
214 tx.execute("DROP INDEX channel_updates_seen", &[]).await.unwrap();
215 tx.execute("DROP INDEX channel_updates_scid_seen", &[]).await.unwrap();
216 tx.execute("DROP INDEX channel_updates_scid_dir_seen", &[]).await.unwrap();
217 tx.execute("UPDATE config SET db_schema = 8 WHERE id = 1", &[]).await.unwrap();
218 tx.commit().await.unwrap();
220 if schema <= 1 || schema > SCHEMA_VERSION {
221 panic!("Unknown schema in db: {}, we support up to {}", schema, SCHEMA_VERSION);
223 // PostgreSQL (at least v13, but likely later versions as well) handles insert-only tables
224 // *very* poorly. After some number of inserts, it refuses to rely on indexes, assuming them to
225 // be possibly-stale, until a VACUUM happens. Thus, we set the vacuum factor really low here,
226 // pushing PostgreSQL to vacuum often.
227 // See https://www.cybertec-postgresql.com/en/postgresql-autovacuum-insert-only-tables/
228 let _ = client.execute("ALTER TABLE channel_updates SET ( autovacuum_vacuum_insert_scale_factor = 0.005 );", &[]).await;
229 let _ = client.execute("ALTER TABLE channel_announcements SET ( autovacuum_vacuum_insert_scale_factor = 0.005 );", &[]).await;
232 pub(crate) fn ln_peers() -> Vec<(PublicKey, SocketAddr)> {
233 const WALLET_OF_SATOSHI: &str = "035e4ff418fc8b5554c5d9eea66396c227bd429a3251c8cbc711002ba215bfc226@170.75.163.209:9735";
234 let list = env::var("LN_PEERS").unwrap_or(WALLET_OF_SATOSHI.to_string());
235 let mut peers = Vec::new();
236 for peer_info in list.split(',') {
237 peers.push(resolve_peer_info(peer_info).expect("Invalid peer info in LN_PEERS"));
242 fn resolve_peer_info(peer_info: &str) -> Result<(PublicKey, SocketAddr), &str> {
243 let mut peer_info = peer_info.splitn(2, '@');
245 let pubkey = peer_info.next().ok_or("Invalid peer info. Should be formatted as: `pubkey@host:port`")?;
246 let pubkey = Vec::from_hex(pubkey).map_err(|_| "Invalid node pubkey")?;
247 let pubkey = PublicKey::from_slice(&pubkey).map_err(|_| "Invalid node pubkey")?;
249 let socket_address = peer_info.next().ok_or("Invalid peer info. Should be formatted as: `pubkey@host:port`")?;
250 let socket_address = socket_address
252 .map_err(|_| "Cannot resolve node address")?
254 .ok_or("Cannot resolve node address")?;
256 Ok((pubkey, socket_address))
261 use super::resolve_peer_info;
262 use bitcoin::hashes::hex::ToHex;
265 fn test_resolve_peer_info() {
266 let wallet_of_satoshi = "035e4ff418fc8b5554c5d9eea66396c227bd429a3251c8cbc711002ba215bfc226@170.75.163.209:9735";
267 let (pubkey, socket_address) = resolve_peer_info(wallet_of_satoshi).unwrap();
268 assert_eq!(pubkey.serialize().to_hex(), "035e4ff418fc8b5554c5d9eea66396c227bd429a3251c8cbc711002ba215bfc226");
269 assert_eq!(socket_address.to_string(), "170.75.163.209:9735");
271 let ipv6 = "033d8656219478701227199cbd6f670335c8d408a92ae88b962c49d4dc0e83e025@[2001:db8::1]:80";
272 let (pubkey, socket_address) = resolve_peer_info(ipv6).unwrap();
273 assert_eq!(pubkey.serialize().to_hex(), "033d8656219478701227199cbd6f670335c8d408a92ae88b962c49d4dc0e83e025");
274 assert_eq!(socket_address.to_string(), "[2001:db8::1]:80");
276 let localhost = "033d8656219478701227199cbd6f670335c8d408a92ae88b962c49d4dc0e83e025@localhost:9735";
277 let (pubkey, socket_address) = resolve_peer_info(localhost).unwrap();
278 assert_eq!(pubkey.serialize().to_hex(), "033d8656219478701227199cbd6f670335c8d408a92ae88b962c49d4dc0e83e025");
279 let socket_address = socket_address.to_string();
280 assert!(socket_address == "127.0.0.1:9735" || socket_address == "[::1]:9735");