From: Matt Corallo <git@bluematt.me>
Date: Sat, 4 Nov 2023 03:53:46 +0000 (+0000)
Subject: Reduce on-startup heap frag due to network graph map/vec doubling
X-Git-Tag: v0.0.119~57^2~11
X-Git-Url: http://git.bitcoin.ninja/?a=commitdiff_plain;h=c0107c6069200f997b7dcbaa997fd2e97d30b44f;p=rust-lightning

Reduce on-startup heap frag due to network graph map/vec doubling

When we're reading a `NetworkGraph`, we know how many
nodes/channels we are reading, there's no reason not to
pre-allocate the `IndexedMap`'s inner `HashMap` and `Vec`, which we
do here.

This seems to reduce on-startup heap fragmentation with glibc by
something like 100MiB.
---

diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs
index c51180fee..e89f63ca2 100644
--- a/lightning/src/routing/gossip.rs
+++ b/lightning/src/routing/gossip.rs
@@ -1312,14 +1312,16 @@ impl<L: Deref> ReadableArgs<L> for NetworkGraph<L> where L::Target: Logger {
 
 		let chain_hash: ChainHash = Readable::read(reader)?;
 		let channels_count: u64 = Readable::read(reader)?;
-		let mut channels = IndexedMap::new();
+		// In Nov, 2023 there were about 15,000 nodes; we cap allocations to 1.5x that.
+		let mut channels = IndexedMap::with_capacity(cmp::min(channels_count as usize, 22500));
 		for _ in 0..channels_count {
 			let chan_id: u64 = Readable::read(reader)?;
 			let chan_info = Readable::read(reader)?;
 			channels.insert(chan_id, chan_info);
 		}
 		let nodes_count: u64 = Readable::read(reader)?;
-		let mut nodes = IndexedMap::new();
+		// In Nov, 2023 there were about 69K channels; we cap allocations to 1.5x that.
+		let mut nodes = IndexedMap::with_capacity(cmp::min(nodes_count as usize, 103500));
 		for _ in 0..nodes_count {
 			let node_id = Readable::read(reader)?;
 			let node_info = Readable::read(reader)?;
diff --git a/lightning/src/util/indexed_map.rs b/lightning/src/util/indexed_map.rs
index bb17d3450..39565f048 100644
--- a/lightning/src/util/indexed_map.rs
+++ b/lightning/src/util/indexed_map.rs
@@ -39,6 +39,14 @@ impl<K: Clone + Hash + Ord, V> IndexedMap<K, V> {
 		}
 	}
 
+	/// Constructs a new, empty map with the given capacity pre-allocated
+	pub fn with_capacity(capacity: usize) -> Self {
+		Self {
+			map: HashMap::with_capacity(capacity),
+			keys: Vec::with_capacity(capacity),
+		}
+	}
+
 	#[inline(always)]
 	/// Fetches the element with the given `key`, if one exists.
 	pub fn get(&self, key: &K) -> Option<&V> {