Reduce on-startup heap frag due to network graph map/vec doubling

When we're reading a `NetworkGraph`, we know how many
nodes/channels we are reading, there's no reason not to
pre-allocate the `IndexedMap`'s inner `HashMap` and `Vec`, which we
do here.

This seems to reduce on-startup heap fragmentation with glibc by
something like 100MiB.

diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs
index c51180fee731c60197d31e4cf8ad022bf3d1f4b5..e89f63ca2cde54531d7777491200827a1fe5d86c 100644 (file)
--- a/lightning/src/routing/gossip.rs
+++ b/lightning/src/routing/gossip.rs
@@ -1312,14 +1312,16 @@ impl<L: Deref> ReadableArgs<L> for NetworkGraph<L> where L::Target: Logger {
 
                let chain_hash: ChainHash = Readable::read(reader)?;
                let channels_count: u64 = Readable::read(reader)?;
-               let mut channels = IndexedMap::new();
+               // In Nov, 2023 there were about 15,000 nodes; we cap allocations to 1.5x that.
+               let mut channels = IndexedMap::with_capacity(cmp::min(channels_count as usize, 22500));
                for _ in 0..channels_count {
                        let chan_id: u64 = Readable::read(reader)?;
                        let chan_info = Readable::read(reader)?;
                        channels.insert(chan_id, chan_info);
                }
                let nodes_count: u64 = Readable::read(reader)?;
-               let mut nodes = IndexedMap::new();
+               // In Nov, 2023 there were about 69K channels; we cap allocations to 1.5x that.
+               let mut nodes = IndexedMap::with_capacity(cmp::min(nodes_count as usize, 103500));
                for _ in 0..nodes_count {
                        let node_id = Readable::read(reader)?;
                        let node_info = Readable::read(reader)?;

diff --git a/lightning/src/util/indexed_map.rs b/lightning/src/util/indexed_map.rs
index bb17d3450ee5ccb7193ebecd00583603071ba435..39565f048c07a25f30a6749b5057a55a15278664 100644 (file)
--- a/lightning/src/util/indexed_map.rs
+++ b/lightning/src/util/indexed_map.rs
@@ -39,6 +39,14 @@ impl<K: Clone + Hash + Ord, V> IndexedMap<K, V> {
                }
        }
 
+       /// Constructs a new, empty map with the given capacity pre-allocated
+       pub fn with_capacity(capacity: usize) -> Self {
+               Self {
+                       map: HashMap::with_capacity(capacity),
+                       keys: Vec::with_capacity(capacity),
+               }
+       }
+
        #[inline(always)]
        /// Fetches the element with the given `key`, if one exists.
        pub fn get(&self, key: &K) -> Option<&V> {