Calculate the padding required on `ChannelLiquidity` dynamically

We expect `ChannelLiquidity` to be exactly three cache lines to
ensure the first bytes we need are all one one cache line, but in
practice its a bit more ideal for `ChannelLiquidity`s to always
start on an even cache line as x86 CPUs will often load the
neighboring cache line automatically.

Further, it looks like some versions of `rustc` on some platforms
don't pack `ChannelLiquidity` as well (in #3415) and the next
commit is going to push us over three cache lines anyway.

Instead, here we calculate out the proper padding for
`ChannelLiquidity` to make it align to four 64-byte cache lines.

Should fix #3415.

diff --git a/lightning/src/routing/scoring.rs b/lightning/src/routing/scoring.rs
index 3dfc06944a84ab219e3174a25131d5a7981ccdf3..97c28a4a748dca705f12ea8cdf80185e03a022c7 100644 (file)
--- a/lightning/src/routing/scoring.rs
+++ b/lightning/src/routing/scoring.rs
@@ -771,6 +771,19 @@ impl ProbabilisticScoringDecayParameters {
        }
 }
 
+/// A dummy copy of [`ChannelLiquidity`] to calculate its unpadded size
+#[repr(C)]
+struct DummyLiquidity {
+       a: u64,
+       b: u64,
+       c: HistoricalLiquidityTracker,
+       d: Duration,
+       e: Duration,
+}
+
+/// The amount of padding required to make [`ChannelLiquidity`] (plus a u64) a full 4 cache lines.
+const LIQ_PADDING_LEN: usize = (256 - ::core::mem::size_of::<(u64, DummyLiquidity)>()) / 8;
+
 /// Accounting for channel liquidity balance uncertainty.
 ///
 /// Direction is defined in terms of [`NodeId`] partial ordering, where the source node is the
@@ -792,17 +805,25 @@ struct ChannelLiquidity {
        /// Time when the historical liquidity bounds were last modified as an offset against the unix
        /// epoch.
        offset_history_last_updated: Duration,
+
+       _padding: [u64; LIQ_PADDING_LEN],
 }
 
-// Check that the liquidity HashMap's entries sit on round cache lines.
+// Check that the liquidity HashMap's entries sit on round cache line pairs.
+//
+// Most modern CPUs have 64-byte cache lines, so we really want to be on round cache lines to avoid
+// hitting memory too much during scoring. Further, many x86 CPUs (and possibly others) load
+// adjacent cache lines opportunistically in case they will be useful.
 //
-// Specifically, the first cache line will have the key, the liquidity offsets, and the total
-// points tracked in the historical tracker.
+// Thus, we really want our HashMap entries to be aligned to 128 bytes. This will leave the first
+// cache line will have the key, the liquidity offsets, and the total points tracked in the
+// historical tracker.
 //
 // The next two cache lines will have the historical points, which we only access last during
-// scoring, followed by the last_updated `Duration`s (which we do not need during scoring).
-const _LIQUIDITY_MAP_SIZING_CHECK: usize = 192 - ::core::mem::size_of::<(u64, ChannelLiquidity)>();
-const _LIQUIDITY_MAP_SIZING_CHECK_2: usize = ::core::mem::size_of::<(u64, ChannelLiquidity)>() - 192;
+// scoring, followed by the last_updated `Duration`s (which we do not need during scoring). The
+// extra padding brings us up to a clean four cache lines.
+const _LIQUIDITY_MAP_SIZING_CHECK: usize = 256 - ::core::mem::size_of::<(u64, ChannelLiquidity)>();
+const _LIQUIDITY_MAP_SIZING_CHECK_2: usize = ::core::mem::size_of::<(u64, ChannelLiquidity)>() - 256;
 
 /// A snapshot of [`ChannelLiquidity`] in one direction assuming a certain channel capacity.
 struct DirectedChannelLiquidity<L: Deref<Target = u64>, HT: Deref<Target = HistoricalLiquidityTracker>, T: Deref<Target = Duration>> {
@@ -988,6 +1009,7 @@ impl ChannelLiquidity {
                        liquidity_history: HistoricalLiquidityTracker::new(),
                        last_updated,
                        offset_history_last_updated: last_updated,
+                       _padding: [0; LIQ_PADDING_LEN],
                }
        }
 
@@ -1980,13 +2002,14 @@ impl Readable for ChannelLiquidity {
                        ),
                        last_updated,
                        offset_history_last_updated: offset_history_last_updated.unwrap_or(last_updated),
+                       _padding: [0; LIQ_PADDING_LEN],
                })
        }
 }
 
 #[cfg(test)]
 mod tests {
-       use super::{ChannelLiquidity, HistoricalLiquidityTracker, ProbabilisticScoringFeeParameters, ProbabilisticScoringDecayParameters, ProbabilisticScorer};
+       use super::*;
        use crate::blinded_path::BlindedHop;
        use crate::util::config::UserConfig;
 
@@ -2160,12 +2183,14 @@ mod tests {
                                        min_liquidity_offset_msat: 700, max_liquidity_offset_msat: 100,
                                        last_updated, offset_history_last_updated,
                                        liquidity_history: HistoricalLiquidityTracker::new(),
+                                       _padding: [0; LIQ_PADDING_LEN],
                                })
                        .with_channel(43,
                                ChannelLiquidity {
                                        min_liquidity_offset_msat: 700, max_liquidity_offset_msat: 100,
                                        last_updated, offset_history_last_updated,
                                        liquidity_history: HistoricalLiquidityTracker::new(),
+                                       _padding: [0; LIQ_PADDING_LEN],
                                });
                let source = source_node_id();
                let target = target_node_id();
@@ -2239,6 +2264,7 @@ mod tests {
                                        min_liquidity_offset_msat: 200, max_liquidity_offset_msat: 400,
                                        last_updated, offset_history_last_updated,
                                        liquidity_history: HistoricalLiquidityTracker::new(),
+                                       _padding: [0; LIQ_PADDING_LEN],
                                });
                let source = source_node_id();
                let target = target_node_id();
@@ -2299,6 +2325,7 @@ mod tests {
                                        min_liquidity_offset_msat: 200, max_liquidity_offset_msat: 400,
                                        last_updated, offset_history_last_updated,
                                        liquidity_history: HistoricalLiquidityTracker::new(),
+                                       _padding: [0; LIQ_PADDING_LEN],
                                });
                let source = source_node_id();
                let target = target_node_id();
@@ -2418,6 +2445,7 @@ mod tests {
                                        min_liquidity_offset_msat: 40, max_liquidity_offset_msat: 40,
                                        last_updated, offset_history_last_updated,
                                        liquidity_history: HistoricalLiquidityTracker::new(),
+                                       _padding: [0; LIQ_PADDING_LEN],
                                });
                let source = source_node_id();