From fe660c57869f97bcdc4317278b4fe0049ad5b9ff Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Sun, 10 Oct 2021 23:36:44 +0000 Subject: [PATCH] Reload pending payments from ChannelMonitor HTLC data on reload If we go to send a payment, add the HTLC(s) to the channel(s), commit the ChannelMonitor updates to disk, and then crash, we'll come back up with no pending payments but HTLC(s) ready to be claim/failed. This makes it rather impractical to write a payment sender/retryer, as you cannot guarantee atomicity - you cannot guarantee you'll have retry data persisted even if the HTLC(s) are actually pending. Because ChannelMonitors are *the* atomically-persisted data in LDK, we lean on their current HTLC data to figure out what HTLC(s) are a part of an outbound payment, rebuilding the pending payments list on reload. --- lightning/src/chain/channelmonitor.rs | 95 +++++++++++++++++++++++++++ lightning/src/ln/channelmanager.rs | 67 ++++++++++++++++++- 2 files changed, 160 insertions(+), 2 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 7fad40d9c..8be785f29 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -1515,6 +1515,101 @@ impl ChannelMonitor { res } + + /// Gets the set of outbound HTLCs which are pending resolution in this channel. + /// This is used to reconstruct pending outbound payments on restart in the ChannelManager. + pub(crate) fn get_pending_outbound_htlcs(&self) -> HashMap { + let mut res = HashMap::new(); + let us = self.inner.lock().unwrap(); + + macro_rules! walk_htlcs { + ($holder_commitment: expr, $htlc_iter: expr) => { + for (htlc, source) in $htlc_iter { + if us.htlcs_resolved_on_chain.iter().any(|v| Some(v.input_idx) == htlc.transaction_output_index) { + // We should assert that funding_spend_confirmed is_some() here, but we + // have some unit tests which violate HTLC transaction CSVs entirely and + // would fail. + // TODO: Once tests all connect transactions at consensus-valid times, we + // should assert here like we do in `get_claimable_balances`. + } else if htlc.offered == $holder_commitment { + // If the payment was outbound, check if there's an HTLCUpdate + // indicating we have spent this HTLC with a timeout, claiming it back + // and awaiting confirmations on it. + let htlc_update_confd = us.onchain_events_awaiting_threshold_conf.iter().any(|event| { + if let OnchainEvent::HTLCUpdate { input_idx: Some(input_idx), .. } = event.event { + // If the HTLC was timed out, we wait for ANTI_REORG_DELAY blocks + // before considering it "no longer pending" - this matches when we + // provide the ChannelManager an HTLC failure event. + Some(input_idx) == htlc.transaction_output_index && + us.best_block.height() >= event.height + ANTI_REORG_DELAY - 1 + } else if let OnchainEvent::HTLCSpendConfirmation { input_idx, .. } = event.event { + // If the HTLC was fulfilled with a preimage, we consider the HTLC + // immediately non-pending, matching when we provide ChannelManager + // the preimage. + Some(input_idx) == htlc.transaction_output_index + } else { false } + }); + if !htlc_update_confd { + res.insert(source.clone(), htlc.clone()); + } + } + } + } + } + + // We're only concerned with the confirmation count of HTLC transactions, and don't + // actually care how many confirmations a commitment transaction may or may not have. Thus, + // we look for either a FundingSpendConfirmation event or a funding_spend_confirmed. + let confirmed_txid = us.funding_spend_confirmed.or_else(|| { + us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| { + if let OnchainEvent::FundingSpendConfirmation { .. } = event.event { + Some(event.txid) + } else { None } + }) + }); + if let Some(txid) = confirmed_txid { + if Some(txid) == us.current_counterparty_commitment_txid || Some(txid) == us.prev_counterparty_commitment_txid { + walk_htlcs!(false, us.counterparty_claimable_outpoints.get(&txid).unwrap().iter().filter_map(|(a, b)| { + if let &Some(ref source) = b { + Some((a, &**source)) + } else { None } + })); + } else if txid == us.current_holder_commitment_tx.txid { + walk_htlcs!(true, us.current_holder_commitment_tx.htlc_outputs.iter().filter_map(|(a, _, c)| { + if let Some(source) = c { Some((a, source)) } else { None } + })); + } else if let Some(prev_commitment) = &us.prev_holder_signed_commitment_tx { + if txid == prev_commitment.txid { + walk_htlcs!(true, prev_commitment.htlc_outputs.iter().filter_map(|(a, _, c)| { + if let Some(source) = c { Some((a, source)) } else { None } + })); + } + } + } else { + // If we have not seen a commitment transaction on-chain (ie the channel is not yet + // closed), just examine the available counterparty commitment transactions. See docs + // on `fail_unbroadcast_htlcs`, below, for justification. + macro_rules! walk_counterparty_commitment { + ($txid: expr) => { + if let Some(ref latest_outpoints) = us.counterparty_claimable_outpoints.get($txid) { + for &(ref htlc, ref source_option) in latest_outpoints.iter() { + if let &Some(ref source) = source_option { + res.insert((**source).clone(), htlc.clone()); + } + } + } + } + } + if let Some(ref txid) = us.current_counterparty_commitment_txid { + walk_counterparty_commitment!(txid); + } + if let Some(ref txid) = us.prev_counterparty_commitment_txid { + walk_counterparty_commitment!(txid); + } + } + + res + } } /// Compares a broadcasted commitment transaction's HTLCs with those in the latest state, diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 6298a0d9a..9e4708dcf 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -145,7 +145,7 @@ pub(super) enum HTLCForwardInfo { } /// Tracks the inbound corresponding to an outbound HTLC -#[derive(Clone, PartialEq)] +#[derive(Clone, Hash, PartialEq, Eq)] pub(crate) struct HTLCPreviousHopData { short_channel_id: u64, htlc_id: u64, @@ -189,7 +189,8 @@ impl Readable for PaymentId { } } /// Tracks the inbound corresponding to an outbound HTLC -#[derive(Clone, PartialEq)] +#[allow(clippy::derive_hash_xor_eq)] // Our Hash is faithful to the data, we just don't have SecretKey::hash +#[derive(Clone, PartialEq, Eq)] pub(crate) enum HTLCSource { PreviousHopData(HTLCPreviousHopData), OutboundRoute { @@ -202,6 +203,25 @@ pub(crate) enum HTLCSource { payment_secret: Option, }, } +#[allow(clippy::derive_hash_xor_eq)] // Our Hash is faithful to the data, we just don't have SecretKey::hash +impl core::hash::Hash for HTLCSource { + fn hash(&self, hasher: &mut H) { + match self { + HTLCSource::PreviousHopData(prev_hop_data) => { + 0u8.hash(hasher); + prev_hop_data.hash(hasher); + }, + HTLCSource::OutboundRoute { path, session_priv, payment_id, payment_secret, first_hop_htlc_msat } => { + 1u8.hash(hasher); + path.hash(hasher); + session_priv[..].hash(hasher); + payment_id.hash(hasher); + payment_secret.hash(hasher); + first_hop_htlc_msat.hash(hasher); + }, + } + } +} #[cfg(test)] impl HTLCSource { pub fn dummy() -> Self { @@ -5878,6 +5898,49 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> outbounds.insert(id, PendingOutboundPayment::Legacy { session_privs }); } pending_outbound_payments = Some(outbounds); + } else { + // If we're tracking pending payments, ensure we haven't lost any by looking at the + // ChannelMonitor data for any channels for which we do not have authorative state + // (i.e. those for which we just force-closed above or we otherwise don't have a + // corresponding `Channel` at all). + // This avoids several edge-cases where we would otherwise "forget" about pending + // payments which are still in-flight via their on-chain state. + // We only rebuild the pending payments map if we were most recently serialized by + // 0.0.102+ + for (_, monitor) in args.channel_monitors { + if by_id.get(&monitor.get_funding_txo().0.to_channel_id()).is_none() { + for (htlc_source, htlc) in monitor.get_pending_outbound_htlcs() { + if let HTLCSource::OutboundRoute { payment_id, session_priv, path, payment_secret, .. } = htlc_source { + if path.is_empty() { + log_error!(args.logger, "Got an empty path for a pending payment"); + return Err(DecodeError::InvalidValue); + } + let path_amt = path.last().unwrap().fee_msat; + let mut session_priv_bytes = [0; 32]; + session_priv_bytes[..].copy_from_slice(&session_priv[..]); + match pending_outbound_payments.as_mut().unwrap().entry(payment_id) { + hash_map::Entry::Occupied(mut entry) => { + let newly_added = entry.get_mut().insert(session_priv_bytes, path_amt); + log_info!(args.logger, "{} a pending payment path for {} msat for session priv {} on an existing pending payment with payment hash {}", + if newly_added { "Added" } else { "Had" }, path_amt, log_bytes!(session_priv_bytes), log_bytes!(htlc.payment_hash.0)); + }, + hash_map::Entry::Vacant(entry) => { + entry.insert(PendingOutboundPayment::Retryable { + session_privs: [session_priv_bytes].iter().map(|a| *a).collect(), + payment_hash: htlc.payment_hash, + payment_secret, + pending_amt_msat: path_amt, + total_msat: path_amt, + starting_block_height: best_block_height, + }); + log_info!(args.logger, "Added a pending payment for {} msat with payment hash {} for path with session priv {}", + path_amt, log_bytes!(htlc.payment_hash.0), log_bytes!(session_priv_bytes)); + } + } + } + } + } + } } let mut secp_ctx = Secp256k1::new(); -- 2.39.5