Reload pending payments from ChannelMonitor HTLC data on reload
authorMatt Corallo <git@bluematt.me>
Sun, 10 Oct 2021 23:36:44 +0000 (23:36 +0000)
committerMatt Corallo <git@bluematt.me>
Fri, 22 Oct 2021 18:41:42 +0000 (18:41 +0000)
If we go to send a payment, add the HTLC(s) to the channel(s),
commit the ChannelMonitor updates to disk, and then crash, we'll
come back up with no pending payments but HTLC(s) ready to be
claim/failed.

This makes it rather impractical to write a payment sender/retryer,
as you cannot guarantee atomicity - you cannot guarantee you'll
have retry data persisted even if the HTLC(s) are actually pending.

Because ChannelMonitors are *the* atomically-persisted data in LDK,
we lean on their current HTLC data to figure out what HTLC(s) are a
part of an outbound payment, rebuilding the pending payments list
on reload.

lightning/src/chain/channelmonitor.rs
lightning/src/ln/channelmanager.rs

index 7fad40d9c2ed70f4252115b3d2f98d16ee2ac2b7..8be785f29d2c38c3165023816e8821b1853418d1 100644 (file)
@@ -1515,6 +1515,101 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
 
                res
        }
+
+       /// Gets the set of outbound HTLCs which are pending resolution in this channel.
+       /// This is used to reconstruct pending outbound payments on restart in the ChannelManager.
+       pub(crate) fn get_pending_outbound_htlcs(&self) -> HashMap<HTLCSource, HTLCOutputInCommitment> {
+               let mut res = HashMap::new();
+               let us = self.inner.lock().unwrap();
+
+               macro_rules! walk_htlcs {
+                       ($holder_commitment: expr, $htlc_iter: expr) => {
+                               for (htlc, source) in $htlc_iter {
+                                       if us.htlcs_resolved_on_chain.iter().any(|v| Some(v.input_idx) == htlc.transaction_output_index) {
+                                               // We should assert that funding_spend_confirmed is_some() here, but we
+                                               // have some unit tests which violate HTLC transaction CSVs entirely and
+                                               // would fail.
+                                               // TODO: Once tests all connect transactions at consensus-valid times, we
+                                               // should assert here like we do in `get_claimable_balances`.
+                                       } else if htlc.offered == $holder_commitment {
+                                               // If the payment was outbound, check if there's an HTLCUpdate
+                                               // indicating we have spent this HTLC with a timeout, claiming it back
+                                               // and awaiting confirmations on it.
+                                               let htlc_update_confd = us.onchain_events_awaiting_threshold_conf.iter().any(|event| {
+                                                       if let OnchainEvent::HTLCUpdate { input_idx: Some(input_idx), .. } = event.event {
+                                                               // If the HTLC was timed out, we wait for ANTI_REORG_DELAY blocks
+                                                               // before considering it "no longer pending" - this matches when we
+                                                               // provide the ChannelManager an HTLC failure event.
+                                                               Some(input_idx) == htlc.transaction_output_index &&
+                                                                       us.best_block.height() >= event.height + ANTI_REORG_DELAY - 1
+                                                       } else if let OnchainEvent::HTLCSpendConfirmation { input_idx, .. } = event.event {
+                                                               // If the HTLC was fulfilled with a preimage, we consider the HTLC
+                                                               // immediately non-pending, matching when we provide ChannelManager
+                                                               // the preimage.
+                                                               Some(input_idx) == htlc.transaction_output_index
+                                                       } else { false }
+                                               });
+                                               if !htlc_update_confd {
+                                                       res.insert(source.clone(), htlc.clone());
+                                               }
+                                       }
+                               }
+                       }
+               }
+
+               // We're only concerned with the confirmation count of HTLC transactions, and don't
+               // actually care how many confirmations a commitment transaction may or may not have. Thus,
+               // we look for either a FundingSpendConfirmation event or a funding_spend_confirmed.
+               let confirmed_txid = us.funding_spend_confirmed.or_else(|| {
+                       us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| {
+                               if let OnchainEvent::FundingSpendConfirmation { .. } = event.event {
+                                       Some(event.txid)
+                               } else { None }
+                       })
+               });
+               if let Some(txid) = confirmed_txid {
+                       if Some(txid) == us.current_counterparty_commitment_txid || Some(txid) == us.prev_counterparty_commitment_txid {
+                               walk_htlcs!(false, us.counterparty_claimable_outpoints.get(&txid).unwrap().iter().filter_map(|(a, b)| {
+                                       if let &Some(ref source) = b {
+                                               Some((a, &**source))
+                                       } else { None }
+                               }));
+                       } else if txid == us.current_holder_commitment_tx.txid {
+                               walk_htlcs!(true, us.current_holder_commitment_tx.htlc_outputs.iter().filter_map(|(a, _, c)| {
+                                       if let Some(source) = c { Some((a, source)) } else { None }
+                               }));
+                       } else if let Some(prev_commitment) = &us.prev_holder_signed_commitment_tx {
+                               if txid == prev_commitment.txid {
+                                       walk_htlcs!(true, prev_commitment.htlc_outputs.iter().filter_map(|(a, _, c)| {
+                                               if let Some(source) = c { Some((a, source)) } else { None }
+                                       }));
+                               }
+                       }
+               } else {
+                       // If we have not seen a commitment transaction on-chain (ie the channel is not yet
+                       // closed), just examine the available counterparty commitment transactions. See docs
+                       // on `fail_unbroadcast_htlcs`, below, for justification.
+                       macro_rules! walk_counterparty_commitment {
+                               ($txid: expr) => {
+                                       if let Some(ref latest_outpoints) = us.counterparty_claimable_outpoints.get($txid) {
+                                               for &(ref htlc, ref source_option) in latest_outpoints.iter() {
+                                                       if let &Some(ref source) = source_option {
+                                                               res.insert((**source).clone(), htlc.clone());
+                                                       }
+                                               }
+                                       }
+                               }
+                       }
+                       if let Some(ref txid) = us.current_counterparty_commitment_txid {
+                               walk_counterparty_commitment!(txid);
+                       }
+                       if let Some(ref txid) = us.prev_counterparty_commitment_txid {
+                               walk_counterparty_commitment!(txid);
+                       }
+               }
+
+               res
+       }
 }
 
 /// Compares a broadcasted commitment transaction's HTLCs with those in the latest state,
index 6298a0d9ae3cbf074e6d8b49f2d54df9f863c658..9e4708dcfd6a6dbeb4e676a5e7f495d09cac7c97 100644 (file)
@@ -145,7 +145,7 @@ pub(super) enum HTLCForwardInfo {
 }
 
 /// Tracks the inbound corresponding to an outbound HTLC
-#[derive(Clone, PartialEq)]
+#[derive(Clone, Hash, PartialEq, Eq)]
 pub(crate) struct HTLCPreviousHopData {
        short_channel_id: u64,
        htlc_id: u64,
@@ -189,7 +189,8 @@ impl Readable for PaymentId {
        }
 }
 /// Tracks the inbound corresponding to an outbound HTLC
-#[derive(Clone, PartialEq)]
+#[allow(clippy::derive_hash_xor_eq)] // Our Hash is faithful to the data, we just don't have SecretKey::hash
+#[derive(Clone, PartialEq, Eq)]
 pub(crate) enum HTLCSource {
        PreviousHopData(HTLCPreviousHopData),
        OutboundRoute {
@@ -202,6 +203,25 @@ pub(crate) enum HTLCSource {
                payment_secret: Option<PaymentSecret>,
        },
 }
+#[allow(clippy::derive_hash_xor_eq)] // Our Hash is faithful to the data, we just don't have SecretKey::hash
+impl core::hash::Hash for HTLCSource {
+       fn hash<H: core::hash::Hasher>(&self, hasher: &mut H) {
+               match self {
+                       HTLCSource::PreviousHopData(prev_hop_data) => {
+                               0u8.hash(hasher);
+                               prev_hop_data.hash(hasher);
+                       },
+                       HTLCSource::OutboundRoute { path, session_priv, payment_id, payment_secret, first_hop_htlc_msat } => {
+                               1u8.hash(hasher);
+                               path.hash(hasher);
+                               session_priv[..].hash(hasher);
+                               payment_id.hash(hasher);
+                               payment_secret.hash(hasher);
+                               first_hop_htlc_msat.hash(hasher);
+                       },
+               }
+       }
+}
 #[cfg(test)]
 impl HTLCSource {
        pub fn dummy() -> Self {
@@ -5878,6 +5898,49 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                                outbounds.insert(id, PendingOutboundPayment::Legacy { session_privs });
                        }
                        pending_outbound_payments = Some(outbounds);
+               } else {
+                       // If we're tracking pending payments, ensure we haven't lost any by looking at the
+                       // ChannelMonitor data for any channels for which we do not have authorative state
+                       // (i.e. those for which we just force-closed above or we otherwise don't have a
+                       // corresponding `Channel` at all).
+                       // This avoids several edge-cases where we would otherwise "forget" about pending
+                       // payments which are still in-flight via their on-chain state.
+                       // We only rebuild the pending payments map if we were most recently serialized by
+                       // 0.0.102+
+                       for (_, monitor) in args.channel_monitors {
+                               if by_id.get(&monitor.get_funding_txo().0.to_channel_id()).is_none() {
+                                       for (htlc_source, htlc) in monitor.get_pending_outbound_htlcs() {
+                                               if let HTLCSource::OutboundRoute { payment_id, session_priv, path, payment_secret, .. } = htlc_source {
+                                                       if path.is_empty() {
+                                                               log_error!(args.logger, "Got an empty path for a pending payment");
+                                                               return Err(DecodeError::InvalidValue);
+                                                       }
+                                                       let path_amt = path.last().unwrap().fee_msat;
+                                                       let mut session_priv_bytes = [0; 32];
+                                                       session_priv_bytes[..].copy_from_slice(&session_priv[..]);
+                                                       match pending_outbound_payments.as_mut().unwrap().entry(payment_id) {
+                                                               hash_map::Entry::Occupied(mut entry) => {
+                                                                       let newly_added = entry.get_mut().insert(session_priv_bytes, path_amt);
+                                                                       log_info!(args.logger, "{} a pending payment path for {} msat for session priv {} on an existing pending payment with payment hash {}",
+                                                                               if newly_added { "Added" } else { "Had" }, path_amt, log_bytes!(session_priv_bytes), log_bytes!(htlc.payment_hash.0));
+                                                               },
+                                                               hash_map::Entry::Vacant(entry) => {
+                                                                       entry.insert(PendingOutboundPayment::Retryable {
+                                                                               session_privs: [session_priv_bytes].iter().map(|a| *a).collect(),
+                                                                               payment_hash: htlc.payment_hash,
+                                                                               payment_secret,
+                                                                               pending_amt_msat: path_amt,
+                                                                               total_msat: path_amt,
+                                                                               starting_block_height: best_block_height,
+                                                                       });
+                                                                       log_info!(args.logger, "Added a pending payment for {} msat with payment hash {} for path with session priv {}",
+                                                                               path_amt, log_bytes!(htlc.payment_hash.0),  log_bytes!(session_priv_bytes));
+                                                               }
+                                                       }
+                                               }
+                                       }
+                               }
+                       }
                }
 
                let mut secp_ctx = Secp256k1::new();