Swap pending_outbound_payments for OutboundPayments struct
authorValentine Wallace <vwallace@protonmail.com>
Sun, 18 Dec 2022 18:28:47 +0000 (13:28 -0500)
committerValentine Wallace <vwallace@protonmail.com>
Tue, 20 Dec 2022 02:04:53 +0000 (21:04 -0500)
This allows us to move a lot of outbound payment logic out of ChannelManager
and into the new outbound_payment module, and helps avoid growing
ChannelManager when we add retry logic to it in upcoming work.

lightning/src/ln/channelmanager.rs
lightning/src/ln/outbound_payment.rs

index ef492c0ce6296b7e838fa1e7099096239c912506..61cc2af81c7d8f50447f8ee02471df230dabe4a6 100644 (file)
@@ -46,12 +46,14 @@ use crate::ln::channel::{Channel, ChannelError, ChannelUpdateStatus, UpdateFulfi
 use crate::ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, NodeFeatures};
 #[cfg(any(feature = "_test_utils", test))]
 use crate::ln::features::InvoiceFeatures;
-use crate::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, RoutePath, RouteParameters};
+use crate::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, RoutePath};
 use crate::ln::msgs;
 use crate::ln::onion_utils;
 use crate::ln::onion_utils::HTLCFailReason;
 use crate::ln::msgs::{ChannelMessageHandler, DecodeError, LightningError, MAX_VALUE_MSAT};
-use crate::ln::outbound_payment::PendingOutboundPayment;
+#[cfg(test)]
+use crate::ln::outbound_payment;
+use crate::ln::outbound_payment::{OutboundPayments, PendingOutboundPayment};
 use crate::ln::wire::Encode;
 use crate::chain::keysinterface::{Sign, KeysInterface, KeysManager, Recipient};
 use crate::util::config::{UserConfig, ChannelConfig};
@@ -563,7 +565,7 @@ pub type SimpleRefChannelManager<'a, 'b, 'c, 'd, 'e, M, T, F, L> = ChannelManage
 //  |   |
 //  |   |__`claimable_payments`
 //  |   |
-//  |   |__`pending_outbound_payments`
+//  |   |__`pending_outbound_payments` // This field's struct contains a map of pending outbounds
 //  |       |
 //  |       |__`channel_state`
 //  |           |
@@ -627,7 +629,7 @@ pub struct ChannelManager<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
        /// See `PendingOutboundPayment` documentation for more info.
        ///
        /// See `ChannelManager` struct-level documentation for lock order requirements.
-       pending_outbound_payments: Mutex<HashMap<PaymentId, PendingOutboundPayment>>,
+       pending_outbound_payments: OutboundPayments,
 
        /// SCID/SCID Alias -> forward infos. Key of 0 means payments received.
        ///
@@ -873,7 +875,7 @@ pub(crate) const MPP_TIMEOUT_TICKS: u8 = 3;
 
 /// The number of ticks of [`ChannelManager::timer_tick_occurred`] until we time-out the
 /// idempotency of payments by [`PaymentId`]. See
-/// [`ChannelManager::remove_stale_resolved_payments`].
+/// [`OutboundPayments::remove_stale_resolved_payments`].
 pub(crate) const IDEMPOTENCY_TIMEOUT_TICKS: u8 = 7;
 
 /// Information needed for constructing an invoice route hint for this channel.
@@ -1393,7 +1395,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                        }),
                        outbound_scid_aliases: Mutex::new(HashSet::new()),
                        pending_inbound_payments: Mutex::new(HashMap::new()),
-                       pending_outbound_payments: Mutex::new(HashMap::new()),
+                       pending_outbound_payments: OutboundPayments::new(),
                        forward_htlcs: Mutex::new(HashMap::new()),
                        claimable_payments: Mutex::new(ClaimablePayments { claimable_htlcs: HashMap::new(), pending_claiming_payments: HashMap::new() }),
                        pending_intercepted_htlcs: Mutex::new(HashMap::new()),
@@ -2366,142 +2368,27 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        /// [`Event::PaymentSent`]: events::Event::PaymentSent
        /// [`PeerManager::process_events`]: crate::ln::peer_handler::PeerManager::process_events
        pub fn send_payment(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, payment_id: PaymentId) -> Result<(), PaymentSendFailure> {
-               let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret, payment_id, route)?;
-               self.send_payment_internal(route, payment_hash, payment_secret, None, payment_id, None, onion_session_privs)
+               let best_block_height = self.best_block.read().unwrap().height();
+               self.pending_outbound_payments
+                       .send_payment(route, payment_hash, payment_secret, payment_id, &self.keys_manager, best_block_height,
+                               |path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv|
+                               self.send_payment_along_path(path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv))
        }
 
        #[cfg(test)]
-       pub(crate) fn test_add_new_pending_payment(&self, payment_hash: PaymentHash, payment_secret: Option<PaymentSecret>, payment_id: PaymentId, route: &Route) -> Result<Vec<[u8; 32]>, PaymentSendFailure> {
-               self.add_new_pending_payment(payment_hash, payment_secret, payment_id, route)
+       fn test_send_payment_internal(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, keysend_preimage: Option<PaymentPreimage>, payment_id: PaymentId, recv_value_msat: Option<u64>, onion_session_privs: Vec<[u8; 32]>) -> Result<(), PaymentSendFailure> {
+               let best_block_height = self.best_block.read().unwrap().height();
+               self.pending_outbound_payments.test_send_payment_internal(route, payment_hash, payment_secret, keysend_preimage, payment_id, recv_value_msat, onion_session_privs, &self.keys_manager, best_block_height,
+                       |path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv|
+                       self.send_payment_along_path(path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv))
        }
 
-       fn add_new_pending_payment(&self, payment_hash: PaymentHash, payment_secret: Option<PaymentSecret>, payment_id: PaymentId, route: &Route) -> Result<Vec<[u8; 32]>, PaymentSendFailure> {
-               let mut onion_session_privs = Vec::with_capacity(route.paths.len());
-               for _ in 0..route.paths.len() {
-                       onion_session_privs.push(self.keys_manager.get_secure_random_bytes());
-               }
-
-               let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap();
-               match pending_outbounds.entry(payment_id) {
-                       hash_map::Entry::Occupied(_) => Err(PaymentSendFailure::DuplicatePayment),
-                       hash_map::Entry::Vacant(entry) => {
-                               let payment = entry.insert(PendingOutboundPayment::Retryable {
-                                       session_privs: HashSet::new(),
-                                       pending_amt_msat: 0,
-                                       pending_fee_msat: Some(0),
-                                       payment_hash,
-                                       payment_secret,
-                                       starting_block_height: self.best_block.read().unwrap().height(),
-                                       total_msat: route.get_total_amount(),
-                               });
-
-                               for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) {
-                                       assert!(payment.insert(*session_priv_bytes, path));
-                               }
-
-                               Ok(onion_session_privs)
-                       },
-               }
+       #[cfg(test)]
+       pub(crate) fn test_add_new_pending_payment(&self, payment_hash: PaymentHash, payment_secret: Option<PaymentSecret>, payment_id: PaymentId, route: &Route) -> Result<Vec<[u8; 32]>, PaymentSendFailure> {
+               let best_block_height = self.best_block.read().unwrap().height();
+               self.pending_outbound_payments.add_new_pending_payment(payment_hash, payment_secret, payment_id, route, &self.keys_manager, best_block_height)
        }
 
-       fn send_payment_internal(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, keysend_preimage: Option<PaymentPreimage>, payment_id: PaymentId, recv_value_msat: Option<u64>, onion_session_privs: Vec<[u8; 32]>) -> Result<(), PaymentSendFailure> {
-               if route.paths.len() < 1 {
-                       return Err(PaymentSendFailure::ParameterError(APIError::InvalidRoute{err: "There must be at least one path to send over"}));
-               }
-               if payment_secret.is_none() && route.paths.len() > 1 {
-                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError{err: "Payment secret is required for multi-path payments".to_string()}));
-               }
-               let mut total_value = 0;
-               let our_node_id = self.get_our_node_id();
-               let mut path_errs = Vec::with_capacity(route.paths.len());
-               'path_check: for path in route.paths.iter() {
-                       if path.len() < 1 || path.len() > 20 {
-                               path_errs.push(Err(APIError::InvalidRoute{err: "Path didn't go anywhere/had bogus size"}));
-                               continue 'path_check;
-                       }
-                       for (idx, hop) in path.iter().enumerate() {
-                               if idx != path.len() - 1 && hop.pubkey == our_node_id {
-                                       path_errs.push(Err(APIError::InvalidRoute{err: "Path went through us but wasn't a simple rebalance loop to us"}));
-                                       continue 'path_check;
-                               }
-                       }
-                       total_value += path.last().unwrap().fee_msat;
-                       path_errs.push(Ok(()));
-               }
-               if path_errs.iter().any(|e| e.is_err()) {
-                       return Err(PaymentSendFailure::PathParameterError(path_errs));
-               }
-               if let Some(amt_msat) = recv_value_msat {
-                       debug_assert!(amt_msat >= total_value);
-                       total_value = amt_msat;
-               }
-
-               let cur_height = self.best_block.read().unwrap().height() + 1;
-               let mut results = Vec::new();
-               debug_assert_eq!(route.paths.len(), onion_session_privs.len());
-               for (path, session_priv) in route.paths.iter().zip(onion_session_privs.into_iter()) {
-                       let mut path_res = self.send_payment_along_path(&path, &route.payment_params, &payment_hash, payment_secret, total_value, cur_height, payment_id, &keysend_preimage, session_priv);
-                       match path_res {
-                               Ok(_) => {},
-                               Err(APIError::MonitorUpdateInProgress) => {
-                                       // While a MonitorUpdateInProgress is an Err(_), the payment is still
-                                       // considered "in flight" and we shouldn't remove it from the
-                                       // PendingOutboundPayment set.
-                               },
-                               Err(_) => {
-                                       let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap();
-                                       if let Some(payment) = pending_outbounds.get_mut(&payment_id) {
-                                               let removed = payment.remove(&session_priv, Some(path));
-                                               debug_assert!(removed, "This can't happen as the payment has an entry for this path added by callers");
-                                       } else {
-                                               debug_assert!(false, "This can't happen as the payment was added by callers");
-                                               path_res = Err(APIError::APIMisuseError { err: "Internal error: payment disappeared during processing. Please report this bug!".to_owned() });
-                                       }
-                               }
-                       }
-                       results.push(path_res);
-               }
-               let mut has_ok = false;
-               let mut has_err = false;
-               let mut pending_amt_unsent = 0;
-               let mut max_unsent_cltv_delta = 0;
-               for (res, path) in results.iter().zip(route.paths.iter()) {
-                       if res.is_ok() { has_ok = true; }
-                       if res.is_err() { has_err = true; }
-                       if let &Err(APIError::MonitorUpdateInProgress) = res {
-                               // MonitorUpdateInProgress is inherently unsafe to retry, so we call it a
-                               // PartialFailure.
-                               has_err = true;
-                               has_ok = true;
-                       } else if res.is_err() {
-                               pending_amt_unsent += path.last().unwrap().fee_msat;
-                               max_unsent_cltv_delta = cmp::max(max_unsent_cltv_delta, path.last().unwrap().cltv_expiry_delta);
-                       }
-               }
-               if has_err && has_ok {
-                       Err(PaymentSendFailure::PartialFailure {
-                               results,
-                               payment_id,
-                               failed_paths_retry: if pending_amt_unsent != 0 {
-                                       if let Some(payment_params) = &route.payment_params {
-                                               Some(RouteParameters {
-                                                       payment_params: payment_params.clone(),
-                                                       final_value_msat: pending_amt_unsent,
-                                                       final_cltv_expiry_delta: max_unsent_cltv_delta,
-                                               })
-                                       } else { None }
-                               } else { None },
-                       })
-               } else if has_err {
-                       // If we failed to send any paths, we should remove the new PaymentId from the
-                       // `pending_outbound_payments` map, as the user isn't expected to `abandon_payment`.
-                       let removed = self.pending_outbound_payments.lock().unwrap().remove(&payment_id).is_some();
-                       debug_assert!(removed, "We should always have a pending payment to remove here");
-                       Err(PaymentSendFailure::AllFailedResendSafe(results.drain(..).map(|r| r.unwrap_err()).collect()))
-               } else {
-                       Ok(())
-               }
-       }
 
        /// Retries a payment along the given [`Route`].
        ///
@@ -2514,64 +2401,10 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        /// [`send_payment`]: [`ChannelManager::send_payment`]
        /// [`abandon_payment`]: [`ChannelManager::abandon_payment`]
        pub fn retry_payment(&self, route: &Route, payment_id: PaymentId) -> Result<(), PaymentSendFailure> {
-               const RETRY_OVERFLOW_PERCENTAGE: u64 = 10;
-               for path in route.paths.iter() {
-                       if path.len() == 0 {
-                               return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
-                                       err: "length-0 path in route".to_string()
-                               }))
-                       }
-               }
-
-               let mut onion_session_privs = Vec::with_capacity(route.paths.len());
-               for _ in 0..route.paths.len() {
-                       onion_session_privs.push(self.keys_manager.get_secure_random_bytes());
-               }
-
-               let (total_msat, payment_hash, payment_secret) = {
-                       let mut outbounds = self.pending_outbound_payments.lock().unwrap();
-                       match outbounds.get_mut(&payment_id) {
-                               Some(payment) => {
-                                       let res = match payment {
-                                               PendingOutboundPayment::Retryable {
-                                                       total_msat, payment_hash, payment_secret, pending_amt_msat, ..
-                                               } => {
-                                                       let retry_amt_msat: u64 = route.paths.iter().map(|path| path.last().unwrap().fee_msat).sum();
-                                                       if retry_amt_msat + *pending_amt_msat > *total_msat * (100 + RETRY_OVERFLOW_PERCENTAGE) / 100 {
-                                                               return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
-                                                                       err: format!("retry_amt_msat of {} will put pending_amt_msat (currently: {}) more than 10% over total_payment_amt_msat of {}", retry_amt_msat, pending_amt_msat, total_msat).to_string()
-                                                               }))
-                                                       }
-                                                       (*total_msat, *payment_hash, *payment_secret)
-                                               },
-                                               PendingOutboundPayment::Legacy { .. } => {
-                                                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
-                                                               err: "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102".to_string()
-                                                       }))
-                                               },
-                                               PendingOutboundPayment::Fulfilled { .. } => {
-                                                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
-                                                               err: "Payment already completed".to_owned()
-                                                       }));
-                                               },
-                                               PendingOutboundPayment::Abandoned { .. } => {
-                                                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
-                                                               err: "Payment already abandoned (with some HTLCs still pending)".to_owned()
-                                                       }));
-                                               },
-                                       };
-                                       for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) {
-                                               assert!(payment.insert(*session_priv_bytes, path));
-                                       }
-                                       res
-                               },
-                               None =>
-                                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
-                                               err: format!("Payment with ID {} not found", log_bytes!(payment_id.0)),
-                                       })),
-                       }
-               };
-               self.send_payment_internal(route, payment_hash, &payment_secret, None, payment_id, Some(total_msat), onion_session_privs)
+               let best_block_height = self.best_block.read().unwrap().height();
+               self.pending_outbound_payments.retry_payment(route, payment_id, &self.keys_manager, best_block_height,
+                       |path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv|
+                       self.send_payment_along_path(path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv))
        }
 
        /// Signals that no further retries for the given payment will occur.
@@ -2596,18 +2429,8 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        /// [`Event::PaymentSent`]: events::Event::PaymentSent
        pub fn abandon_payment(&self, payment_id: PaymentId) {
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
-
-               let mut outbounds = self.pending_outbound_payments.lock().unwrap();
-               if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(payment_id) {
-                       if let Ok(()) = payment.get_mut().mark_abandoned() {
-                               if payment.get().remaining_parts() == 0 {
-                                       self.pending_events.lock().unwrap().push(events::Event::PaymentFailed {
-                                               payment_id,
-                                               payment_hash: payment.get().payment_hash().expect("PendingOutboundPayments::RetriesExceeded always has a payment hash set"),
-                                       });
-                                       payment.remove();
-                               }
-                       }
+               if let Some(payment_failed_ev) = self.pending_outbound_payments.abandon_payment(payment_id) {
+                       self.pending_events.lock().unwrap().push(payment_failed_ev);
                }
        }
 
@@ -2627,55 +2450,27 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        ///
        /// [`send_payment`]: Self::send_payment
        pub fn send_spontaneous_payment(&self, route: &Route, payment_preimage: Option<PaymentPreimage>, payment_id: PaymentId) -> Result<PaymentHash, PaymentSendFailure> {
-               let preimage = match payment_preimage {
-                       Some(p) => p,
-                       None => PaymentPreimage(self.keys_manager.get_secure_random_bytes()),
-               };
-               let payment_hash = PaymentHash(Sha256::hash(&preimage.0).into_inner());
-               let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, &route)?;
-
-               match self.send_payment_internal(route, payment_hash, &None, Some(preimage), payment_id, None, onion_session_privs) {
-                       Ok(()) => Ok(payment_hash),
-                       Err(e) => Err(e)
-               }
+               let best_block_height = self.best_block.read().unwrap().height();
+               self.pending_outbound_payments.send_spontaneous_payment(route, payment_preimage, payment_id, &self.keys_manager, best_block_height,
+                       |path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv|
+                       self.send_payment_along_path(path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv))
        }
 
        /// Send a payment that is probing the given route for liquidity. We calculate the
        /// [`PaymentHash`] of probes based on a static secret and a random [`PaymentId`], which allows
        /// us to easily discern them from real payments.
        pub fn send_probe(&self, hops: Vec<RouteHop>) -> Result<(PaymentHash, PaymentId), PaymentSendFailure> {
-               let payment_id = PaymentId(self.keys_manager.get_secure_random_bytes());
-
-               let payment_hash = self.probing_cookie_from_id(&payment_id);
-
-               if hops.len() < 2 {
-                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
-                               err: "No need probing a path with less than two hops".to_string()
-                       }))
-               }
-
-               let route = Route { paths: vec![hops], payment_params: None };
-               let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, &route)?;
-
-               match self.send_payment_internal(&route, payment_hash, &None, None, payment_id, None, onion_session_privs) {
-                       Ok(()) => Ok((payment_hash, payment_id)),
-                       Err(e) => Err(e)
-               }
+               let best_block_height = self.best_block.read().unwrap().height();
+               self.pending_outbound_payments.send_probe(hops, self.probing_cookie_secret, &self.keys_manager, best_block_height,
+                       |path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv|
+                       self.send_payment_along_path(path, payment_params, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv))
        }
 
        /// Returns whether a payment with the given [`PaymentHash`] and [`PaymentId`] is, in fact, a
        /// payment probe.
+       #[cfg(test)]
        pub(crate) fn payment_is_probe(&self, payment_hash: &PaymentHash, payment_id: &PaymentId) -> bool {
-               let target_payment_hash = self.probing_cookie_from_id(payment_id);
-               target_payment_hash == *payment_hash
-       }
-
-       /// Returns the 'probing cookie' for the given [`PaymentId`].
-       fn probing_cookie_from_id(&self, payment_id: &PaymentId) -> PaymentHash {
-               let mut preimage = [0u8; 64];
-               preimage[..32].copy_from_slice(&self.probing_cookie_secret);
-               preimage[32..].copy_from_slice(&payment_id.0);
-               PaymentHash(Sha256::hash(&preimage).into_inner())
+               outbound_payment::payment_is_probe(payment_hash, payment_id, self.probing_cookie_secret)
        }
 
        /// Handles the generation of a funding transaction, optionally (for tests) with a function
@@ -3450,45 +3245,6 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                });
        }
 
-       fn remove_stale_resolved_payments(&self) {
-               // If an outbound payment was completed, and no pending HTLCs remain, we should remove it
-               // from the map. However, if we did that immediately when the last payment HTLC is claimed,
-               // this could race the user making a duplicate send_payment call and our idempotency
-               // guarantees would be violated. Instead, we wait a few timer ticks to do the actual
-               // removal. This should be more than sufficient to ensure the idempotency of any
-               // `send_payment` calls that were made at the same time the `PaymentSent` event was being
-               // processed.
-               let mut pending_outbound_payments = self.pending_outbound_payments.lock().unwrap();
-               let pending_events = self.pending_events.lock().unwrap();
-               pending_outbound_payments.retain(|payment_id, payment| {
-                       if let PendingOutboundPayment::Fulfilled { session_privs, timer_ticks_without_htlcs, .. } = payment {
-                               let mut no_remaining_entries = session_privs.is_empty();
-                               if no_remaining_entries {
-                                       for ev in pending_events.iter() {
-                                               match ev {
-                                                       events::Event::PaymentSent { payment_id: Some(ev_payment_id), .. } |
-                                                       events::Event::PaymentPathSuccessful { payment_id: ev_payment_id, .. } |
-                                                       events::Event::PaymentPathFailed { payment_id: Some(ev_payment_id), .. } => {
-                                                               if payment_id == ev_payment_id {
-                                                                       no_remaining_entries = false;
-                                                                       break;
-                                                               }
-                                                       },
-                                                       _ => {},
-                                               }
-                                       }
-                               }
-                               if no_remaining_entries {
-                                       *timer_ticks_without_htlcs += 1;
-                                       *timer_ticks_without_htlcs <= IDEMPOTENCY_TIMEOUT_TICKS
-                               } else {
-                                       *timer_ticks_without_htlcs = 0;
-                                       true
-                               }
-                       } else { true }
-               });
-       }
-
        /// Performs actions which should happen on startup and roughly once per minute thereafter.
        ///
        /// This currently includes:
@@ -3589,7 +3345,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                let _ = handle_error!(self, err, counterparty_node_id);
                        }
 
-                       self.remove_stale_resolved_payments();
+                       self.pending_outbound_payments.remove_stale_resolved_payments(&self.pending_events);
 
                        // Technically we don't need to do this here, but if we have holding cell entries in a
                        // channel that need freeing, it's better to do that here and block a background task
@@ -3722,91 +3478,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                // being fully configured. See the docs for `ChannelManagerReadArgs` for more.
                match source {
                        HTLCSource::OutboundRoute { ref path, ref session_priv, ref payment_id, ref payment_params, .. } => {
-                               let mut session_priv_bytes = [0; 32];
-                               session_priv_bytes.copy_from_slice(&session_priv[..]);
-                               let mut outbounds = self.pending_outbound_payments.lock().unwrap();
-                               let mut all_paths_failed = false;
-                               let mut full_failure_ev = None;
-                               if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(*payment_id) {
-                                       if !payment.get_mut().remove(&session_priv_bytes, Some(&path)) {
-                                               log_trace!(self.logger, "Received duplicative fail for HTLC with payment_hash {}", log_bytes!(payment_hash.0));
-                                               return;
-                                       }
-                                       if payment.get().is_fulfilled() {
-                                               log_trace!(self.logger, "Received failure of HTLC with payment_hash {} after payment completion", log_bytes!(payment_hash.0));
-                                               return;
-                                       }
-                                       if payment.get().remaining_parts() == 0 {
-                                               all_paths_failed = true;
-                                               if payment.get().abandoned() {
-                                                       full_failure_ev = Some(events::Event::PaymentFailed {
-                                                               payment_id: *payment_id,
-                                                               payment_hash: payment.get().payment_hash().expect("PendingOutboundPayments::RetriesExceeded always has a payment hash set"),
-                                                       });
-                                                       payment.remove();
-                                               }
-                                       }
-                               } else {
-                                       log_trace!(self.logger, "Received duplicative fail for HTLC with payment_hash {}", log_bytes!(payment_hash.0));
-                                       return;
-                               }
-                               let mut retry = if let Some(payment_params_data) = payment_params {
-                                       let path_last_hop = path.last().expect("Outbound payments must have had a valid path");
-                                       Some(RouteParameters {
-                                               payment_params: payment_params_data.clone(),
-                                               final_value_msat: path_last_hop.fee_msat,
-                                               final_cltv_expiry_delta: path_last_hop.cltv_expiry_delta,
-                                       })
-                               } else { None };
-                               log_trace!(self.logger, "Failing outbound payment HTLC with payment_hash {}", log_bytes!(payment_hash.0));
-
-                               let path_failure = {
-#[cfg(test)]
-                                       let (network_update, short_channel_id, payment_retryable, onion_error_code, onion_error_data) = onion_error.decode_onion_failure(&self.secp_ctx, &self.logger, &source);
-#[cfg(not(test))]
-                                       let (network_update, short_channel_id, payment_retryable, _, _) = onion_error.decode_onion_failure(&self.secp_ctx, &self.logger, &source);
-
-                                       if self.payment_is_probe(payment_hash, &payment_id) {
-                                               if !payment_retryable {
-                                                       events::Event::ProbeSuccessful {
-                                                               payment_id: *payment_id,
-                                                               payment_hash: payment_hash.clone(),
-                                                               path: path.clone(),
-                                                       }
-                                               } else {
-                                                       events::Event::ProbeFailed {
-                                                               payment_id: *payment_id,
-                                                               payment_hash: payment_hash.clone(),
-                                                               path: path.clone(),
-                                                               short_channel_id,
-                                                       }
-                                               }
-                                       } else {
-                                               // TODO: If we decided to blame ourselves (or one of our channels) in
-                                               // process_onion_failure we should close that channel as it implies our
-                                               // next-hop is needlessly blaming us!
-                                               if let Some(scid) = short_channel_id {
-                                                       retry.as_mut().map(|r| r.payment_params.previously_failed_channels.push(scid));
-                                               }
-                                               events::Event::PaymentPathFailed {
-                                                       payment_id: Some(*payment_id),
-                                                       payment_hash: payment_hash.clone(),
-                                                       payment_failed_permanently: !payment_retryable,
-                                                       network_update,
-                                                       all_paths_failed,
-                                                       path: path.clone(),
-                                                       short_channel_id,
-                                                       retry,
-                                                       #[cfg(test)]
-                                                       error_code: onion_error_code,
-                                                       #[cfg(test)]
-                                                       error_data: onion_error_data
-                                               }
-                                       }
-                               };
-                               let mut pending_events = self.pending_events.lock().unwrap();
-                               pending_events.push(path_failure);
-                               if let Some(ev) = full_failure_ev { pending_events.push(ev); }
+                               self.pending_outbound_payments.fail_htlc(source, payment_hash, onion_error, path, session_priv, payment_id, payment_params, self.probing_cookie_secret, &self.secp_ctx, &self.pending_events, &self.logger);
                        },
                        HTLCSource::PreviousHopData(HTLCPreviousHopData { ref short_channel_id, ref htlc_id, ref incoming_packet_shared_secret, ref phantom_shared_secret, ref outpoint }) => {
                                log_trace!(self.logger, "Failing HTLC with payment_hash {} backwards from us with {:?}", log_bytes!(payment_hash.0), onion_error);
@@ -4089,73 +3761,15 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                }
        }
 
-       fn finalize_claims(&self, mut sources: Vec<HTLCSource>) {
-               let mut outbounds = self.pending_outbound_payments.lock().unwrap();
-               let mut pending_events = self.pending_events.lock().unwrap();
-               for source in sources.drain(..) {
-                       if let HTLCSource::OutboundRoute { session_priv, payment_id, path, .. } = source {
-                               let mut session_priv_bytes = [0; 32];
-                               session_priv_bytes.copy_from_slice(&session_priv[..]);
-                               if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(payment_id) {
-                                       assert!(payment.get().is_fulfilled());
-                                       if payment.get_mut().remove(&session_priv_bytes, None) {
-                                               pending_events.push(
-                                                       events::Event::PaymentPathSuccessful {
-                                                               payment_id,
-                                                               payment_hash: payment.get().payment_hash(),
-                                                               path,
-                                                       }
-                                               );
-                                       }
-                               }
-                       }
-               }
+       fn finalize_claims(&self, sources: Vec<HTLCSource>) {
+               self.pending_outbound_payments.finalize_claims(sources, &self.pending_events);
        }
 
        fn claim_funds_internal(&self, channel_state_lock: MutexGuard<ChannelHolder<<K::Target as KeysInterface>::Signer>>, source: HTLCSource, payment_preimage: PaymentPreimage, forwarded_htlc_value_msat: Option<u64>, from_onchain: bool, next_channel_id: [u8; 32]) {
                match source {
                        HTLCSource::OutboundRoute { session_priv, payment_id, path, .. } => {
                                mem::drop(channel_state_lock);
-                               let mut session_priv_bytes = [0; 32];
-                               session_priv_bytes.copy_from_slice(&session_priv[..]);
-                               let mut outbounds = self.pending_outbound_payments.lock().unwrap();
-                               if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(payment_id) {
-                                       let mut pending_events = self.pending_events.lock().unwrap();
-                                       if !payment.get().is_fulfilled() {
-                                               let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0).into_inner());
-                                               let fee_paid_msat = payment.get().get_pending_fee_msat();
-                                               pending_events.push(
-                                                       events::Event::PaymentSent {
-                                                               payment_id: Some(payment_id),
-                                                               payment_preimage,
-                                                               payment_hash,
-                                                               fee_paid_msat,
-                                                       }
-                                               );
-                                               payment.get_mut().mark_fulfilled();
-                                       }
-
-                                       if from_onchain {
-                                               // We currently immediately remove HTLCs which were fulfilled on-chain.
-                                               // This could potentially lead to removing a pending payment too early,
-                                               // with a reorg of one block causing us to re-add the fulfilled payment on
-                                               // restart.
-                                               // TODO: We should have a second monitor event that informs us of payments
-                                               // irrevocably fulfilled.
-                                               if payment.get_mut().remove(&session_priv_bytes, Some(&path)) {
-                                                       let payment_hash = Some(PaymentHash(Sha256::hash(&payment_preimage.0).into_inner()));
-                                                       pending_events.push(
-                                                               events::Event::PaymentPathSuccessful {
-                                                                       payment_id,
-                                                                       payment_hash,
-                                                                       path,
-                                                               }
-                                                       );
-                                               }
-                                       }
-                               } else {
-                                       log_trace!(self.logger, "Received duplicative fulfill for HTLC with payment_preimage {}", log_bytes!(payment_preimage.0));
-                               }
+                               self.pending_outbound_payments.claim_htlc(payment_id, payment_preimage, session_priv, path, from_onchain, &self.pending_events, &self.logger);
                        },
                        HTLCSource::PreviousHopData(hop_data) => {
                                let prev_outpoint = hop_data.outpoint;
@@ -5608,12 +5222,12 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
        #[cfg(test)]
        pub fn has_pending_payments(&self) -> bool {
-               !self.pending_outbound_payments.lock().unwrap().is_empty()
+               self.pending_outbound_payments.has_pending_payments()
        }
 
        #[cfg(test)]
        pub fn clear_pending_payments(&self) {
-               self.pending_outbound_payments.lock().unwrap().clear()
+               self.pending_outbound_payments.clear_pending_payments()
        }
 
        /// Processes any events asynchronously in the order they were generated since the last call
@@ -6787,7 +6401,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Writeable for ChannelMana
 
                let pending_inbound_payments = self.pending_inbound_payments.lock().unwrap();
                let claimable_payments = self.claimable_payments.lock().unwrap();
-               let pending_outbound_payments = self.pending_outbound_payments.lock().unwrap();
+               let pending_outbound_payments = self.pending_outbound_payments.pending_outbound_payments.lock().unwrap();
 
                let mut htlc_purposes: Vec<&events::PaymentPurpose> = Vec::new();
                (claimable_payments.claimable_htlcs.len() as u64).write(writer)?;
@@ -7491,7 +7105,7 @@ impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                        }),
                        inbound_payment_key: expanded_inbound_key,
                        pending_inbound_payments: Mutex::new(pending_inbound_payments),
-                       pending_outbound_payments: Mutex::new(pending_outbound_payments.unwrap()),
+                       pending_outbound_payments: OutboundPayments { pending_outbound_payments: Mutex::new(pending_outbound_payments.unwrap()) },
                        pending_intercepted_htlcs: Mutex::new(pending_intercepted_htlcs.unwrap()),
 
                        forward_htlcs: Mutex::new(forward_htlcs),
@@ -7649,7 +7263,7 @@ mod tests {
                // Use the utility function send_payment_along_path to send the payment with MPP data which
                // indicates there are more HTLCs coming.
                let cur_height = CHAN_CONFIRM_DEPTH + 1; // route_payment calls send_payment, which adds 1 to the current height. So we do the same here to match.
-               let session_privs = nodes[0].node.add_new_pending_payment(our_payment_hash, Some(payment_secret), payment_id, &mpp_route).unwrap();
+               let session_privs = nodes[0].node.test_add_new_pending_payment(our_payment_hash, Some(payment_secret), payment_id, &mpp_route).unwrap();
                nodes[0].node.send_payment_along_path(&mpp_route.paths[0], &route.payment_params, &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None, session_privs[0]).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -7878,8 +7492,8 @@ mod tests {
 
                let test_preimage = PaymentPreimage([42; 32]);
                let mismatch_payment_hash = PaymentHash([43; 32]);
-               let session_privs = nodes[0].node.add_new_pending_payment(mismatch_payment_hash, None, PaymentId(mismatch_payment_hash.0), &route).unwrap();
-               nodes[0].node.send_payment_internal(&route, mismatch_payment_hash, &None, Some(test_preimage), PaymentId(mismatch_payment_hash.0), None, session_privs).unwrap();
+               let session_privs = nodes[0].node.test_add_new_pending_payment(mismatch_payment_hash, None, PaymentId(mismatch_payment_hash.0), &route).unwrap();
+               nodes[0].node.test_send_payment_internal(&route, mismatch_payment_hash, &None, Some(test_preimage), PaymentId(mismatch_payment_hash.0), None, session_privs).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -7924,8 +7538,8 @@ mod tests {
                let test_preimage = PaymentPreimage([42; 32]);
                let test_secret = PaymentSecret([43; 32]);
                let payment_hash = PaymentHash(Sha256::hash(&test_preimage.0).into_inner());
-               let session_privs = nodes[0].node.add_new_pending_payment(payment_hash, Some(test_secret), PaymentId(payment_hash.0), &route).unwrap();
-               nodes[0].node.send_payment_internal(&route, payment_hash, &Some(test_secret), Some(test_preimage), PaymentId(payment_hash.0), None, session_privs).unwrap();
+               let session_privs = nodes[0].node.test_add_new_pending_payment(payment_hash, Some(test_secret), PaymentId(payment_hash.0), &route).unwrap();
+               nodes[0].node.test_send_payment_internal(&route, payment_hash, &Some(test_secret), Some(test_preimage), PaymentId(payment_hash.0), None, session_privs).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
index d6770d5a127f473ab5a1d7e92c17145d9fa78c81..68978c117c5d0b062c5011b1e9f1487ced5c8c9e 100644 (file)
@@ -9,12 +9,24 @@
 
 //! Utilities to send payments and manage outbound payment information.
 
-use crate::ln::{PaymentHash, PaymentSecret};
-use crate::ln::channelmanager::PaymentId;
+use bitcoin::hashes::Hash;
+use bitcoin::hashes::sha256::Hash as Sha256;
+use bitcoin::secp256k1::{self, Secp256k1, SecretKey};
+
+use crate::chain::keysinterface::{KeysInterface, Recipient};
+use crate::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
+use crate::ln::channelmanager::{HTLCSource, IDEMPOTENCY_TIMEOUT_TICKS, PaymentId};
 use crate::ln::msgs::DecodeError;
-use crate::routing::router::{RouteHop, RouteParameters, RoutePath};
+use crate::ln::onion_utils::HTLCFailReason;
+use crate::routing::router::{PaymentParameters, Route, RouteHop, RouteParameters, RoutePath};
 use crate::util::errors::APIError;
+use crate::util::events;
+use crate::util::logger::Logger;
+
+use core::cmp;
+use core::ops::Deref;
 use crate::prelude::*;
+use crate::sync::Mutex;
 
 /// Stores the session_priv for each part of a payment that is still pending. For versions 0.0.102
 /// and later, also stores information for retrying the payment.
@@ -248,6 +260,549 @@ pub enum PaymentSendFailure {
        },
 }
 
+pub(super) struct OutboundPayments {
+       pub(super) pending_outbound_payments: Mutex<HashMap<PaymentId, PendingOutboundPayment>>,
+}
+
+impl OutboundPayments {
+       pub(super) fn new() -> Self {
+               Self {
+                       pending_outbound_payments: Mutex::new(HashMap::new())
+               }
+       }
+
+       pub(super) fn send_payment<K: Deref, F>(&self, route: &Route, payment_hash: PaymentHash,
+               payment_secret: &Option<PaymentSecret>, payment_id: PaymentId, keys_manager: &K,
+               best_block_height: u32, send_payment_along_path: F)
+       -> Result<(), PaymentSendFailure>
+       where
+               K::Target: KeysInterface,
+               F: Fn(&Vec<RouteHop>, &Option<PaymentParameters>, &PaymentHash, &Option<PaymentSecret>, u64,
+                  u32, PaymentId, &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
+       {
+               let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret, payment_id, route, keys_manager, best_block_height)?;
+               self.send_payment_internal(route, payment_hash, payment_secret, None, payment_id, None, onion_session_privs, keys_manager, best_block_height, send_payment_along_path)
+       }
+
+       pub(super) fn send_spontaneous_payment<K: Deref, F>(&self, route: &Route,
+               payment_preimage: Option<PaymentPreimage>, payment_id: PaymentId, keys_manager: &K,
+               best_block_height: u32, send_payment_along_path: F)
+       -> Result<PaymentHash, PaymentSendFailure>
+       where
+               K::Target: KeysInterface,
+               F: Fn(&Vec<RouteHop>, &Option<PaymentParameters>, &PaymentHash, &Option<PaymentSecret>, u64,
+                  u32, PaymentId, &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
+       {
+               let preimage = match payment_preimage {
+                       Some(p) => p,
+                       None => PaymentPreimage(keys_manager.get_secure_random_bytes()),
+               };
+               let payment_hash = PaymentHash(Sha256::hash(&preimage.0).into_inner());
+               let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, &route, keys_manager, best_block_height)?;
+
+               match self.send_payment_internal(route, payment_hash, &None, Some(preimage), payment_id, None, onion_session_privs, keys_manager, best_block_height, send_payment_along_path) {
+                       Ok(()) => Ok(payment_hash),
+                       Err(e) => Err(e)
+               }
+       }
+
+       pub(super) fn retry_payment<K: Deref, F>(&self, route: &Route, payment_id: PaymentId,
+               keys_manager: &K, best_block_height: u32, send_payment_along_path: F)
+       -> Result<(), PaymentSendFailure>
+       where
+               K::Target: KeysInterface,
+               F: Fn(&Vec<RouteHop>, &Option<PaymentParameters>, &PaymentHash, &Option<PaymentSecret>, u64,
+                  u32, PaymentId, &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
+       {
+               const RETRY_OVERFLOW_PERCENTAGE: u64 = 10;
+               for path in route.paths.iter() {
+                       if path.len() == 0 {
+                               return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
+                                       err: "length-0 path in route".to_string()
+                               }))
+                       }
+               }
+
+               let mut onion_session_privs = Vec::with_capacity(route.paths.len());
+               for _ in 0..route.paths.len() {
+                       onion_session_privs.push(keys_manager.get_secure_random_bytes());
+               }
+
+               let (total_msat, payment_hash, payment_secret) = {
+                       let mut outbounds = self.pending_outbound_payments.lock().unwrap();
+                       match outbounds.get_mut(&payment_id) {
+                               Some(payment) => {
+                                       let res = match payment {
+                                               PendingOutboundPayment::Retryable {
+                                                       total_msat, payment_hash, payment_secret, pending_amt_msat, ..
+                                               } => {
+                                                       let retry_amt_msat: u64 = route.paths.iter().map(|path| path.last().unwrap().fee_msat).sum();
+                                                       if retry_amt_msat + *pending_amt_msat > *total_msat * (100 + RETRY_OVERFLOW_PERCENTAGE) / 100 {
+                                                               return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
+                                                                       err: format!("retry_amt_msat of {} will put pending_amt_msat (currently: {}) more than 10% over total_payment_amt_msat of {}", retry_amt_msat, pending_amt_msat, total_msat).to_string()
+                                                               }))
+                                                       }
+                                                       (*total_msat, *payment_hash, *payment_secret)
+                                               },
+                                               PendingOutboundPayment::Legacy { .. } => {
+                                                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
+                                                               err: "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102".to_string()
+                                                       }))
+                                               },
+                                               PendingOutboundPayment::Fulfilled { .. } => {
+                                                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
+                                                               err: "Payment already completed".to_owned()
+                                                       }));
+                                               },
+                                               PendingOutboundPayment::Abandoned { .. } => {
+                                                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
+                                                               err: "Payment already abandoned (with some HTLCs still pending)".to_owned()
+                                                       }));
+                                               },
+                                       };
+                                       for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) {
+                                               assert!(payment.insert(*session_priv_bytes, path));
+                                       }
+                                       res
+                               },
+                               None =>
+                                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
+                                               err: format!("Payment with ID {} not found", log_bytes!(payment_id.0)),
+                                       })),
+                       }
+               };
+               self.send_payment_internal(route, payment_hash, &payment_secret, None, payment_id, Some(total_msat), onion_session_privs, keys_manager, best_block_height, send_payment_along_path)
+       }
+
+       pub(super) fn send_probe<K: Deref, F>(&self, hops: Vec<RouteHop>, probing_cookie_secret: [u8; 32],
+               keys_manager: &K, best_block_height: u32, send_payment_along_path: F)
+       -> Result<(PaymentHash, PaymentId), PaymentSendFailure>
+       where
+               K::Target: KeysInterface,
+               F: Fn(&Vec<RouteHop>, &Option<PaymentParameters>, &PaymentHash, &Option<PaymentSecret>, u64,
+                  u32, PaymentId, &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
+       {
+               let payment_id = PaymentId(keys_manager.get_secure_random_bytes());
+
+               let payment_hash = probing_cookie_from_id(&payment_id, probing_cookie_secret);
+
+               if hops.len() < 2 {
+                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
+                               err: "No need probing a path with less than two hops".to_string()
+                       }))
+               }
+
+               let route = Route { paths: vec![hops], payment_params: None };
+               let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, &route, keys_manager, best_block_height)?;
+
+               match self.send_payment_internal(&route, payment_hash, &None, None, payment_id, None, onion_session_privs, keys_manager, best_block_height, send_payment_along_path) {
+                       Ok(()) => Ok((payment_hash, payment_id)),
+                       Err(e) => Err(e)
+               }
+       }
+
+       pub(super) fn add_new_pending_payment<K: Deref>(&self, payment_hash: PaymentHash,
+               payment_secret: Option<PaymentSecret>, payment_id: PaymentId, route: &Route, keys_manager: &K,
+               best_block_height: u32)
+       -> Result<Vec<[u8; 32]>, PaymentSendFailure> where K::Target: KeysInterface {
+               let mut onion_session_privs = Vec::with_capacity(route.paths.len());
+               for _ in 0..route.paths.len() {
+                       onion_session_privs.push(keys_manager.get_secure_random_bytes());
+               }
+
+               let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap();
+               match pending_outbounds.entry(payment_id) {
+                       hash_map::Entry::Occupied(_) => Err(PaymentSendFailure::DuplicatePayment),
+                       hash_map::Entry::Vacant(entry) => {
+                               let payment = entry.insert(PendingOutboundPayment::Retryable {
+                                       session_privs: HashSet::new(),
+                                       pending_amt_msat: 0,
+                                       pending_fee_msat: Some(0),
+                                       payment_hash,
+                                       payment_secret,
+                                       starting_block_height: best_block_height,
+                                       total_msat: route.get_total_amount(),
+                               });
+
+                               for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) {
+                                       assert!(payment.insert(*session_priv_bytes, path));
+                               }
+
+                               Ok(onion_session_privs)
+                       },
+               }
+       }
+
+       fn send_payment_internal<K: Deref, F>
+               (&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>,
+               keysend_preimage: Option<PaymentPreimage>, payment_id: PaymentId, recv_value_msat: Option<u64>,
+               onion_session_privs: Vec<[u8; 32]>, keys_manager: &K, best_block_height: u32,
+               send_payment_along_path: F) -> Result<(), PaymentSendFailure>
+       where
+               K::Target: KeysInterface,
+               F: Fn(&Vec<RouteHop>, &Option<PaymentParameters>, &PaymentHash, &Option<PaymentSecret>, u64,
+                  u32, PaymentId, &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
+       {
+               if route.paths.len() < 1 {
+                       return Err(PaymentSendFailure::ParameterError(APIError::InvalidRoute{err: "There must be at least one path to send over"}));
+               }
+               if payment_secret.is_none() && route.paths.len() > 1 {
+                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError{err: "Payment secret is required for multi-path payments".to_string()}));
+               }
+               let mut total_value = 0;
+               let our_node_id = keys_manager.get_node_id(Recipient::Node).unwrap(); // TODO no unwrap
+               let mut path_errs = Vec::with_capacity(route.paths.len());
+               'path_check: for path in route.paths.iter() {
+                       if path.len() < 1 || path.len() > 20 {
+                               path_errs.push(Err(APIError::InvalidRoute{err: "Path didn't go anywhere/had bogus size"}));
+                               continue 'path_check;
+                       }
+                       for (idx, hop) in path.iter().enumerate() {
+                               if idx != path.len() - 1 && hop.pubkey == our_node_id {
+                                       path_errs.push(Err(APIError::InvalidRoute{err: "Path went through us but wasn't a simple rebalance loop to us"}));
+                                       continue 'path_check;
+                               }
+                       }
+                       total_value += path.last().unwrap().fee_msat;
+                       path_errs.push(Ok(()));
+               }
+               if path_errs.iter().any(|e| e.is_err()) {
+                       return Err(PaymentSendFailure::PathParameterError(path_errs));
+               }
+               if let Some(amt_msat) = recv_value_msat {
+                       debug_assert!(amt_msat >= total_value);
+                       total_value = amt_msat;
+               }
+
+               let cur_height = best_block_height + 1;
+               let mut results = Vec::new();
+               debug_assert_eq!(route.paths.len(), onion_session_privs.len());
+               for (path, session_priv) in route.paths.iter().zip(onion_session_privs.into_iter()) {
+                       let mut path_res = send_payment_along_path(&path, &route.payment_params, &payment_hash, payment_secret, total_value, cur_height, payment_id, &keysend_preimage, session_priv);
+                       match path_res {
+                               Ok(_) => {},
+                               Err(APIError::MonitorUpdateInProgress) => {
+                                       // While a MonitorUpdateInProgress is an Err(_), the payment is still
+                                       // considered "in flight" and we shouldn't remove it from the
+                                       // PendingOutboundPayment set.
+                               },
+                               Err(_) => {
+                                       let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap();
+                                       if let Some(payment) = pending_outbounds.get_mut(&payment_id) {
+                                               let removed = payment.remove(&session_priv, Some(path));
+                                               debug_assert!(removed, "This can't happen as the payment has an entry for this path added by callers");
+                                       } else {
+                                               debug_assert!(false, "This can't happen as the payment was added by callers");
+                                               path_res = Err(APIError::APIMisuseError { err: "Internal error: payment disappeared during processing. Please report this bug!".to_owned() });
+                                       }
+                               }
+                       }
+                       results.push(path_res);
+               }
+               let mut has_ok = false;
+               let mut has_err = false;
+               let mut pending_amt_unsent = 0;
+               let mut max_unsent_cltv_delta = 0;
+               for (res, path) in results.iter().zip(route.paths.iter()) {
+                       if res.is_ok() { has_ok = true; }
+                       if res.is_err() { has_err = true; }
+                       if let &Err(APIError::MonitorUpdateInProgress) = res {
+                               // MonitorUpdateInProgress is inherently unsafe to retry, so we call it a
+                               // PartialFailure.
+                               has_err = true;
+                               has_ok = true;
+                       } else if res.is_err() {
+                               pending_amt_unsent += path.last().unwrap().fee_msat;
+                               max_unsent_cltv_delta = cmp::max(max_unsent_cltv_delta, path.last().unwrap().cltv_expiry_delta);
+                       }
+               }
+               if has_err && has_ok {
+                       Err(PaymentSendFailure::PartialFailure {
+                               results,
+                               payment_id,
+                               failed_paths_retry: if pending_amt_unsent != 0 {
+                                       if let Some(payment_params) = &route.payment_params {
+                                               Some(RouteParameters {
+                                                       payment_params: payment_params.clone(),
+                                                       final_value_msat: pending_amt_unsent,
+                                                       final_cltv_expiry_delta: max_unsent_cltv_delta,
+                                               })
+                                       } else { None }
+                               } else { None },
+                       })
+               } else if has_err {
+                       // If we failed to send any paths, we should remove the new PaymentId from the
+                       // `pending_outbound_payments` map, as the user isn't expected to `abandon_payment`.
+                       let removed = self.pending_outbound_payments.lock().unwrap().remove(&payment_id).is_some();
+                       debug_assert!(removed, "We should always have a pending payment to remove here");
+                       Err(PaymentSendFailure::AllFailedResendSafe(results.drain(..).map(|r| r.unwrap_err()).collect()))
+               } else {
+                       Ok(())
+               }
+       }
+
+       #[cfg(test)]
+       pub(super) fn test_send_payment_internal<K: Deref, F>
+               (&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>,
+                keysend_preimage: Option<PaymentPreimage>, payment_id: PaymentId, recv_value_msat: Option<u64>,
+                onion_session_privs: Vec<[u8; 32]>, keys_manager: &K, best_block_height: u32,
+                send_payment_along_path: F) -> Result<(), PaymentSendFailure>
+       where
+               K::Target: KeysInterface,
+               F: Fn(&Vec<RouteHop>, &Option<PaymentParameters>, &PaymentHash, &Option<PaymentSecret>, u64,
+                  u32, PaymentId, &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
+       {
+               self.send_payment_internal(route, payment_hash, payment_secret, keysend_preimage, payment_id,
+                       recv_value_msat, onion_session_privs, keys_manager, best_block_height,
+                       send_payment_along_path)
+       }
+
+       pub(super) fn claim_htlc<L: Deref>(&self, payment_id: PaymentId,
+               payment_preimage: PaymentPreimage, session_priv: SecretKey, path: Vec<RouteHop>,
+               from_onchain: bool, pending_events: &Mutex<Vec<events::Event>>, logger: &L)
+        where L::Target: Logger {
+               let mut session_priv_bytes = [0; 32];
+               session_priv_bytes.copy_from_slice(&session_priv[..]);
+               let mut outbounds = self.pending_outbound_payments.lock().unwrap();
+               let mut pending_events = pending_events.lock().unwrap();
+               if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(payment_id) {
+                       if !payment.get().is_fulfilled() {
+                               let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0).into_inner());
+                               let fee_paid_msat = payment.get().get_pending_fee_msat();
+                               pending_events.push(
+                                       events::Event::PaymentSent {
+                                               payment_id: Some(payment_id),
+                                               payment_preimage,
+                                               payment_hash,
+                                               fee_paid_msat,
+                                       }
+                               );
+                               payment.get_mut().mark_fulfilled();
+                       }
+
+                       if from_onchain {
+                               // We currently immediately remove HTLCs which were fulfilled on-chain.
+                               // This could potentially lead to removing a pending payment too early,
+                               // with a reorg of one block causing us to re-add the fulfilled payment on
+                               // restart.
+                               // TODO: We should have a second monitor event that informs us of payments
+                               // irrevocably fulfilled.
+                               if payment.get_mut().remove(&session_priv_bytes, Some(&path)) {
+                                       let payment_hash = Some(PaymentHash(Sha256::hash(&payment_preimage.0).into_inner()));
+                                       pending_events.push(
+                                               events::Event::PaymentPathSuccessful {
+                                                       payment_id,
+                                                       payment_hash,
+                                                       path,
+                                               }
+                                       );
+                               }
+                       }
+               } else {
+                       log_trace!(logger, "Received duplicative fulfill for HTLC with payment_preimage {}", log_bytes!(payment_preimage.0));
+               }
+       }
+
+       pub(super) fn finalize_claims(&self, mut sources: Vec<HTLCSource>, pending_events: &Mutex<Vec<events::Event>>) {
+               let mut outbounds = self.pending_outbound_payments.lock().unwrap();
+               let mut pending_events = pending_events.lock().unwrap();
+               for source in sources.drain(..) {
+                       if let HTLCSource::OutboundRoute { session_priv, payment_id, path, .. } = source {
+                               let mut session_priv_bytes = [0; 32];
+                               session_priv_bytes.copy_from_slice(&session_priv[..]);
+                               if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(payment_id) {
+                                       assert!(payment.get().is_fulfilled());
+                                       if payment.get_mut().remove(&session_priv_bytes, None) {
+                                               pending_events.push(
+                                                       events::Event::PaymentPathSuccessful {
+                                                               payment_id,
+                                                               payment_hash: payment.get().payment_hash(),
+                                                               path,
+                                                       }
+                                               );
+                                       }
+                               }
+                       }
+               }
+       }
+
+       pub(super) fn remove_stale_resolved_payments(&self, pending_events: &Mutex<Vec<events::Event>>) {
+               // If an outbound payment was completed, and no pending HTLCs remain, we should remove it
+               // from the map. However, if we did that immediately when the last payment HTLC is claimed,
+               // this could race the user making a duplicate send_payment call and our idempotency
+               // guarantees would be violated. Instead, we wait a few timer ticks to do the actual
+               // removal. This should be more than sufficient to ensure the idempotency of any
+               // `send_payment` calls that were made at the same time the `PaymentSent` event was being
+               // processed.
+               let mut pending_outbound_payments = self.pending_outbound_payments.lock().unwrap();
+               let pending_events = pending_events.lock().unwrap();
+               pending_outbound_payments.retain(|payment_id, payment| {
+                       if let PendingOutboundPayment::Fulfilled { session_privs, timer_ticks_without_htlcs, .. } = payment {
+                               let mut no_remaining_entries = session_privs.is_empty();
+                               if no_remaining_entries {
+                                       for ev in pending_events.iter() {
+                                               match ev {
+                                                       events::Event::PaymentSent { payment_id: Some(ev_payment_id), .. } |
+                                                               events::Event::PaymentPathSuccessful { payment_id: ev_payment_id, .. } |
+                                                               events::Event::PaymentPathFailed { payment_id: Some(ev_payment_id), .. } => {
+                                                                       if payment_id == ev_payment_id {
+                                                                               no_remaining_entries = false;
+                                                                               break;
+                                                                       }
+                                                               },
+                                                       _ => {},
+                                               }
+                                       }
+                               }
+                               if no_remaining_entries {
+                                       *timer_ticks_without_htlcs += 1;
+                                       *timer_ticks_without_htlcs <= IDEMPOTENCY_TIMEOUT_TICKS
+                               } else {
+                                       *timer_ticks_without_htlcs = 0;
+                                       true
+                               }
+                       } else { true }
+               });
+       }
+
+       pub(super) fn fail_htlc<L: Deref>(&self, source: &HTLCSource, payment_hash: &PaymentHash,
+               onion_error: &HTLCFailReason, path: &Vec<RouteHop>, session_priv: &SecretKey,
+               payment_id: &PaymentId, payment_params: &Option<PaymentParameters>,
+               probing_cookie_secret: [u8; 32], secp_ctx: &Secp256k1<secp256k1::All>,
+               pending_events: &Mutex<Vec<events::Event>>, logger: &L)
+       where L::Target: Logger {
+               let mut session_priv_bytes = [0; 32];
+               session_priv_bytes.copy_from_slice(&session_priv[..]);
+               let mut outbounds = self.pending_outbound_payments.lock().unwrap();
+               let mut all_paths_failed = false;
+               let mut full_failure_ev = None;
+               if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(*payment_id) {
+                       if !payment.get_mut().remove(&session_priv_bytes, Some(&path)) {
+                               log_trace!(logger, "Received duplicative fail for HTLC with payment_hash {}", log_bytes!(payment_hash.0));
+                               return
+                       }
+                       if payment.get().is_fulfilled() {
+                               log_trace!(logger, "Received failure of HTLC with payment_hash {} after payment completion", log_bytes!(payment_hash.0));
+                               return
+                       }
+                       if payment.get().remaining_parts() == 0 {
+                               all_paths_failed = true;
+                               if payment.get().abandoned() {
+                                       full_failure_ev = Some(events::Event::PaymentFailed {
+                                               payment_id: *payment_id,
+                                               payment_hash: payment.get().payment_hash().expect("PendingOutboundPayments::RetriesExceeded always has a payment hash set"),
+                                       });
+                                       payment.remove();
+                               }
+                       }
+               } else {
+                       log_trace!(logger, "Received duplicative fail for HTLC with payment_hash {}", log_bytes!(payment_hash.0));
+                       return
+               }
+               let mut retry = if let Some(payment_params_data) = payment_params {
+                       let path_last_hop = path.last().expect("Outbound payments must have had a valid path");
+                       Some(RouteParameters {
+                               payment_params: payment_params_data.clone(),
+                               final_value_msat: path_last_hop.fee_msat,
+                               final_cltv_expiry_delta: path_last_hop.cltv_expiry_delta,
+                       })
+               } else { None };
+               log_trace!(logger, "Failing outbound payment HTLC with payment_hash {}", log_bytes!(payment_hash.0));
+
+               let path_failure = {
+       #[cfg(test)]
+                       let (network_update, short_channel_id, payment_retryable, onion_error_code, onion_error_data) = onion_error.decode_onion_failure(secp_ctx, logger, &source);
+       #[cfg(not(test))]
+                       let (network_update, short_channel_id, payment_retryable, _, _) = onion_error.decode_onion_failure(secp_ctx, logger, &source);
+
+                       if payment_is_probe(payment_hash, &payment_id, probing_cookie_secret) {
+                               if !payment_retryable {
+                                       events::Event::ProbeSuccessful {
+                                               payment_id: *payment_id,
+                                               payment_hash: payment_hash.clone(),
+                                               path: path.clone(),
+                                       }
+                               } else {
+                                       events::Event::ProbeFailed {
+                                               payment_id: *payment_id,
+                                               payment_hash: payment_hash.clone(),
+                                               path: path.clone(),
+                                               short_channel_id,
+                                       }
+                               }
+                       } else {
+                               // TODO: If we decided to blame ourselves (or one of our channels) in
+                               // process_onion_failure we should close that channel as it implies our
+                               // next-hop is needlessly blaming us!
+                               if let Some(scid) = short_channel_id {
+                                       retry.as_mut().map(|r| r.payment_params.previously_failed_channels.push(scid));
+                               }
+                               events::Event::PaymentPathFailed {
+                                       payment_id: Some(*payment_id),
+                                       payment_hash: payment_hash.clone(),
+                                       payment_failed_permanently: !payment_retryable,
+                                       network_update,
+                                       all_paths_failed,
+                                       path: path.clone(),
+                                       short_channel_id,
+                                       retry,
+                                       #[cfg(test)]
+                                       error_code: onion_error_code,
+                                       #[cfg(test)]
+                                       error_data: onion_error_data
+                               }
+                       }
+               };
+               let mut pending_events = pending_events.lock().unwrap();
+               pending_events.push(path_failure);
+               if let Some(ev) = full_failure_ev { pending_events.push(ev); }
+       }
+
+       pub(super) fn abandon_payment(&self, payment_id: PaymentId) -> Option<events::Event> {
+               let mut failed_ev = None;
+               let mut outbounds = self.pending_outbound_payments.lock().unwrap();
+               if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(payment_id) {
+                       if let Ok(()) = payment.get_mut().mark_abandoned() {
+                               if payment.get().remaining_parts() == 0 {
+                                       failed_ev = Some(events::Event::PaymentFailed {
+                                               payment_id,
+                                               payment_hash: payment.get().payment_hash().expect("PendingOutboundPayments::RetriesExceeded always has a payment hash set"),
+                                       });
+                                       payment.remove();
+                               }
+                       }
+               }
+               failed_ev
+       }
+
+       #[cfg(test)]
+       pub fn has_pending_payments(&self) -> bool {
+               !self.pending_outbound_payments.lock().unwrap().is_empty()
+       }
+
+       #[cfg(test)]
+       pub fn clear_pending_payments(&self) {
+               self.pending_outbound_payments.lock().unwrap().clear()
+       }
+}
+
+/// Returns whether a payment with the given [`PaymentHash`] and [`PaymentId`] is, in fact, a
+/// payment probe.
+pub(super) fn payment_is_probe(payment_hash: &PaymentHash, payment_id: &PaymentId,
+       probing_cookie_secret: [u8; 32]) -> bool
+{
+       let target_payment_hash = probing_cookie_from_id(payment_id, probing_cookie_secret);
+       target_payment_hash == *payment_hash
+}
+
+/// Returns the 'probing cookie' for the given [`PaymentId`].
+fn probing_cookie_from_id(payment_id: &PaymentId, probing_cookie_secret: [u8; 32]) -> PaymentHash {
+       let mut preimage = [0u8; 64];
+       preimage[..32].copy_from_slice(&probing_cookie_secret);
+       preimage[32..].copy_from_slice(&payment_id.0);
+       PaymentHash(Sha256::hash(&preimage).into_inner())
+}
+
 impl_writeable_tlv_based_enum_upgradable!(PendingOutboundPayment,
        (0, Legacy) => {
                (0, session_privs, required),