X-Git-Url: http://git.bitcoin.ninja/index.cgi?a=blobdiff_plain;f=lightning%2Fsrc%2Fln%2Foutbound_payment.rs;h=dd6227349f3f0968cd7595cd03fb9aa463716949;hb=ba1349982ba28657c9e2d03a5b02c3ecc054b5cc;hp=8867b2e96ae14c3e75bb960d5dd9fa97ed10d746;hpb=8ecd7c30c903b9e2db0c280110200f7c61565c11;p=rust-lightning diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index 8867b2e9..dd622734 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -14,22 +14,21 @@ use bitcoin::hashes::sha256::Hash as Sha256; use bitcoin::secp256k1::{self, Secp256k1, SecretKey}; use crate::chain::keysinterface::{EntropySource, NodeSigner, Recipient}; +use crate::events; use crate::ln::{PaymentHash, PaymentPreimage, PaymentSecret}; -use crate::ln::channelmanager::{ChannelDetails, HTLCSource, IDEMPOTENCY_TIMEOUT_TICKS, MIN_HTLC_RELAY_HOLDING_CELL_MILLIS, PaymentId}; -use crate::ln::msgs::DecodeError; +use crate::ln::channelmanager::{ChannelDetails, HTLCSource, IDEMPOTENCY_TIMEOUT_TICKS, PaymentId}; use crate::ln::onion_utils::HTLCFailReason; use crate::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, RouteParameters, RoutePath, Router}; use crate::util::errors::APIError; -use crate::util::events; use crate::util::logger::Logger; use crate::util::time::Time; #[cfg(all(not(feature = "no-std"), test))] use crate::util::time::tests::SinceEpoch; +use crate::util::ser::ReadableArgs; use core::cmp; use core::fmt::{self, Display, Formatter}; use core::ops::Deref; -use core::time::Duration; use crate::prelude::*; use crate::sync::Mutex; @@ -41,12 +40,13 @@ pub(crate) enum PendingOutboundPayment { session_privs: HashSet<[u8; 32]>, }, Retryable { - retry_strategy: Retry, + retry_strategy: Option, attempts: PaymentAttempts, - route_params: Option, + payment_params: Option, session_privs: HashSet<[u8; 32]>, payment_hash: PaymentHash, payment_secret: Option, + keysend_preimage: Option, pending_amt_msat: u64, /// Used to track the fee paid. Only present if the payment was serialized on 0.0.103+. pending_fee_msat: Option, @@ -63,13 +63,8 @@ pub(crate) enum PendingOutboundPayment { payment_hash: Option, timer_ticks_without_htlcs: u8, }, - /// When a payer gives up trying to retry a payment, they inform us, letting us generate a - /// `PaymentFailed` event when all HTLCs have irrevocably failed. This avoids a number of race - /// conditions in MPP-aware payment retriers (1), where the possibility of multiple - /// `PaymentPathFailed` events with `all_paths_failed` can be pending at once, confusing a - /// downstream event handler as to when a payment has actually failed. - /// - /// (1) + /// When we've decided to give up retrying a payment, we mark it as abandoned so we can eventually + /// generate a `PaymentFailed` event when all HTLCs have irrevocably failed. Abandoned { session_privs: HashSet<[u8; 32]>, payment_hash: PaymentHash, @@ -82,15 +77,31 @@ impl PendingOutboundPayment { attempts.count += 1; } } + fn is_auto_retryable_now(&self) -> bool { + match self { + PendingOutboundPayment::Retryable { + retry_strategy: Some(strategy), attempts, payment_params: Some(_), .. + } => { + strategy.is_retryable_now(&attempts) + }, + _ => false, + } + } fn is_retryable_now(&self) -> bool { - if let PendingOutboundPayment::Retryable { retry_strategy, attempts, .. } = self { - return retry_strategy.is_retryable_now(&attempts) + match self { + PendingOutboundPayment::Retryable { retry_strategy: None, .. } => { + // We're handling retries manually, we can always retry. + true + }, + PendingOutboundPayment::Retryable { retry_strategy: Some(strategy), attempts, .. } => { + strategy.is_retryable_now(&attempts) + }, + _ => false, } - false } pub fn insert_previously_failed_scid(&mut self, scid: u64) { - if let PendingOutboundPayment::Retryable { route_params: Some(params), .. } = self { - params.payment_params.previously_failed_channels.push(scid); + if let PendingOutboundPayment::Retryable { payment_params: Some(params), .. } = self { + params.previously_failed_channels.push(scid); } } pub(super) fn is_fulfilled(&self) -> bool { @@ -139,13 +150,13 @@ impl PendingOutboundPayment { let our_payment_hash; core::mem::swap(&mut session_privs, match self { PendingOutboundPayment::Legacy { .. } | - PendingOutboundPayment::Fulfilled { .. } => + PendingOutboundPayment::Fulfilled { .. } => return Err(()), - PendingOutboundPayment::Retryable { session_privs, payment_hash, .. } | - PendingOutboundPayment::Abandoned { session_privs, payment_hash, .. } => { - our_payment_hash = *payment_hash; - session_privs - }, + PendingOutboundPayment::Retryable { session_privs, payment_hash, .. } | + PendingOutboundPayment::Abandoned { session_privs, payment_hash, .. } => { + our_payment_hash = *payment_hash; + session_privs + }, }); *self = PendingOutboundPayment::Abandoned { session_privs, payment_hash: our_payment_hash }; Ok(()) @@ -212,12 +223,15 @@ impl PendingOutboundPayment { pub enum Retry { /// Max number of attempts to retry payment. /// - /// Note that this is the number of *path* failures, not full payment retries. For multi-path - /// payments, if this is less than the total number of paths, we will never even retry all of the - /// payment's paths. + /// Each attempt may be multiple HTLCs along multiple paths if the router decides to split up a + /// retry, and may retry multiple failed HTLCs at once if they failed around the same time and + /// were retried along a route from a single call to [`Router::find_route_with_id`]. Attempts(usize), #[cfg(not(feature = "no-std"))] - /// Time elapsed before abandoning retries for a payment. + /// Time elapsed before abandoning retries for a payment. At least one attempt at payment is made; + /// see [`PaymentParameters::expiry_time`] to avoid any attempt at payment after a specific time. + /// + /// [`PaymentParameters::expiry_time`]: crate::routing::router::PaymentParameters::expiry_time Timeout(core::time::Duration), } @@ -256,7 +270,11 @@ pub(crate) struct PaymentAttemptsUsingTime { /// it means the result of the first attempt is not known yet. pub(crate) count: usize, /// This field is only used when retry is `Retry::Timeout` which is only build with feature std - first_attempted_at: T + #[cfg(not(feature = "no-std"))] + first_attempted_at: T, + #[cfg(feature = "no-std")] + phantom: core::marker::PhantomData, + } #[cfg(not(any(feature = "no-std", test)))] @@ -270,7 +288,10 @@ impl PaymentAttemptsUsingTime { pub(crate) fn new() -> Self { PaymentAttemptsUsingTime { count: 0, - first_attempted_at: T::now() + #[cfg(not(feature = "no-std"))] + first_attempted_at: T::now(), + #[cfg(feature = "no-std")] + phantom: core::marker::PhantomData, } } } @@ -289,9 +310,35 @@ impl Display for PaymentAttemptsUsingTime { } } -/// If a payment fails to send, it can be in one of several states. This enum is returned as the -/// Err() type describing which state the payment is in, see the description of individual enum -/// states for more. +/// Indicates an immediate error on [`ChannelManager::send_payment_with_retry`]. Further errors +/// may be surfaced later via [`Event::PaymentPathFailed`] and [`Event::PaymentFailed`]. +/// +/// [`ChannelManager::send_payment_with_retry`]: crate::ln::channelmanager::ChannelManager::send_payment_with_retry +/// [`Event::PaymentPathFailed`]: crate::events::Event::PaymentPathFailed +/// [`Event::PaymentFailed`]: crate::events::Event::PaymentFailed +#[derive(Clone, Debug)] +pub enum RetryableSendFailure { + /// The provided [`PaymentParameters::expiry_time`] indicated that the payment has expired. Note + /// that this error is *not* caused by [`Retry::Timeout`]. + /// + /// [`PaymentParameters::expiry_time`]: crate::routing::router::PaymentParameters::expiry_time + PaymentExpired, + /// We were unable to find a route to the destination. + RouteNotFound, + /// Indicates that a payment for the provided [`PaymentId`] is already in-flight and has not + /// yet completed (i.e. generated an [`Event::PaymentSent`] or [`Event::PaymentFailed`]). + /// + /// [`PaymentId`]: crate::ln::channelmanager::PaymentId + /// [`Event::PaymentSent`]: crate::events::Event::PaymentSent + /// [`Event::PaymentFailed`]: crate::events::Event::PaymentFailed + DuplicatePayment, +} + +/// If a payment fails to send with [`ChannelManager::send_payment`], it can be in one of several +/// states. This enum is returned as the Err() type describing which state the payment is in, see +/// the description of individual enum states for more. +/// +/// [`ChannelManager::send_payment`]: crate::ln::channelmanager::ChannelManager::send_payment #[derive(Clone, Debug)] pub enum PaymentSendFailure { /// A parameter which was passed to send_payment was invalid, preventing us from attempting to @@ -299,68 +346,58 @@ pub enum PaymentSendFailure { /// /// You can freely resend the payment in full (with the parameter error fixed). /// - /// Because the payment failed outright, no payment tracking is done, you do not need to call - /// [`ChannelManager::abandon_payment`] and [`ChannelManager::retry_payment`] will *not* work - /// for this payment. + /// Because the payment failed outright, no payment tracking is done and no + /// [`Event::PaymentPathFailed`] or [`Event::PaymentFailed`] events will be generated. /// - /// [`ChannelManager::abandon_payment`]: crate::ln::channelmanager::ChannelManager::abandon_payment - /// [`ChannelManager::retry_payment`]: crate::ln::channelmanager::ChannelManager::retry_payment + /// [`Event::PaymentPathFailed`]: crate::events::Event::PaymentPathFailed + /// [`Event::PaymentFailed`]: crate::events::Event::PaymentFailed ParameterError(APIError), /// A parameter in a single path which was passed to send_payment was invalid, preventing us /// from attempting to send the payment at all. /// /// You can freely resend the payment in full (with the parameter error fixed). /// + /// Because the payment failed outright, no payment tracking is done and no + /// [`Event::PaymentPathFailed`] or [`Event::PaymentFailed`] events will be generated. + /// /// The results here are ordered the same as the paths in the route object which was passed to /// send_payment. /// - /// Because the payment failed outright, no payment tracking is done, you do not need to call - /// [`ChannelManager::abandon_payment`] and [`ChannelManager::retry_payment`] will *not* work - /// for this payment. - /// - /// [`ChannelManager::abandon_payment`]: crate::ln::channelmanager::ChannelManager::abandon_payment - /// [`ChannelManager::retry_payment`]: crate::ln::channelmanager::ChannelManager::retry_payment + /// [`Event::PaymentPathFailed`]: crate::events::Event::PaymentPathFailed + /// [`Event::PaymentFailed`]: crate::events::Event::PaymentFailed PathParameterError(Vec>), /// All paths which were attempted failed to send, with no channel state change taking place. /// You can freely resend the payment in full (though you probably want to do so over different /// paths than the ones selected). /// - /// Because the payment failed outright, no payment tracking is done, you do not need to call - /// [`ChannelManager::abandon_payment`] and [`ChannelManager::retry_payment`] will *not* work - /// for this payment. + /// Because the payment failed outright, no payment tracking is done and no + /// [`Event::PaymentPathFailed`] or [`Event::PaymentFailed`] events will be generated. /// - /// [`ChannelManager::abandon_payment`]: crate::ln::channelmanager::ChannelManager::abandon_payment - /// [`ChannelManager::retry_payment`]: crate::ln::channelmanager::ChannelManager::retry_payment + /// [`Event::PaymentPathFailed`]: crate::events::Event::PaymentPathFailed + /// [`Event::PaymentFailed`]: crate::events::Event::PaymentFailed AllFailedResendSafe(Vec), /// Indicates that a payment for the provided [`PaymentId`] is already in-flight and has not - /// yet completed (i.e. generated an [`Event::PaymentSent`]) or been abandoned (via - /// [`ChannelManager::abandon_payment`]). + /// yet completed (i.e. generated an [`Event::PaymentSent`] or [`Event::PaymentFailed`]). /// /// [`PaymentId`]: crate::ln::channelmanager::PaymentId - /// [`Event::PaymentSent`]: crate::util::events::Event::PaymentSent - /// [`ChannelManager::abandon_payment`]: crate::ln::channelmanager::ChannelManager::abandon_payment + /// [`Event::PaymentSent`]: crate::events::Event::PaymentSent + /// [`Event::PaymentFailed`]: crate::events::Event::PaymentFailed DuplicatePayment, - /// Some paths which were attempted failed to send, though possibly not all. At least some - /// paths have irrevocably committed to the HTLC and retrying the payment in full would result - /// in over-/re-payment. + /// Some paths that were attempted failed to send, though some paths may have succeeded. At least + /// some paths have irrevocably committed to the HTLC. /// - /// The results here are ordered the same as the paths in the route object which was passed to - /// send_payment, and any `Err`s which are not [`APIError::MonitorUpdateInProgress`] can be - /// safely retried via [`ChannelManager::retry_payment`]. + /// The results here are ordered the same as the paths in the route object that was passed to + /// send_payment. /// - /// Any entries which contain `Err(APIError::MonitorUpdateInprogress)` or `Ok(())` MUST NOT be - /// retried as they will result in over-/re-payment. These HTLCs all either successfully sent - /// (in the case of `Ok(())`) or will send once a [`MonitorEvent::Completed`] is provided for - /// the next-hop channel with the latest update_id. + /// Any entries that contain `Err(APIError::MonitorUpdateInprogress)` will send once a + /// [`MonitorEvent::Completed`] is provided for the next-hop channel with the latest update_id. /// - /// [`ChannelManager::retry_payment`]: crate::ln::channelmanager::ChannelManager::retry_payment /// [`MonitorEvent::Completed`]: crate::chain::channelmonitor::MonitorEvent::Completed PartialFailure { - /// The errors themselves, in the same order as the route hops. + /// The errors themselves, in the same order as the paths from the route. results: Vec>, /// If some paths failed without irrevocably committing to the new HTLC(s), this will - /// contain a [`RouteParameters`] object which can be used to calculate a new route that - /// will pay all remaining unpaid balance. + /// contain a [`RouteParameters`] object for the failing paths. failed_paths_retry: Option, /// The payment id for the payment, which is now at least partially pending. payment_id: PaymentId, @@ -369,32 +406,36 @@ pub enum PaymentSendFailure { pub(super) struct OutboundPayments { pub(super) pending_outbound_payments: Mutex>, + pub(super) retry_lock: Mutex<()>, } impl OutboundPayments { pub(super) fn new() -> Self { Self { - pending_outbound_payments: Mutex::new(HashMap::new()) + pending_outbound_payments: Mutex::new(HashMap::new()), + retry_lock: Mutex::new(()), } } - pub(super) fn send_payment( + pub(super) fn send_payment( &self, payment_hash: PaymentHash, payment_secret: &Option, payment_id: PaymentId, retry_strategy: Retry, route_params: RouteParameters, router: &R, - first_hops: Vec, inflight_htlcs: InFlightHtlcs, entropy_source: &ES, - node_signer: &NS, best_block_height: u32, send_payment_along_path: F - ) -> Result<(), PaymentSendFailure> + first_hops: Vec, compute_inflight_htlcs: IH, entropy_source: &ES, + node_signer: &NS, best_block_height: u32, logger: &L, + pending_events: &Mutex>, send_payment_along_path: SP, + ) -> Result<(), RetryableSendFailure> where R::Target: Router, ES::Target: EntropySource, NS::Target: NodeSigner, - F: Fn(&Vec, &Option, &PaymentHash, &Option, u64, - u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError>, + L::Target: Logger, + IH: Fn() -> InFlightHtlcs, + SP: Fn(&Vec, &PaymentHash, &Option, u64, u32, PaymentId, + &Option, [u8; 32]) -> Result<(), APIError>, { - self.pay_internal(payment_id, Some((payment_hash, payment_secret, retry_strategy)), - route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, - best_block_height, &send_payment_along_path) - .map_err(|e| { self.remove_outbound_if_all_failed(payment_id, &e); e }) + self.send_payment_internal(payment_id, payment_hash, payment_secret, None, retry_strategy, + route_params, router, first_hops, &compute_inflight_htlcs, entropy_source, node_signer, + best_block_height, logger, pending_events, &send_payment_along_path) } pub(super) fn send_payment_with_route( @@ -405,31 +446,54 @@ impl OutboundPayments { where ES::Target: EntropySource, NS::Target: NodeSigner, - F: Fn(&Vec, &Option, &PaymentHash, &Option, u64, - u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError> + F: Fn(&Vec, &PaymentHash, &Option, u64, u32, PaymentId, + &Option, [u8; 32]) -> Result<(), APIError> { - let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret, payment_id, route, Retry::Attempts(0), None, entropy_source, best_block_height)?; + let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret, payment_id, None, route, None, None, entropy_source, best_block_height)?; self.pay_route_internal(route, payment_hash, payment_secret, None, payment_id, None, onion_session_privs, node_signer, best_block_height, &send_payment_along_path) .map_err(|e| { self.remove_outbound_if_all_failed(payment_id, &e); e }) } - pub(super) fn send_spontaneous_payment( + pub(super) fn send_spontaneous_payment( + &self, payment_preimage: Option, payment_id: PaymentId, + retry_strategy: Retry, route_params: RouteParameters, router: &R, + first_hops: Vec, inflight_htlcs: IH, entropy_source: &ES, + node_signer: &NS, best_block_height: u32, logger: &L, + pending_events: &Mutex>, send_payment_along_path: SP + ) -> Result + where + R::Target: Router, + ES::Target: EntropySource, + NS::Target: NodeSigner, + L::Target: Logger, + IH: Fn() -> InFlightHtlcs, + SP: Fn(&Vec, &PaymentHash, &Option, u64, u32, PaymentId, + &Option, [u8; 32]) -> Result<(), APIError>, + { + let preimage = payment_preimage + .unwrap_or_else(|| PaymentPreimage(entropy_source.get_secure_random_bytes())); + let payment_hash = PaymentHash(Sha256::hash(&preimage.0).into_inner()); + self.send_payment_internal(payment_id, payment_hash, &None, Some(preimage), retry_strategy, + route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, + best_block_height, logger, pending_events, send_payment_along_path) + .map(|()| payment_hash) + } + + pub(super) fn send_spontaneous_payment_with_route( &self, route: &Route, payment_preimage: Option, payment_id: PaymentId, entropy_source: &ES, node_signer: &NS, best_block_height: u32, send_payment_along_path: F ) -> Result where ES::Target: EntropySource, NS::Target: NodeSigner, - F: Fn(&Vec, &Option, &PaymentHash, &Option, u64, - u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError> + F: Fn(&Vec, &PaymentHash, &Option, u64, u32, PaymentId, + &Option, [u8; 32]) -> Result<(), APIError> { - let preimage = match payment_preimage { - Some(p) => p, - None => PaymentPreimage(entropy_source.get_secure_random_bytes()), - }; + let preimage = payment_preimage + .unwrap_or_else(|| PaymentPreimage(entropy_source.get_secure_random_bytes())); let payment_hash = PaymentHash(Sha256::hash(&preimage.0).into_inner()); - let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, &route, Retry::Attempts(0), None, entropy_source, best_block_height)?; + let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, Some(preimage), &route, None, None, entropy_source, best_block_height)?; match self.pay_route_internal(route, payment_hash, &None, Some(preimage), payment_id, None, onion_session_privs, node_signer, best_block_height, &send_payment_along_path) { Ok(()) => Ok(payment_hash), @@ -442,182 +506,299 @@ impl OutboundPayments { pub(super) fn check_retry_payments( &self, router: &R, first_hops: FH, inflight_htlcs: IH, entropy_source: &ES, node_signer: &NS, - best_block_height: u32, logger: &L, send_payment_along_path: SP, + best_block_height: u32, pending_events: &Mutex>, logger: &L, + send_payment_along_path: SP, ) where R::Target: Router, ES::Target: EntropySource, NS::Target: NodeSigner, - SP: Fn(&Vec, &Option, &PaymentHash, &Option, u64, - u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError>, + SP: Fn(&Vec, &PaymentHash, &Option, u64, u32, PaymentId, + &Option, [u8; 32]) -> Result<(), APIError>, IH: Fn() -> InFlightHtlcs, FH: Fn() -> Vec, L::Target: Logger, { + let _single_thread = self.retry_lock.lock().unwrap(); loop { let mut outbounds = self.pending_outbound_payments.lock().unwrap(); let mut retry_id_route_params = None; for (pmt_id, pmt) in outbounds.iter_mut() { - if pmt.is_retryable_now() { - if let PendingOutboundPayment::Retryable { pending_amt_msat, total_msat, route_params: Some(params), .. } = pmt { + if pmt.is_auto_retryable_now() { + if let PendingOutboundPayment::Retryable { pending_amt_msat, total_msat, payment_params: Some(params), payment_hash, .. } = pmt { if pending_amt_msat < total_msat { - retry_id_route_params = Some((*pmt_id, params.clone())); - pmt.increment_attempts(); + retry_id_route_params = Some((*payment_hash, *pmt_id, RouteParameters { + final_value_msat: *total_msat - *pending_amt_msat, + payment_params: params.clone(), + })); break } - } + } else { debug_assert!(false); } } } - if let Some((payment_id, route_params)) = retry_id_route_params { - core::mem::drop(outbounds); - if let Err(e) = self.pay_internal(payment_id, None, route_params, router, first_hops(), inflight_htlcs(), entropy_source, node_signer, best_block_height, &send_payment_along_path) { - log_trace!(logger, "Errored retrying payment: {:?}", e); - } + core::mem::drop(outbounds); + if let Some((payment_hash, payment_id, route_params)) = retry_id_route_params { + self.retry_payment_internal(payment_hash, payment_id, route_params, router, first_hops(), &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, &send_payment_along_path) } else { break } } + + let mut outbounds = self.pending_outbound_payments.lock().unwrap(); + outbounds.retain(|pmt_id, pmt| { + let mut retain = true; + if !pmt.is_auto_retryable_now() && pmt.remaining_parts() == 0 { + if pmt.mark_abandoned().is_ok() { + pending_events.lock().unwrap().push(events::Event::PaymentFailed { + payment_id: *pmt_id, + payment_hash: pmt.payment_hash().expect("PendingOutboundPayments::Retryable always has a payment hash set"), + }); + retain = false; + } + } + retain + }); } - fn pay_internal( - &self, payment_id: PaymentId, - initial_send_info: Option<(PaymentHash, &Option, Retry)>, - route_params: RouteParameters, router: &R, first_hops: Vec, - inflight_htlcs: InFlightHtlcs, entropy_source: &ES, node_signer: &NS, best_block_height: u32, - send_payment_along_path: &F - ) -> Result<(), PaymentSendFailure> + pub(super) fn needs_abandon(&self) -> bool { + let outbounds = self.pending_outbound_payments.lock().unwrap(); + outbounds.iter().any(|(_, pmt)| + !pmt.is_auto_retryable_now() && pmt.remaining_parts() == 0 && !pmt.is_fulfilled()) + } + + /// Errors immediately on [`RetryableSendFailure`] error conditions. Otherwise, further errors may + /// be surfaced asynchronously via [`Event::PaymentPathFailed`] and [`Event::PaymentFailed`]. + /// + /// [`Event::PaymentPathFailed`]: crate::events::Event::PaymentPathFailed + /// [`Event::PaymentFailed`]: crate::events::Event::PaymentFailed + fn send_payment_internal( + &self, payment_id: PaymentId, payment_hash: PaymentHash, payment_secret: &Option, + keysend_preimage: Option, retry_strategy: Retry, route_params: RouteParameters, + router: &R, first_hops: Vec, inflight_htlcs: IH, entropy_source: &ES, + node_signer: &NS, best_block_height: u32, logger: &L, + pending_events: &Mutex>, send_payment_along_path: SP, + ) -> Result<(), RetryableSendFailure> where R::Target: Router, ES::Target: EntropySource, NS::Target: NodeSigner, - F: Fn(&Vec, &Option, &PaymentHash, &Option, u64, - u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError> + L::Target: Logger, + IH: Fn() -> InFlightHtlcs, + SP: Fn(&Vec, &PaymentHash, &Option, u64, u32, PaymentId, + &Option, [u8; 32]) -> Result<(), APIError> { #[cfg(feature = "std")] { if has_expired(&route_params) { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: format!("Invoice expired for payment id {}", log_bytes!(payment_id.0)), - })) + return Err(RetryableSendFailure::PaymentExpired) } } - let route = router.find_route( + let route = router.find_route_with_id( &node_signer.get_node_id(Recipient::Node).unwrap(), &route_params, - Some(&first_hops.iter().collect::>()), &inflight_htlcs - ).map_err(|e| PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: format!("Failed to find a route for payment {}: {:?}", log_bytes!(payment_id.0), e), // TODO: add APIError::RouteNotFound - }))?; - - let res = if let Some((payment_hash, payment_secret, retry_strategy)) = initial_send_info { - let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret, payment_id, &route, retry_strategy, Some(route_params.clone()), entropy_source, best_block_height)?; - self.pay_route_internal(&route, payment_hash, payment_secret, None, payment_id, None, onion_session_privs, node_signer, best_block_height, send_payment_along_path) - } else { - self.retry_payment_with_route(&route, payment_id, entropy_source, node_signer, best_block_height, send_payment_along_path) - }; - match res { - Err(PaymentSendFailure::AllFailedResendSafe(_)) => { - let mut outbounds = self.pending_outbound_payments.lock().unwrap(); - if let Some(payment) = outbounds.get_mut(&payment_id) { - let retryable = payment.is_retryable_now(); - if retryable { - payment.increment_attempts(); - } else { return res } - } else { return res } - core::mem::drop(outbounds); - self.pay_internal(payment_id, None, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, send_payment_along_path) - }, - Err(PaymentSendFailure::PartialFailure { failed_paths_retry: Some(retry), results, .. }) => { - let mut outbounds = self.pending_outbound_payments.lock().unwrap(); - if let Some(payment) = outbounds.get_mut(&payment_id) { - let retryable = payment.is_retryable_now(); - if retryable { - payment.increment_attempts(); - } else { return Err(PaymentSendFailure::PartialFailure { failed_paths_retry: Some(retry), results, payment_id }) } - } else { return Err(PaymentSendFailure::PartialFailure { failed_paths_retry: Some(retry), results, payment_id }) } - core::mem::drop(outbounds); - - // Some paths were sent, even if we failed to send the full MPP value our recipient may - // misbehave and claim the funds, at which point we have to consider the payment sent, so - // return `Ok()` here, ignoring any retry errors. - let _ = self.pay_internal(payment_id, None, retry, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, send_payment_along_path); - Ok(()) - }, - Err(PaymentSendFailure::PartialFailure { failed_paths_retry: None, .. }) => { - // This may happen if we send a payment and some paths fail, but only due to a temporary - // monitor failure or the like, implying they're really in-flight, but we haven't sent the - // initial HTLC-Add messages yet. - Ok(()) - }, - res => res, + Some(&first_hops.iter().collect::>()), &inflight_htlcs(), + payment_hash, payment_id, + ).map_err(|_| RetryableSendFailure::RouteNotFound)?; + + let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret, + payment_id, keysend_preimage, &route, Some(retry_strategy), + Some(route_params.payment_params.clone()), entropy_source, best_block_height) + .map_err(|_| RetryableSendFailure::DuplicatePayment)?; + + let res = self.pay_route_internal(&route, payment_hash, payment_secret, None, payment_id, None, + onion_session_privs, node_signer, best_block_height, &send_payment_along_path); + log_info!(logger, "Result sending payment with id {}: {:?}", log_bytes!(payment_id.0), res); + if let Err(e) = res { + self.handle_pay_route_err(e, payment_id, payment_hash, route, route_params, router, first_hops, &inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, &send_payment_along_path); } + Ok(()) } - pub(super) fn retry_payment_with_route( - &self, route: &Route, payment_id: PaymentId, entropy_source: &ES, node_signer: &NS, best_block_height: u32, - send_payment_along_path: F - ) -> Result<(), PaymentSendFailure> + fn retry_payment_internal( + &self, payment_hash: PaymentHash, payment_id: PaymentId, route_params: RouteParameters, + router: &R, first_hops: Vec, inflight_htlcs: &IH, entropy_source: &ES, + node_signer: &NS, best_block_height: u32, logger: &L, + pending_events: &Mutex>, send_payment_along_path: &SP, + ) where + R::Target: Router, ES::Target: EntropySource, NS::Target: NodeSigner, - F: Fn(&Vec, &Option, &PaymentHash, &Option, u64, - u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError> + L::Target: Logger, + IH: Fn() -> InFlightHtlcs, + SP: Fn(&Vec, &PaymentHash, &Option, u64, u32, PaymentId, + &Option, [u8; 32]) -> Result<(), APIError> { - const RETRY_OVERFLOW_PERCENTAGE: u64 = 10; + #[cfg(feature = "std")] { + if has_expired(&route_params) { + log_error!(logger, "Payment params expired on retry, abandoning payment {}", log_bytes!(payment_id.0)); + self.abandon_payment(payment_id, pending_events); + return + } + } + + let route = match router.find_route_with_id( + &node_signer.get_node_id(Recipient::Node).unwrap(), &route_params, + Some(&first_hops.iter().collect::>()), &inflight_htlcs(), + payment_hash, payment_id, + ) { + Ok(route) => route, + Err(e) => { + log_error!(logger, "Failed to find a route on retry, abandoning payment {}: {:#?}", log_bytes!(payment_id.0), e); + self.abandon_payment(payment_id, pending_events); + return + } + }; for path in route.paths.iter() { if path.len() == 0 { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: "length-0 path in route".to_string() - })) + log_error!(logger, "length-0 path in route"); + self.abandon_payment(payment_id, pending_events); + return } } + const RETRY_OVERFLOW_PERCENTAGE: u64 = 10; let mut onion_session_privs = Vec::with_capacity(route.paths.len()); for _ in 0..route.paths.len() { onion_session_privs.push(entropy_source.get_secure_random_bytes()); } - let (total_msat, payment_hash, payment_secret) = { + macro_rules! abandon_with_entry { + ($payment: expr) => { + if $payment.get_mut().mark_abandoned().is_ok() && $payment.get().remaining_parts() == 0 { + pending_events.lock().unwrap().push(events::Event::PaymentFailed { + payment_id, + payment_hash, + }); + $payment.remove(); + } + } + } + let (total_msat, payment_secret, keysend_preimage) = { let mut outbounds = self.pending_outbound_payments.lock().unwrap(); - match outbounds.get_mut(&payment_id) { - Some(payment) => { - let res = match payment { + match outbounds.entry(payment_id) { + hash_map::Entry::Occupied(mut payment) => { + let res = match payment.get() { PendingOutboundPayment::Retryable { - total_msat, payment_hash, payment_secret, pending_amt_msat, .. + total_msat, keysend_preimage, payment_secret, pending_amt_msat, .. } => { let retry_amt_msat: u64 = route.paths.iter().map(|path| path.last().unwrap().fee_msat).sum(); if retry_amt_msat + *pending_amt_msat > *total_msat * (100 + RETRY_OVERFLOW_PERCENTAGE) / 100 { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: format!("retry_amt_msat of {} will put pending_amt_msat (currently: {}) more than 10% over total_payment_amt_msat of {}", retry_amt_msat, pending_amt_msat, total_msat).to_string() - })) + log_error!(logger, "retry_amt_msat of {} will put pending_amt_msat (currently: {}) more than 10% over total_payment_amt_msat of {}", retry_amt_msat, pending_amt_msat, total_msat); + abandon_with_entry!(payment); + return } - (*total_msat, *payment_hash, *payment_secret) + (*total_msat, *payment_secret, *keysend_preimage) }, PendingOutboundPayment::Legacy { .. } => { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102".to_string() - })) + log_error!(logger, "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102"); + return }, PendingOutboundPayment::Fulfilled { .. } => { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: "Payment already completed".to_owned() - })); + log_error!(logger, "Payment already completed"); + return }, PendingOutboundPayment::Abandoned { .. } => { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: "Payment already abandoned (with some HTLCs still pending)".to_owned() - })); + log_error!(logger, "Payment already abandoned (with some HTLCs still pending)"); + return }, }; + if !payment.get().is_retryable_now() { + log_error!(logger, "Retries exhausted for payment id {}", log_bytes!(payment_id.0)); + abandon_with_entry!(payment); + return + } + payment.get_mut().increment_attempts(); for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) { - assert!(payment.insert(*session_priv_bytes, path)); + assert!(payment.get_mut().insert(*session_priv_bytes, path)); } res }, - None => - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { - err: format!("Payment with ID {} not found", log_bytes!(payment_id.0)), - })), + hash_map::Entry::Vacant(_) => { + log_error!(logger, "Payment with ID {} not found", log_bytes!(payment_id.0)); + return + } } }; - self.pay_route_internal(route, payment_hash, &payment_secret, None, payment_id, Some(total_msat), onion_session_privs, node_signer, best_block_height, &send_payment_along_path) + let res = self.pay_route_internal(&route, payment_hash, &payment_secret, keysend_preimage, + payment_id, Some(total_msat), onion_session_privs, node_signer, best_block_height, + &send_payment_along_path); + log_info!(logger, "Result retrying payment id {}: {:?}", log_bytes!(payment_id.0), res); + if let Err(e) = res { + self.handle_pay_route_err(e, payment_id, payment_hash, route, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); + } + } + + fn handle_pay_route_err( + &self, err: PaymentSendFailure, payment_id: PaymentId, payment_hash: PaymentHash, route: Route, + mut route_params: RouteParameters, router: &R, first_hops: Vec, + inflight_htlcs: &IH, entropy_source: &ES, node_signer: &NS, best_block_height: u32, logger: &L, + pending_events: &Mutex>, send_payment_along_path: &SP, + ) + where + R::Target: Router, + ES::Target: EntropySource, + NS::Target: NodeSigner, + L::Target: Logger, + IH: Fn() -> InFlightHtlcs, + SP: Fn(&Vec, &PaymentHash, &Option, u64, u32, PaymentId, + &Option, [u8; 32]) -> Result<(), APIError> + { + match err { + PaymentSendFailure::AllFailedResendSafe(errs) => { + Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut route_params, route.paths, errs.into_iter().map(|e| Err(e)), pending_events); + self.retry_payment_internal(payment_hash, payment_id, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); + }, + PaymentSendFailure::PartialFailure { failed_paths_retry: Some(mut retry), results, .. } => { + Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut retry, route.paths, results.into_iter(), pending_events); + // Some paths were sent, even if we failed to send the full MPP value our recipient may + // misbehave and claim the funds, at which point we have to consider the payment sent, so + // return `Ok()` here, ignoring any retry errors. + self.retry_payment_internal(payment_hash, payment_id, retry, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path); + }, + PaymentSendFailure::PartialFailure { failed_paths_retry: None, .. } => { + // This may happen if we send a payment and some paths fail, but only due to a temporary + // monitor failure or the like, implying they're really in-flight, but we haven't sent the + // initial HTLC-Add messages yet. + }, + PaymentSendFailure::PathParameterError(results) => { + Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut route_params, route.paths, results.into_iter(), pending_events); + self.abandon_payment(payment_id, pending_events); + }, + PaymentSendFailure::ParameterError(e) => { + log_error!(logger, "Failed to send to route due to parameter error: {:?}. Your router is buggy", e); + self.abandon_payment(payment_id, pending_events); + }, + PaymentSendFailure::DuplicatePayment => debug_assert!(false), // unreachable + } + } + + fn push_path_failed_evs_and_scids>>( + payment_id: PaymentId, payment_hash: PaymentHash, route_params: &mut RouteParameters, + paths: Vec>, path_results: I, pending_events: &Mutex> + ) { + let mut events = pending_events.lock().unwrap(); + debug_assert_eq!(paths.len(), path_results.len()); + for (path, path_res) in paths.into_iter().zip(path_results) { + if let Err(e) = path_res { + if let APIError::MonitorUpdateInProgress = e { continue } + let mut failed_scid = None; + if let APIError::ChannelUnavailable { .. } = e { + let scid = path[0].short_channel_id; + failed_scid = Some(scid); + route_params.payment_params.previously_failed_channels.push(scid); + } + events.push(events::Event::PaymentPathFailed { + payment_id: Some(payment_id), + payment_hash, + payment_failed_permanently: false, + failure: events::PathFailure::InitialSend { err: e }, + path, + short_channel_id: failed_scid, + #[cfg(test)] + error_code: None, + #[cfg(test)] + error_data: None, + }); + } + } } pub(super) fn send_probe( @@ -627,8 +808,8 @@ impl OutboundPayments { where ES::Target: EntropySource, NS::Target: NodeSigner, - F: Fn(&Vec, &Option, &PaymentHash, &Option, u64, - u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError> + F: Fn(&Vec, &PaymentHash, &Option, u64, u32, PaymentId, + &Option, [u8; 32]) -> Result<(), APIError> { let payment_id = PaymentId(entropy_source.get_secure_random_bytes()); @@ -641,7 +822,7 @@ impl OutboundPayments { } let route = Route { paths: vec![hops], payment_params: None }; - let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, &route, Retry::Attempts(0), None, entropy_source, best_block_height)?; + let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, None, &route, None, None, entropy_source, best_block_height)?; match self.pay_route_internal(&route, payment_hash, &None, None, payment_id, None, onion_session_privs, node_signer, best_block_height, &send_payment_along_path) { Ok(()) => Ok((payment_hash, payment_id)), @@ -655,15 +836,15 @@ impl OutboundPayments { #[cfg(test)] pub(super) fn test_add_new_pending_payment( &self, payment_hash: PaymentHash, payment_secret: Option, payment_id: PaymentId, - route: &Route, retry_strategy: Retry, entropy_source: &ES, best_block_height: u32 + route: &Route, retry_strategy: Option, entropy_source: &ES, best_block_height: u32 ) -> Result, PaymentSendFailure> where ES::Target: EntropySource { - self.add_new_pending_payment(payment_hash, payment_secret, payment_id, route, retry_strategy, None, entropy_source, best_block_height) + self.add_new_pending_payment(payment_hash, payment_secret, payment_id, None, route, retry_strategy, None, entropy_source, best_block_height) } pub(super) fn add_new_pending_payment( &self, payment_hash: PaymentHash, payment_secret: Option, payment_id: PaymentId, - route: &Route, retry_strategy: Retry, route_params: Option, - entropy_source: &ES, best_block_height: u32 + keysend_preimage: Option, route: &Route, retry_strategy: Option, + payment_params: Option, entropy_source: &ES, best_block_height: u32 ) -> Result, PaymentSendFailure> where ES::Target: EntropySource { let mut onion_session_privs = Vec::with_capacity(route.paths.len()); for _ in 0..route.paths.len() { @@ -677,12 +858,13 @@ impl OutboundPayments { let payment = entry.insert(PendingOutboundPayment::Retryable { retry_strategy, attempts: PaymentAttempts::new(), - route_params, + payment_params, session_privs: HashSet::new(), pending_amt_msat: 0, pending_fee_msat: Some(0), payment_hash, payment_secret, + keysend_preimage, starting_block_height: best_block_height, total_msat: route.get_total_amount(), }); @@ -704,26 +886,26 @@ impl OutboundPayments { ) -> Result<(), PaymentSendFailure> where NS::Target: NodeSigner, - F: Fn(&Vec, &Option, &PaymentHash, &Option, u64, - u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError> + F: Fn(&Vec, &PaymentHash, &Option, u64, u32, PaymentId, + &Option, [u8; 32]) -> Result<(), APIError> { if route.paths.len() < 1 { - return Err(PaymentSendFailure::ParameterError(APIError::InvalidRoute{err: "There must be at least one path to send over"})); + return Err(PaymentSendFailure::ParameterError(APIError::InvalidRoute{err: "There must be at least one path to send over".to_owned()})); } if payment_secret.is_none() && route.paths.len() > 1 { - return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError{err: "Payment secret is required for multi-path payments".to_string()})); + return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError{err: "Payment secret is required for multi-path payments".to_owned()})); } let mut total_value = 0; let our_node_id = node_signer.get_node_id(Recipient::Node).unwrap(); // TODO no unwrap let mut path_errs = Vec::with_capacity(route.paths.len()); 'path_check: for path in route.paths.iter() { if path.len() < 1 || path.len() > 20 { - path_errs.push(Err(APIError::InvalidRoute{err: "Path didn't go anywhere/had bogus size"})); + path_errs.push(Err(APIError::InvalidRoute{err: "Path didn't go anywhere/had bogus size".to_owned()})); continue 'path_check; } for (idx, hop) in path.iter().enumerate() { if idx != path.len() - 1 && hop.pubkey == our_node_id { - path_errs.push(Err(APIError::InvalidRoute{err: "Path went through us but wasn't a simple rebalance loop to us"})); + path_errs.push(Err(APIError::InvalidRoute{err: "Path went through us but wasn't a simple rebalance loop to us".to_owned()})); continue 'path_check; } } @@ -734,7 +916,6 @@ impl OutboundPayments { return Err(PaymentSendFailure::PathParameterError(path_errs)); } if let Some(amt_msat) = recv_value_msat { - debug_assert!(amt_msat >= total_value); total_value = amt_msat; } @@ -742,7 +923,7 @@ impl OutboundPayments { let mut results = Vec::new(); debug_assert_eq!(route.paths.len(), onion_session_privs.len()); for (path, session_priv) in route.paths.iter().zip(onion_session_privs.into_iter()) { - let mut path_res = send_payment_along_path(&path, &route.payment_params, &payment_hash, payment_secret, total_value, cur_height, payment_id, &keysend_preimage, session_priv); + let mut path_res = send_payment_along_path(&path, &payment_hash, payment_secret, total_value, cur_height, payment_id, &keysend_preimage, session_priv); match path_res { Ok(_) => {}, Err(APIError::MonitorUpdateInProgress) => { @@ -789,7 +970,6 @@ impl OutboundPayments { Some(RouteParameters { payment_params: payment_params.clone(), final_value_msat: pending_amt_unsent, - final_cltv_expiry_delta: max_unsent_cltv_delta, }) } else { None } } else { None }, @@ -810,8 +990,8 @@ impl OutboundPayments { ) -> Result<(), PaymentSendFailure> where NS::Target: NodeSigner, - F: Fn(&Vec, &Option, &PaymentHash, &Option, u64, - u32, PaymentId, &Option, [u8; 32]) -> Result<(), APIError> + F: Fn(&Vec, &PaymentHash, &Option, u64, u32, PaymentId, + &Option, [u8; 32]) -> Result<(), APIError> { self.pay_route_internal(route, payment_hash, payment_secret, keysend_preimage, payment_id, recv_value_msat, onion_session_privs, node_signer, best_block_height, @@ -819,8 +999,8 @@ impl OutboundPayments { .map_err(|e| { self.remove_outbound_if_all_failed(payment_id, &e); e }) } - // If we failed to send any paths, we should remove the new PaymentId from the - // `pending_outbound_payments` map, as the user isn't expected to `abandon_payment`. + // If we failed to send any paths, remove the new PaymentId from the `pending_outbound_payments` + // map as the payment is free to be resent. fn remove_outbound_if_all_failed(&self, payment_id: PaymentId, err: &PaymentSendFailure) { if let &PaymentSendFailure::AllFailedResendSafe(_) = err { let removed = self.pending_outbound_payments.lock().unwrap().remove(&payment_id).is_some(); @@ -936,64 +1116,81 @@ impl OutboundPayments { }); } + // Returns a bool indicating whether a PendingHTLCsForwardable event should be generated. pub(super) fn fail_htlc( &self, source: &HTLCSource, payment_hash: &PaymentHash, onion_error: &HTLCFailReason, path: &Vec, session_priv: &SecretKey, payment_id: &PaymentId, - payment_params: &Option, probing_cookie_secret: [u8; 32], - secp_ctx: &Secp256k1, pending_events: &Mutex>, logger: &L - ) where L::Target: Logger { + probing_cookie_secret: [u8; 32], secp_ctx: &Secp256k1, + pending_events: &Mutex>, logger: &L + ) -> bool where L::Target: Logger { #[cfg(test)] let (network_update, short_channel_id, payment_retryable, onion_error_code, onion_error_data) = onion_error.decode_onion_failure(secp_ctx, logger, &source); #[cfg(not(test))] let (network_update, short_channel_id, payment_retryable, _, _) = onion_error.decode_onion_failure(secp_ctx, logger, &source); + let payment_is_probe = payment_is_probe(payment_hash, &payment_id, probing_cookie_secret); let mut session_priv_bytes = [0; 32]; session_priv_bytes.copy_from_slice(&session_priv[..]); let mut outbounds = self.pending_outbound_payments.lock().unwrap(); - let mut all_paths_failed = false; + + // If any payments already need retry, there's no need to generate a redundant + // `PendingHTLCsForwardable`. + let already_awaiting_retry = outbounds.iter().any(|(_, pmt)| { + let mut awaiting_retry = false; + if pmt.is_auto_retryable_now() { + if let PendingOutboundPayment::Retryable { pending_amt_msat, total_msat, .. } = pmt { + if pending_amt_msat < total_msat { + awaiting_retry = true; + } + } + } + awaiting_retry + }); + let mut full_failure_ev = None; - let mut pending_retry_ev = None; + let mut pending_retry_ev = false; let attempts_remaining = if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(*payment_id) { if !payment.get_mut().remove(&session_priv_bytes, Some(&path)) { log_trace!(logger, "Received duplicative fail for HTLC with payment_hash {}", log_bytes!(payment_hash.0)); - return + return false } if payment.get().is_fulfilled() { log_trace!(logger, "Received failure of HTLC with payment_hash {} after payment completion", log_bytes!(payment_hash.0)); - return + return false } - let is_retryable_now = payment.get().is_retryable_now(); + let mut is_retryable_now = payment.get().is_auto_retryable_now(); if let Some(scid) = short_channel_id { + // TODO: If we decided to blame ourselves (or one of our channels) in + // process_onion_failure we should close that channel as it implies our + // next-hop is needlessly blaming us! payment.get_mut().insert_previously_failed_scid(scid); } + + if payment_is_probe || !is_retryable_now || !payment_retryable { + let _ = payment.get_mut().mark_abandoned(); // we'll only Err if it's a legacy payment + is_retryable_now = false; + } if payment.get().remaining_parts() == 0 { - all_paths_failed = true; if payment.get().abandoned() { - full_failure_ev = Some(events::Event::PaymentFailed { - payment_id: *payment_id, - payment_hash: payment.get().payment_hash().expect("PendingOutboundPayments::RetriesExceeded always has a payment hash set"), - }); + if !payment_is_probe { + full_failure_ev = Some(events::Event::PaymentFailed { + payment_id: *payment_id, + payment_hash: payment.get().payment_hash().expect("PendingOutboundPayments::RetriesExceeded always has a payment hash set"), + }); + } payment.remove(); } } is_retryable_now } else { log_trace!(logger, "Received duplicative fail for HTLC with payment_hash {}", log_bytes!(payment_hash.0)); - return + return false }; core::mem::drop(outbounds); - let mut retry = if let Some(payment_params_data) = payment_params { - let path_last_hop = path.last().expect("Outbound payments must have had a valid path"); - Some(RouteParameters { - payment_params: payment_params_data.clone(), - final_value_msat: path_last_hop.fee_msat, - final_cltv_expiry_delta: path_last_hop.cltv_expiry_delta, - }) - } else { None }; log_trace!(logger, "Failing outbound payment HTLC with payment_hash {}", log_bytes!(payment_hash.0)); let path_failure = { - if payment_is_probe(payment_hash, &payment_id, probing_cookie_secret) { + if payment_is_probe { if !payment_retryable { events::Event::ProbeSuccessful { payment_id: *payment_id, @@ -1009,27 +1206,19 @@ impl OutboundPayments { } } } else { - // TODO: If we decided to blame ourselves (or one of our channels) in - // process_onion_failure we should close that channel as it implies our - // next-hop is needlessly blaming us! - if let Some(scid) = short_channel_id { - retry.as_mut().map(|r| r.payment_params.previously_failed_channels.push(scid)); - } - if payment_retryable && attempts_remaining && retry.is_some() { + // If we miss abandoning the payment above, we *must* generate an event here or else the + // payment will sit in our outbounds forever. + if attempts_remaining && !already_awaiting_retry { debug_assert!(full_failure_ev.is_none()); - pending_retry_ev = Some(events::Event::PendingHTLCsForwardable { - time_forwardable: Duration::from_millis(MIN_HTLC_RELAY_HOLDING_CELL_MILLIS), - }); + pending_retry_ev = true; } events::Event::PaymentPathFailed { payment_id: Some(*payment_id), payment_hash: payment_hash.clone(), payment_failed_permanently: !payment_retryable, - network_update, - all_paths_failed, + failure: events::PathFailure::OnPath { network_update }, path: path.clone(), short_channel_id, - retry, #[cfg(test)] error_code: onion_error_code, #[cfg(test)] @@ -1040,16 +1229,17 @@ impl OutboundPayments { let mut pending_events = pending_events.lock().unwrap(); pending_events.push(path_failure); if let Some(ev) = full_failure_ev { pending_events.push(ev); } - if let Some(ev) = pending_retry_ev { pending_events.push(ev); } + pending_retry_ev } - pub(super) fn abandon_payment(&self, payment_id: PaymentId) -> Option { - let mut failed_ev = None; + pub(super) fn abandon_payment( + &self, payment_id: PaymentId, pending_events: &Mutex> + ) { let mut outbounds = self.pending_outbound_payments.lock().unwrap(); if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(payment_id) { if let Ok(()) = payment.get_mut().mark_abandoned() { if payment.get().remaining_parts() == 0 { - failed_ev = Some(events::Event::PaymentFailed { + pending_events.lock().unwrap().push(events::Event::PaymentFailed { payment_id, payment_hash: payment.get().payment_hash().expect("PendingOutboundPayments::RetriesExceeded always has a payment hash set"), }); @@ -1057,7 +1247,6 @@ impl OutboundPayments { } } } - failed_ev } #[cfg(test)] @@ -1101,13 +1290,16 @@ impl_writeable_tlv_based_enum_upgradable!(PendingOutboundPayment, (0, session_privs, required), (1, pending_fee_msat, option), (2, payment_hash, required), - (not_written, retry_strategy, (static_value, Retry::Attempts(0))), + // Note that while we "default" payment_param's final CLTV expiry delta to 0 we should + // never see it - `payment_params` was added here after the field was added/required. + (3, payment_params, (option: ReadableArgs, 0)), (4, payment_secret, option), - (not_written, attempts, (static_value, PaymentAttempts::new())), + (5, keysend_preimage, option), (6, total_msat, required), - (not_written, route_params, (static_value, None)), (8, pending_amt_msat, required), (10, starting_block_height, required), + (not_written, retry_strategy, (static_value, None)), + (not_written, attempts, (static_value, PaymentAttempts::new())), }, (3, Abandoned) => { (0, session_privs, required), @@ -1117,17 +1309,18 @@ impl_writeable_tlv_based_enum_upgradable!(PendingOutboundPayment, #[cfg(test)] mod tests { - use bitcoin::blockdata::constants::genesis_block; use bitcoin::network::constants::Network; use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey}; + use crate::events::{Event, PathFailure}; use crate::ln::PaymentHash; - use crate::ln::channelmanager::{PaymentId, PaymentSendFailure}; + use crate::ln::channelmanager::PaymentId; + use crate::ln::features::{ChannelFeatures, NodeFeatures}; use crate::ln::msgs::{ErrorAction, LightningError}; - use crate::ln::outbound_payment::{OutboundPayments, Retry}; + use crate::ln::outbound_payment::{OutboundPayments, Retry, RetryableSendFailure}; use crate::routing::gossip::NetworkGraph; - use crate::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteParameters}; - use crate::sync::Arc; + use crate::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, RouteParameters}; + use crate::sync::{Arc, Mutex}; use crate::util::errors::APIError; use crate::util::test_utils; @@ -1141,33 +1334,40 @@ mod tests { fn do_fails_paying_after_expiration(on_retry: bool) { let outbound_payments = OutboundPayments::new(); let logger = test_utils::TestLogger::new(); - let genesis_hash = genesis_block(Network::Testnet).header.block_hash(); - let network_graph = Arc::new(NetworkGraph::new(genesis_hash, &logger)); - let router = test_utils::TestRouter::new(network_graph); + let network_graph = Arc::new(NetworkGraph::new(Network::Testnet, &logger)); + let scorer = Mutex::new(test_utils::TestScorer::new()); + let router = test_utils::TestRouter::new(network_graph, &scorer); let secp_ctx = Secp256k1::new(); let keys_manager = test_utils::TestKeysInterface::new(&[0; 32], Network::Testnet); let past_expiry_time = std::time::SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() - 2; let payment_params = PaymentParameters::from_node_id( - PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap())) - .with_expiry_time(past_expiry_time); + PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap()), + 0 + ).with_expiry_time(past_expiry_time); let expired_route_params = RouteParameters { payment_params, final_value_msat: 0, - final_cltv_expiry_delta: 0, }; - let err = if on_retry { - outbound_payments.pay_internal( - PaymentId([0; 32]), None, expired_route_params, &&router, vec![], InFlightHtlcs::new(), - &&keys_manager, &&keys_manager, 0, &|_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() + let pending_events = Mutex::new(Vec::new()); + if on_retry { + outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), None, PaymentId([0; 32]), None, + &Route { paths: vec![], payment_params: None }, Some(Retry::Attempts(1)), + Some(expired_route_params.payment_params.clone()), &&keys_manager, 0).unwrap(); + outbound_payments.retry_payment_internal( + PaymentHash([0; 32]), PaymentId([0; 32]), expired_route_params, &&router, vec![], + &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, + &pending_events, &|_, _, _, _, _, _, _, _| Ok(())); + let events = pending_events.lock().unwrap(); + assert_eq!(events.len(), 1); + if let Event::PaymentFailed { .. } = events[0] { } else { panic!("Unexpected event"); } } else { - outbound_payments.send_payment( + let err = outbound_payments.send_payment( PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), expired_route_params, - &&router, vec![], InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, |_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() - }; - if let PaymentSendFailure::ParameterError(APIError::APIMisuseError { err }) = err { - assert!(err.contains("Invoice expired")); - } else { panic!("Unexpected error"); } + &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, + &pending_events, |_, _, _, _, _, _, _, _| Ok(())).unwrap_err(); + if let RetryableSendFailure::PaymentExpired = err { } else { panic!("Unexpected error"); } + } } #[test] @@ -1178,35 +1378,126 @@ mod tests { fn do_find_route_error(on_retry: bool) { let outbound_payments = OutboundPayments::new(); let logger = test_utils::TestLogger::new(); - let genesis_hash = genesis_block(Network::Testnet).header.block_hash(); - let network_graph = Arc::new(NetworkGraph::new(genesis_hash, &logger)); - let router = test_utils::TestRouter::new(network_graph); + let network_graph = Arc::new(NetworkGraph::new(Network::Testnet, &logger)); + let scorer = Mutex::new(test_utils::TestScorer::new()); + let router = test_utils::TestRouter::new(network_graph, &scorer); let secp_ctx = Secp256k1::new(); let keys_manager = test_utils::TestKeysInterface::new(&[0; 32], Network::Testnet); - router.expect_find_route(Err(LightningError { err: String::new(), action: ErrorAction::IgnoreError })); - let payment_params = PaymentParameters::from_node_id( - PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap())); + PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap()), 0); let route_params = RouteParameters { payment_params, final_value_msat: 0, - final_cltv_expiry_delta: 0, }; - let err = if on_retry { - outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), None, PaymentId([0; 32]), - &Route { paths: vec![], payment_params: None }, Retry::Attempts(1), Some(route_params.clone()), - &&keys_manager, 0).unwrap(); - outbound_payments.pay_internal( - PaymentId([0; 32]), None, route_params, &&router, vec![], InFlightHtlcs::new(), - &&keys_manager, &&keys_manager, 0, &|_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() + router.expect_find_route(route_params.clone(), + Err(LightningError { err: String::new(), action: ErrorAction::IgnoreError })); + + let pending_events = Mutex::new(Vec::new()); + if on_retry { + outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), None, PaymentId([0; 32]), None, + &Route { paths: vec![], payment_params: None }, Some(Retry::Attempts(1)), + Some(route_params.payment_params.clone()), &&keys_manager, 0).unwrap(); + outbound_payments.retry_payment_internal( + PaymentHash([0; 32]), PaymentId([0; 32]), route_params, &&router, vec![], + &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, + &pending_events, &|_, _, _, _, _, _, _, _| Ok(())); + let events = pending_events.lock().unwrap(); + assert_eq!(events.len(), 1); + if let Event::PaymentFailed { .. } = events[0] { } else { panic!("Unexpected event"); } } else { - outbound_payments.send_payment( + let err = outbound_payments.send_payment( PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), route_params, - &&router, vec![], InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, |_, _, _, _, _, _, _, _, _| Ok(())).unwrap_err() + &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, + &pending_events, |_, _, _, _, _, _, _, _| Ok(())).unwrap_err(); + if let RetryableSendFailure::RouteNotFound = err { + } else { panic!("Unexpected error"); } + } + } + + #[test] + fn initial_send_payment_path_failed_evs() { + let outbound_payments = OutboundPayments::new(); + let logger = test_utils::TestLogger::new(); + let network_graph = Arc::new(NetworkGraph::new(Network::Testnet, &logger)); + let scorer = Mutex::new(test_utils::TestScorer::new()); + let router = test_utils::TestRouter::new(network_graph, &scorer); + let secp_ctx = Secp256k1::new(); + let keys_manager = test_utils::TestKeysInterface::new(&[0; 32], Network::Testnet); + + let sender_pk = PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap()); + let receiver_pk = PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap()); + let payment_params = PaymentParameters::from_node_id(sender_pk, 0); + let route_params = RouteParameters { + payment_params: payment_params.clone(), + final_value_msat: 0, + }; + let failed_scid = 42; + let route = Route { + paths: vec![vec![RouteHop { + pubkey: receiver_pk, + node_features: NodeFeatures::empty(), + short_channel_id: failed_scid, + channel_features: ChannelFeatures::empty(), + fee_msat: 0, + cltv_expiry_delta: 0, + }]], + payment_params: Some(payment_params), }; - if let PaymentSendFailure::ParameterError(APIError::APIMisuseError { err }) = err { - assert!(err.contains("Failed to find a route")); - } else { panic!("Unexpected error"); } + router.expect_find_route(route_params.clone(), Ok(route.clone())); + let mut route_params_w_failed_scid = route_params.clone(); + route_params_w_failed_scid.payment_params.previously_failed_channels.push(failed_scid); + router.expect_find_route(route_params_w_failed_scid, Ok(route.clone())); + router.expect_find_route(route_params.clone(), Ok(route.clone())); + router.expect_find_route(route_params.clone(), Ok(route.clone())); + + // Ensure that a ChannelUnavailable error will result in blaming an scid in the + // PaymentPathFailed event. + let pending_events = Mutex::new(Vec::new()); + outbound_payments.send_payment( + PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), route_params.clone(), + &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, + &pending_events, + |_, _, _, _, _, _, _, _| Err(APIError::ChannelUnavailable { err: "test".to_owned() })) + .unwrap(); + let mut events = pending_events.lock().unwrap(); + assert_eq!(events.len(), 2); + if let Event::PaymentPathFailed { + short_channel_id, + failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }}, .. } = events[0] + { + assert_eq!(short_channel_id, Some(failed_scid)); + } else { panic!("Unexpected event"); } + if let Event::PaymentFailed { .. } = events[1] { } else { panic!("Unexpected event"); } + events.clear(); + core::mem::drop(events); + + // Ensure that a MonitorUpdateInProgress "error" will not result in a PaymentPathFailed event. + outbound_payments.send_payment( + PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), route_params.clone(), + &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, + &pending_events, |_, _, _, _, _, _, _, _| Err(APIError::MonitorUpdateInProgress)) + .unwrap(); + { + let events = pending_events.lock().unwrap(); + assert_eq!(events.len(), 0); + } + + // Ensure that any other error will result in a PaymentPathFailed event but no blamed scid. + outbound_payments.send_payment( + PaymentHash([0; 32]), &None, PaymentId([1; 32]), Retry::Attempts(0), route_params.clone(), + &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger, + &pending_events, + |_, _, _, _, _, _, _, _| Err(APIError::APIMisuseError { err: "test".to_owned() })) + .unwrap(); + let events = pending_events.lock().unwrap(); + assert_eq!(events.len(), 2); + if let Event::PaymentPathFailed { + short_channel_id, + failure: PathFailure::InitialSend { err: APIError::APIMisuseError { .. }}, .. } = events[0] + { + assert_eq!(short_channel_id, None); + } else { panic!("Unexpected event"); } + if let Event::PaymentFailed { .. } = events[1] { } else { panic!("Unexpected event"); } } }