_ Git - rust-lightning/blob - lightning/src/ln/payment_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Tests that test the payment retry logic in ChannelManager, including various edge-cases around
  11 //! serialization ordering between ChannelManager/ChannelMonitors and ensuring we can still retry
  12 //! payments thereafter.
  13
  14 use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Listen, Watch};
  15 use crate::chain::channelmonitor::{ANTI_REORG_DELAY, LATENCY_GRACE_PERIOD_BLOCKS};
  16 use crate::chain::keysinterface::EntropySource;
  17 use crate::chain::transaction::OutPoint;
  18 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
  19 use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, ChannelManager, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS, RecentPaymentDetails};
  20 use crate::ln::features::InvoiceFeatures;
  21 use crate::ln::msgs;
  22 use crate::ln::msgs::ChannelMessageHandler;
  23 use crate::ln::outbound_payment::Retry;
  24 use crate::routing::gossip::RoutingFees;
  25 use crate::routing::router::{get_route, PaymentParameters, Route, RouteHint, RouteHintHop, RouteHop, RouteParameters};
  26 use crate::util::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
  27 use crate::util::test_utils;
  28 use crate::util::errors::APIError;
  29 use crate::util::ser::Writeable;
  30
  31 use bitcoin::{Block, BlockHeader, TxMerkleNode};
  32 use bitcoin::hashes::Hash;
  33 use bitcoin::network::constants::Network;
  34
  35 use crate::prelude::*;
  36
  37 use crate::ln::functional_test_utils::*;
  38 use crate::routing::gossip::NodeId;
  39 #[cfg(feature = "std")]
  40 use {
  41         crate::util::time::tests::SinceEpoch,
  42         std::time::{SystemTime, Duration}
  43 };
  44
  45 #[test]
  46 fn retry_single_path_payment() {
  47         let chanmon_cfgs = create_chanmon_cfgs(3);
  48         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
  49         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
  50         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
  51
  52         let _chan_0 = create_announced_chan_between_nodes(&nodes, 0, 1);
  53         let chan_1 = create_announced_chan_between_nodes(&nodes, 2, 1);
  54         // Rebalance to find a route
  55         send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000);
  56
  57         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 100_000);
  58
  59         // Rebalance so that the first hop fails.
  60         send_payment(&nodes[1], &vec!(&nodes[2])[..], 2_000_000);
  61
  62         // Make sure the payment fails on the first hop.
  63         let payment_id = PaymentId(payment_hash.0);
  64         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
  65         check_added_monitors!(nodes[0], 1);
  66         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
  67         assert_eq!(events.len(), 1);
  68         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
  69         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
  70         check_added_monitors!(nodes[1], 0);
  71         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  72         expect_pending_htlcs_forwardable!(nodes[1]);
  73         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_1.2 }]);
  74         let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
  75         assert!(htlc_updates.update_add_htlcs.is_empty());
  76         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
  77         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
  78         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
  79         check_added_monitors!(nodes[1], 1);
  80         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
  81         commitment_signed_dance!(nodes[0], nodes[1], htlc_updates.commitment_signed, false);
  82         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
  83
  84         // Rebalance the channel so the retry succeeds.
  85         send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000);
  86
  87         // Mine two blocks (we expire retries after 3, so this will check that we don't expire early)
  88         connect_blocks(&nodes[0], 2);
  89
  90         // Retry the payment and make sure it succeeds.
  91         nodes[0].node.retry_payment(&route, payment_id).unwrap();
  92         check_added_monitors!(nodes[0], 1);
  93         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
  94         assert_eq!(events.len(), 1);
  95         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 100_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
  96         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
  97 }
  98
  99 #[test]
 100 fn mpp_failure() {
 101         let chanmon_cfgs = create_chanmon_cfgs(4);
 102         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 103         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 104         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 105
 106         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
 107         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
 108         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
 109         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3).0.contents.short_channel_id;
 110
 111         let (mut route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
 112         let path = route.paths[0].clone();
 113         route.paths.push(path);
 114         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
 115         route.paths[0][0].short_channel_id = chan_1_id;
 116         route.paths[0][1].short_channel_id = chan_3_id;
 117         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
 118         route.paths[1][0].short_channel_id = chan_2_id;
 119         route.paths[1][1].short_channel_id = chan_4_id;
 120         send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
 121         fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
 122 }
 123
 124 #[test]
 125 fn mpp_retry() {
 126         let chanmon_cfgs = create_chanmon_cfgs(4);
 127         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 128         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 129         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 130
 131         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 132         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 133         let (chan_3_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 134         let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes(&nodes, 3, 2);
 135         // Rebalance
 136         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 137
 138         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 1_000_000);
 139         let path = route.paths[0].clone();
 140         route.paths.push(path);
 141         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
 142         route.paths[0][0].short_channel_id = chan_1_update.contents.short_channel_id;
 143         route.paths[0][1].short_channel_id = chan_3_update.contents.short_channel_id;
 144         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
 145         route.paths[1][0].short_channel_id = chan_2_update.contents.short_channel_id;
 146         route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id;
 147
 148         // Initiate the MPP payment.
 149         let payment_id = PaymentId(payment_hash.0);
 150         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
 151         check_added_monitors!(nodes[0], 2); // one monitor per path
 152         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 153         assert_eq!(events.len(), 2);
 154
 155         // Pass half of the payment along the success path.
 156         let (success_path_msgs, mut events) = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &events);
 157         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), success_path_msgs, false, None);
 158
 159         // Add the HTLC along the first hop.
 160         let (fail_path_msgs_1, _events) = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &events);
 161         let (update_add, commitment_signed) = match fail_path_msgs_1 {
 162                 MessageSendEvent::UpdateHTLCs { node_id: _, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 163                         assert_eq!(update_add_htlcs.len(), 1);
 164                         assert!(update_fail_htlcs.is_empty());
 165                         assert!(update_fulfill_htlcs.is_empty());
 166                         assert!(update_fail_malformed_htlcs.is_empty());
 167                         assert!(update_fee.is_none());
 168                         (update_add_htlcs[0].clone(), commitment_signed.clone())
 169                 },
 170                 _ => panic!("Unexpected event"),
 171         };
 172         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
 173         commitment_signed_dance!(nodes[2], nodes[0], commitment_signed, false);
 174
 175         // Attempt to forward the payment and complete the 2nd path's failure.
 176         expect_pending_htlcs_forwardable!(&nodes[2]);
 177         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id }]);
 178         let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 179         assert!(htlc_updates.update_add_htlcs.is_empty());
 180         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 181         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 182         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 183         check_added_monitors!(nodes[2], 1);
 184         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 185         commitment_signed_dance!(nodes[0], nodes[2], htlc_updates.commitment_signed, false);
 186         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 187
 188         // Rebalance the channel so the second half of the payment can succeed.
 189         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 190
 191         // Make sure it errors as expected given a too-large amount.
 192         if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, payment_id) {
 193                 assert!(err.contains("over total_payment_amt_msat"));
 194         } else { panic!("Unexpected error"); }
 195
 196         // Make sure it errors as expected given the wrong payment_id.
 197         if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, PaymentId([0; 32])) {
 198                 assert!(err.contains("not found"));
 199         } else { panic!("Unexpected error"); }
 200
 201         // Retry the second half of the payment and make sure it succeeds.
 202         let mut path = route.clone();
 203         path.paths.remove(0);
 204         nodes[0].node.retry_payment(&path, payment_id).unwrap();
 205         check_added_monitors!(nodes[0], 1);
 206         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 207         assert_eq!(events.len(), 1);
 208         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 209         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 210 }
 211
 212 fn do_mpp_receive_timeout(send_partial_mpp: bool) {
 213         let chanmon_cfgs = create_chanmon_cfgs(4);
 214         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 215         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 216         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 217
 218         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 219         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 220         let (chan_3_update, _, chan_3_id, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 221         let (chan_4_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 222
 223         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 100_000);
 224         let path = route.paths[0].clone();
 225         route.paths.push(path);
 226         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
 227         route.paths[0][0].short_channel_id = chan_1_update.contents.short_channel_id;
 228         route.paths[0][1].short_channel_id = chan_3_update.contents.short_channel_id;
 229         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
 230         route.paths[1][0].short_channel_id = chan_2_update.contents.short_channel_id;
 231         route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id;
 232
 233         // Initiate the MPP payment.
 234         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 235         check_added_monitors!(nodes[0], 2); // one monitor per path
 236         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 237         assert_eq!(events.len(), 2);
 238
 239         // Pass half of the payment along the first path.
 240         let (node_1_msgs, mut events) = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &events);
 241         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_1_msgs, false, None);
 242
 243         if send_partial_mpp {
 244                 // Time out the partial MPP
 245                 for _ in 0..MPP_TIMEOUT_TICKS {
 246                         nodes[3].node.timer_tick_occurred();
 247                 }
 248
 249                 // Failed HTLC from node 3 -> 1
 250                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 251                 let htlc_fail_updates_3_1 = get_htlc_update_msgs!(nodes[3], nodes[1].node.get_our_node_id());
 252                 assert_eq!(htlc_fail_updates_3_1.update_fail_htlcs.len(), 1);
 253                 nodes[1].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &htlc_fail_updates_3_1.update_fail_htlcs[0]);
 254                 check_added_monitors!(nodes[3], 1);
 255                 commitment_signed_dance!(nodes[1], nodes[3], htlc_fail_updates_3_1.commitment_signed, false);
 256
 257                 // Failed HTLC from node 1 -> 0
 258                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_3_id }]);
 259                 let htlc_fail_updates_1_0 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 260                 assert_eq!(htlc_fail_updates_1_0.update_fail_htlcs.len(), 1);
 261                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates_1_0.update_fail_htlcs[0]);
 262                 check_added_monitors!(nodes[1], 1);
 263                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates_1_0.commitment_signed, false);
 264
 265                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain().expected_htlc_error_data(23, &[][..]));
 266         } else {
 267                 // Pass half of the payment along the second path.
 268                 let (node_2_msgs, _events) = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &events);
 269                 pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_2_msgs, true, None);
 270
 271                 // Even after MPP_TIMEOUT_TICKS we should not timeout the MPP if we have all the parts
 272                 for _ in 0..MPP_TIMEOUT_TICKS {
 273                         nodes[3].node.timer_tick_occurred();
 274                 }
 275
 276                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 277         }
 278 }
 279
 280 #[test]
 281 fn mpp_receive_timeout() {
 282         do_mpp_receive_timeout(true);
 283         do_mpp_receive_timeout(false);
 284 }
 285
 286 #[test]
 287 fn retry_expired_payment() {
 288         let chanmon_cfgs = create_chanmon_cfgs(3);
 289         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 290         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 291         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 292
 293         let _chan_0 = create_announced_chan_between_nodes(&nodes, 0, 1);
 294         let chan_1 = create_announced_chan_between_nodes(&nodes, 2, 1);
 295         // Rebalance to find a route
 296         send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000);
 297
 298         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 100_000);
 299
 300         // Rebalance so that the first hop fails.
 301         send_payment(&nodes[1], &vec!(&nodes[2])[..], 2_000_000);
 302
 303         // Make sure the payment fails on the first hop.
 304         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 305         check_added_monitors!(nodes[0], 1);
 306         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 307         assert_eq!(events.len(), 1);
 308         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
 309         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 310         check_added_monitors!(nodes[1], 0);
 311         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 312         expect_pending_htlcs_forwardable!(nodes[1]);
 313         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_1.2 }]);
 314         let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 315         assert!(htlc_updates.update_add_htlcs.is_empty());
 316         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 317         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 318         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 319         check_added_monitors!(nodes[1], 1);
 320         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 321         commitment_signed_dance!(nodes[0], nodes[1], htlc_updates.commitment_signed, false);
 322         expect_payment_failed!(nodes[0], payment_hash, false);
 323
 324         // Mine blocks so the payment will have expired.
 325         connect_blocks(&nodes[0], 3);
 326
 327         // Retry the payment and make sure it errors as expected.
 328         if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, PaymentId(payment_hash.0)) {
 329                 assert!(err.contains("not found"));
 330         } else {
 331                 panic!("Unexpected error");
 332         }
 333 }
 334
 335 #[test]
 336 fn no_pending_leak_on_initial_send_failure() {
 337         // In an earlier version of our payment tracking, we'd have a retry entry even when the initial
 338         // HTLC for payment failed to send due to local channel errors (e.g. peer disconnected). In this
 339         // case, the user wouldn't have a PaymentId to retry the payment with, but we'd think we have a
 340         // pending payment forever and never time it out.
 341         // Here we test exactly that - retrying a payment when a peer was disconnected on the first
 342         // try, and then check that no pending payment is being tracked.
 343         let chanmon_cfgs = create_chanmon_cfgs(2);
 344         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 345         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 346         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 347
 348         create_announced_chan_between_nodes(&nodes, 0, 1);
 349
 350         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 351
 352         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 353         nodes[1].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 354
 355         unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)),
 356                 true, APIError::ChannelUnavailable { ref err },
 357                 assert_eq!(err, "Peer for first hop currently disconnected/pending monitor update!"));
 358
 359         assert!(!nodes[0].node.has_pending_payments());
 360 }
 361
 362 fn do_retry_with_no_persist(confirm_before_reload: bool) {
 363         // If we send a pending payment and `send_payment` returns success, we should always either
 364         // return a payment failure event or a payment success event, and on failure the payment should
 365         // be retryable.
 366         //
 367         // In order to do so when the ChannelManager isn't immediately persisted (which is normal - its
 368         // always persisted asynchronously), the ChannelManager has to reload some payment data from
 369         // ChannelMonitor(s) in some cases. This tests that reloading.
 370         //
 371         // `confirm_before_reload` confirms the channel-closing commitment transaction on-chain prior
 372         // to reloading the ChannelManager, increasing test coverage in ChannelMonitor HTLC tracking
 373         // which has separate codepaths for "commitment transaction already confirmed" and not.
 374         let chanmon_cfgs = create_chanmon_cfgs(3);
 375         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 376         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 377         let persister: test_utils::TestPersister;
 378         let new_chain_monitor: test_utils::TestChainMonitor;
 379         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 380         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 381
 382         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 383         let (_, _, chan_id_2, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
 384
 385         // Serialize the ChannelManager prior to sending payments
 386         let nodes_0_serialized = nodes[0].node.encode();
 387
 388         // Send two payments - one which will get to nodes[2] and will be claimed, one which we'll time
 389         // out and retry.
 390         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 391         let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000);
 392         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 393         check_added_monitors!(nodes[0], 1);
 394
 395         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 396         assert_eq!(events.len(), 1);
 397         let payment_event = SendEvent::from_event(events.pop().unwrap());
 398         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 399
 400         // We relay the payment to nodes[1] while its disconnected from nodes[2], causing the payment
 401         // to be returned immediately to nodes[0], without having nodes[2] fail the inbound payment
 402         // which would prevent retry.
 403         nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id(), false);
 404         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 405
 406         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 407         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true);
 408         // nodes[1] now immediately fails the HTLC as the next-hop channel is disconnected
 409         let _ = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 410
 411         reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 412
 413         let as_commitment_tx = get_local_commitment_txn!(nodes[0], chan_id)[0].clone();
 414         if confirm_before_reload {
 415                 mine_transaction(&nodes[0], &as_commitment_tx);
 416                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 417         }
 418
 419         // The ChannelMonitor should always be the latest version, as we're required to persist it
 420         // during the `commitment_signed_dance!()`.
 421         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 422         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 423
 424         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 425         // force-close the channel.
 426         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 427         assert!(nodes[0].node.list_channels().is_empty());
 428         assert!(nodes[0].node.has_pending_payments());
 429         let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 430         assert_eq!(as_broadcasted_txn.len(), 1);
 431         assert_eq!(as_broadcasted_txn[0], as_commitment_tx);
 432
 433         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 434         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
 435         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 436
 437         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 438         // error, as the channel has hit the chain.
 439         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
 440         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 441         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 442         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 443         assert_eq!(as_err.len(), 1);
 444         match as_err[0] {
 445                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 446                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 447                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 448                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()) });
 449                         check_added_monitors!(nodes[1], 1);
 450                         assert_eq!(nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 451                 },
 452                 _ => panic!("Unexpected event"),
 453         }
 454         check_closed_broadcast!(nodes[1], false);
 455
 456         // Now claim the first payment, which should allow nodes[1] to claim the payment on-chain when
 457         // we close in a moment.
 458         nodes[2].node.claim_funds(payment_preimage_1);
 459         check_added_monitors!(nodes[2], 1);
 460         expect_payment_claimed!(nodes[2], payment_hash_1, 1_000_000);
 461
 462         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 463         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 464         check_added_monitors!(nodes[1], 1);
 465         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 466         expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], None, false, false);
 467
 468         if confirm_before_reload {
 469                 let best_block = nodes[0].blocks.lock().unwrap().last().unwrap().clone();
 470                 nodes[0].node.best_block_updated(&best_block.0.header, best_block.1);
 471         }
 472
 473         // Create a new channel on which to retry the payment before we fail the payment via the
 474         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 475         // connecting several blocks while creating the channel (implying time has passed).
 476         create_announced_chan_between_nodes(&nodes, 0, 1);
 477         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 478
 479         mine_transaction(&nodes[1], &as_commitment_tx);
 480         let bs_htlc_claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 481         assert_eq!(bs_htlc_claim_txn.len(), 1);
 482         check_spends!(bs_htlc_claim_txn[0], as_commitment_tx);
 483
 484         if !confirm_before_reload {
 485                 mine_transaction(&nodes[0], &as_commitment_tx);
 486         }
 487         mine_transaction(&nodes[0], &bs_htlc_claim_txn[0]);
 488         expect_payment_sent!(nodes[0], payment_preimage_1);
 489         connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
 490         let as_htlc_timeout_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 491         assert_eq!(as_htlc_timeout_txn.len(), 2);
 492         let (first_htlc_timeout_tx, second_htlc_timeout_tx) = (&as_htlc_timeout_txn[0], &as_htlc_timeout_txn[1]);
 493         check_spends!(first_htlc_timeout_tx, as_commitment_tx);
 494         check_spends!(second_htlc_timeout_tx, as_commitment_tx);
 495         if first_htlc_timeout_tx.input[0].previous_output == bs_htlc_claim_txn[0].input[0].previous_output {
 496                 confirm_transaction(&nodes[0], &second_htlc_timeout_tx);
 497         } else {
 498                 confirm_transaction(&nodes[0], &first_htlc_timeout_tx);
 499         }
 500         nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 501         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 502
 503         // Finally, retry the payment (which was reloaded from the ChannelMonitor when nodes[0] was
 504         // reloaded) via a route over the new channel, which work without issue and eventually be
 505         // received and claimed at the recipient just like any other payment.
 506         let (mut new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 507
 508         // Update the fee on the middle hop to ensure PaymentSent events have the correct (retried) fee
 509         // and not the original fee. We also update node[1]'s relevant config as
 510         // do_claim_payment_along_route expects us to never overpay.
 511         {
 512                 let per_peer_state = nodes[1].node.per_peer_state.read().unwrap();
 513                 let mut peer_state = per_peer_state.get(&nodes[2].node.get_our_node_id())
 514                         .unwrap().lock().unwrap();
 515                 let mut channel = peer_state.channel_by_id.get_mut(&chan_id_2).unwrap();
 516                 let mut new_config = channel.config();
 517                 new_config.forwarding_fee_base_msat += 100_000;
 518                 channel.update_config(&new_config);
 519                 new_route.paths[0][0].fee_msat += 100_000;
 520         }
 521
 522         // Force expiration of the channel's previous config.
 523         for _ in 0..EXPIRE_PREV_CONFIG_TICKS {
 524                 nodes[1].node.timer_tick_occurred();
 525         }
 526
 527         assert!(nodes[0].node.retry_payment(&new_route, payment_id_1).is_err()); // Shouldn't be allowed to retry a fulfilled payment
 528         nodes[0].node.retry_payment(&new_route, PaymentId(payment_hash.0)).unwrap();
 529         check_added_monitors!(nodes[0], 1);
 530         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 531         assert_eq!(events.len(), 1);
 532         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 533         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
 534         expect_payment_sent!(nodes[0], payment_preimage, Some(new_route.paths[0][0].fee_msat));
 535 }
 536
 537 #[test]
 538 fn retry_with_no_persist() {
 539         do_retry_with_no_persist(true);
 540         do_retry_with_no_persist(false);
 541 }
 542
 543 fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 544         // Test that an off-chain completed payment is not retryable on restart. This was previously
 545         // broken for dust payments, but we test for both dust and non-dust payments.
 546         //
 547         // `use_dust` switches to using a dust HTLC, which results in the HTLC not having an on-chain
 548         // output at all.
 549         let chanmon_cfgs = create_chanmon_cfgs(3);
 550         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 551
 552         let mut manually_accept_config = test_default_channel_config();
 553         manually_accept_config.manually_accept_inbound_channels = true;
 554
 555         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(manually_accept_config), None]);
 556
 557         let first_persister: test_utils::TestPersister;
 558         let first_new_chain_monitor: test_utils::TestChainMonitor;
 559         let first_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 560         let second_persister: test_utils::TestPersister;
 561         let second_new_chain_monitor: test_utils::TestChainMonitor;
 562         let second_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 563         let third_persister: test_utils::TestPersister;
 564         let third_new_chain_monitor: test_utils::TestChainMonitor;
 565         let third_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 566
 567         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 568
 569         // Because we set nodes[1] to manually accept channels, just open a 0-conf channel.
 570         let (funding_tx, chan_id) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 571         confirm_transaction(&nodes[0], &funding_tx);
 572         confirm_transaction(&nodes[1], &funding_tx);
 573         // Ignore the announcement_signatures messages
 574         nodes[0].node.get_and_clear_pending_msg_events();
 575         nodes[1].node.get_and_clear_pending_msg_events();
 576         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 577
 578         // Serialize the ChannelManager prior to sending payments
 579         let mut nodes_0_serialized = nodes[0].node.encode();
 580
 581         let route = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 }).0;
 582         let (payment_preimage, payment_hash, payment_secret, payment_id) = send_along_route(&nodes[0], route, &[&nodes[1], &nodes[2]], if use_dust { 1_000 } else { 1_000_000 });
 583
 584         // The ChannelMonitor should always be the latest version, as we're required to persist it
 585         // during the `commitment_signed_dance!()`.
 586         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 587
 588         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized], first_persister, first_new_chain_monitor, first_nodes_0_deserialized);
 589         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 590
 591         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 592         // force-close the channel.
 593         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 594         assert!(nodes[0].node.list_channels().is_empty());
 595         assert!(nodes[0].node.has_pending_payments());
 596         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 597
 598         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
 599         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 600
 601         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 602         // error, as the channel has hit the chain.
 603         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
 604         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 605         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 606         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 607         assert_eq!(as_err.len(), 1);
 608         let bs_commitment_tx;
 609         match as_err[0] {
 610                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 611                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 612                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 613                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()) });
 614                         check_added_monitors!(nodes[1], 1);
 615                         bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 616                 },
 617                 _ => panic!("Unexpected event"),
 618         }
 619         check_closed_broadcast!(nodes[1], false);
 620
 621         // Now fail back the payment from nodes[2] to nodes[1]. This doesn't really matter as the
 622         // previous hop channel is already on-chain, but it makes nodes[2] willing to see additional
 623         // incoming HTLCs with the same payment hash later.
 624         nodes[2].node.fail_htlc_backwards(&payment_hash);
 625         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
 626         check_added_monitors!(nodes[2], 1);
 627
 628         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 629         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fail_htlcs[0]);
 630         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 631         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1],
 632                 [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 633
 634         // Connect the HTLC-Timeout transaction, timing out the HTLC on both nodes (but not confirming
 635         // the HTLC-Timeout transaction beyond 1 conf). For dust HTLCs, the HTLC is considered resolved
 636         // after the commitment transaction, so always connect the commitment transaction.
 637         mine_transaction(&nodes[0], &bs_commitment_tx[0]);
 638         mine_transaction(&nodes[1], &bs_commitment_tx[0]);
 639         if !use_dust {
 640                 connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1 + (MIN_CLTV_EXPIRY_DELTA as u32));
 641                 connect_blocks(&nodes[1], TEST_FINAL_CLTV - 1 + (MIN_CLTV_EXPIRY_DELTA as u32));
 642                 let as_htlc_timeout = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 643                 check_spends!(as_htlc_timeout[0], bs_commitment_tx[0]);
 644                 assert_eq!(as_htlc_timeout.len(), 1);
 645
 646                 mine_transaction(&nodes[0], &as_htlc_timeout[0]);
 647                 // nodes[0] may rebroadcast (or RBF-bump) its HTLC-Timeout, so wipe the announced set.
 648                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 649                 mine_transaction(&nodes[1], &as_htlc_timeout[0]);
 650         }
 651
 652         // Create a new channel on which to retry the payment before we fail the payment via the
 653         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 654         // connecting several blocks while creating the channel (implying time has passed).
 655         // We do this with a zero-conf channel to avoid connecting blocks as a side-effect.
 656         let (_, chan_id_3) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 657         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 658
 659         // If we attempt to retry prior to the HTLC-Timeout (or commitment transaction, for dust HTLCs)
 660         // confirming, we will fail as it's considered still-pending...
 661         let (new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 });
 662         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_err());
 663         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 664
 665         // After ANTI_REORG_DELAY confirmations, the HTLC should be failed and we can try the payment
 666         // again. We serialize the node first as we'll then test retrying the HTLC after a restart
 667         // (which should also still work).
 668         connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
 669         connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 670         // We set mpp_parts_remain to avoid having abandon_payment called
 671         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 672
 673         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 674         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 675         nodes_0_serialized = nodes[0].node.encode();
 676
 677         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_ok());
 678         assert!(!nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 679
 680         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], second_persister, second_new_chain_monitor, second_nodes_0_deserialized);
 681         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 682
 683         reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 684
 685         // Now resend the payment, delivering the HTLC and actually claiming it this time. This ensures
 686         // the payment is not (spuriously) listed as still pending.
 687         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_ok());
 688         check_added_monitors!(nodes[0], 1);
 689         pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], if use_dust { 1_000 } else { 1_000_000 }, payment_hash, payment_secret);
 690         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
 691
 692         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_err());
 693         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 694
 695         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 696         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 697         nodes_0_serialized = nodes[0].node.encode();
 698
 699         // Ensure that after reload we cannot retry the payment.
 700         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], third_persister, third_new_chain_monitor, third_nodes_0_deserialized);
 701         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 702
 703         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 704
 705         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_err());
 706         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 707 }
 708
 709 #[test]
 710 fn test_completed_payment_not_retryable_on_reload() {
 711         do_test_completed_payment_not_retryable_on_reload(true);
 712         do_test_completed_payment_not_retryable_on_reload(false);
 713 }
 714
 715
 716 fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, confirm_commitment_tx: bool, payment_timeout: bool) {
 717         // When a Channel is closed, any outbound HTLCs which were relayed through it are simply
 718         // dropped when the Channel is. From there, the ChannelManager relies on the ChannelMonitor
 719         // having a copy of the relevant fail-/claim-back data and processes the HTLC fail/claim when
 720         // the ChannelMonitor tells it to.
 721         //
 722         // If, due to an on-chain event, an HTLC is failed/claimed, we should avoid providing the
 723         // ChannelManager the HTLC event until after the monitor is re-persisted. This should prevent a
 724         // duplicate HTLC fail/claim (e.g. via a PaymentPathFailed event).
 725         let chanmon_cfgs = create_chanmon_cfgs(2);
 726         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 727         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 728         let persister: test_utils::TestPersister;
 729         let new_chain_monitor: test_utils::TestChainMonitor;
 730         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 731         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 732
 733         let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1);
 734
 735         // Route a payment, but force-close the channel before the HTLC fulfill message arrives at
 736         // nodes[0].
 737         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 10_000_000);
 738         nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap();
 739         check_closed_broadcast!(nodes[0], true);
 740         check_added_monitors!(nodes[0], 1);
 741         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 742
 743         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 744         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 745
 746         // Connect blocks until the CLTV timeout is up so that we get an HTLC-Timeout transaction
 747         connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
 748         let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 749         assert_eq!(node_txn.len(), 3);
 750         assert_eq!(node_txn[0], node_txn[1]);
 751         check_spends!(node_txn[1], funding_tx);
 752         check_spends!(node_txn[2], node_txn[1]);
 753         let timeout_txn = vec![node_txn[2].clone()];
 754
 755         nodes[1].node.claim_funds(payment_preimage);
 756         check_added_monitors!(nodes[1], 1);
 757         expect_payment_claimed!(nodes[1], payment_hash, 10_000_000);
 758
 759         let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 };
 760         connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[1].clone()]});
 761         check_closed_broadcast!(nodes[1], true);
 762         check_added_monitors!(nodes[1], 1);
 763         check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 764         let claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 765         assert_eq!(claim_txn.len(), 1);
 766         check_spends!(claim_txn[0], node_txn[1]);
 767
 768         header.prev_blockhash = nodes[0].best_block_hash();
 769         connect_block(&nodes[0], &Block { header, txdata: vec![node_txn[1].clone()]});
 770
 771         if confirm_commitment_tx {
 772                 connect_blocks(&nodes[0], BREAKDOWN_TIMEOUT as u32 - 1);
 773         }
 774
 775         header.prev_blockhash = nodes[0].best_block_hash();
 776         let claim_block = Block { header, txdata: if payment_timeout { timeout_txn } else { vec![claim_txn[0].clone()] } };
 777
 778         if payment_timeout {
 779                 assert!(confirm_commitment_tx); // Otherwise we're spending below our CSV!
 780                 connect_block(&nodes[0], &claim_block);
 781                 connect_blocks(&nodes[0], ANTI_REORG_DELAY - 2);
 782         }
 783
 784         // Now connect the HTLC claim transaction with the ChainMonitor-generated ChannelMonitor update
 785         // returning InProgress. This should cause the claim event to never make its way to the
 786         // ChannelManager.
 787         chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap().clear();
 788         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
 789
 790         if payment_timeout {
 791                 connect_blocks(&nodes[0], 1);
 792         } else {
 793                 connect_block(&nodes[0], &claim_block);
 794         }
 795
 796         let funding_txo = OutPoint { txid: funding_tx.txid(), index: 0 };
 797         let mon_updates: Vec<_> = chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap()
 798                 .get_mut(&funding_txo).unwrap().drain().collect();
 799         // If we are using chain::Confirm instead of chain::Listen, we will get the same update twice.
 800         // If we're testing connection idempotency we may get substantially more.
 801         assert!(mon_updates.len() >= 1);
 802         assert!(nodes[0].chain_monitor.release_pending_monitor_events().is_empty());
 803         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 804
 805         // If we persist the ChannelManager here, we should get the PaymentSent event after
 806         // deserialization.
 807         let mut chan_manager_serialized = Vec::new();
 808         if !persist_manager_post_event {
 809                 chan_manager_serialized = nodes[0].node.encode();
 810         }
 811
 812         // Now persist the ChannelMonitor and inform the ChainMonitor that we're done, generating the
 813         // payment sent event.
 814         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
 815         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 816         for update in mon_updates {
 817                 nodes[0].chain_monitor.chain_monitor.channel_monitor_updated(funding_txo, update).unwrap();
 818         }
 819         if payment_timeout {
 820                 expect_payment_failed!(nodes[0], payment_hash, false);
 821         } else {
 822                 expect_payment_sent!(nodes[0], payment_preimage);
 823         }
 824
 825         // If we persist the ChannelManager after we get the PaymentSent event, we shouldn't get it
 826         // twice.
 827         if persist_manager_post_event {
 828                 chan_manager_serialized = nodes[0].node.encode();
 829         }
 830
 831         // Now reload nodes[0]...
 832         reload_node!(nodes[0], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 833
 834         if persist_manager_post_event {
 835                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 836         } else if payment_timeout {
 837                 expect_payment_failed!(nodes[0], payment_hash, false);
 838         } else {
 839                 expect_payment_sent!(nodes[0], payment_preimage);
 840         }
 841
 842         // Note that if we re-connect the block which exposed nodes[0] to the payment preimage (but
 843         // which the current ChannelMonitor has not seen), the ChannelManager's de-duplication of
 844         // payment events should kick in, leaving us with no pending events here.
 845         let height = nodes[0].blocks.lock().unwrap().len() as u32 - 1;
 846         nodes[0].chain_monitor.chain_monitor.block_connected(&claim_block, height);
 847         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 848 }
 849
 850 #[test]
 851 fn test_dup_htlc_onchain_fails_on_reload() {
 852         do_test_dup_htlc_onchain_fails_on_reload(true, true, true);
 853         do_test_dup_htlc_onchain_fails_on_reload(true, true, false);
 854         do_test_dup_htlc_onchain_fails_on_reload(true, false, false);
 855         do_test_dup_htlc_onchain_fails_on_reload(false, true, true);
 856         do_test_dup_htlc_onchain_fails_on_reload(false, true, false);
 857         do_test_dup_htlc_onchain_fails_on_reload(false, false, false);
 858 }
 859
 860 #[test]
 861 fn test_fulfill_restart_failure() {
 862         // When we receive an update_fulfill_htlc message, we immediately consider the HTLC fully
 863         // fulfilled. At this point, the peer can reconnect and decide to either fulfill the HTLC
 864         // again, or fail it, giving us free money.
 865         //
 866         // Of course probably they won't fail it and give us free money, but because we have code to
 867         // handle it, we should test the logic for it anyway. We do that here.
 868         let chanmon_cfgs = create_chanmon_cfgs(2);
 869         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 870         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 871         let persister: test_utils::TestPersister;
 872         let new_chain_monitor: test_utils::TestChainMonitor;
 873         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 874         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 875
 876         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 877         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 100_000);
 878
 879         // The simplest way to get a failure after a fulfill is to reload nodes[1] from a state
 880         // pre-fulfill, which we do by serializing it here.
 881         let chan_manager_serialized = nodes[1].node.encode();
 882         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id).encode();
 883
 884         nodes[1].node.claim_funds(payment_preimage);
 885         check_added_monitors!(nodes[1], 1);
 886         expect_payment_claimed!(nodes[1], payment_hash, 100_000);
 887
 888         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 889         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 890         expect_payment_sent_without_paths!(nodes[0], payment_preimage);
 891
 892         // Now reload nodes[1]...
 893         reload_node!(nodes[1], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 894
 895         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 896         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 897
 898         nodes[1].node.fail_htlc_backwards(&payment_hash);
 899         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]);
 900         check_added_monitors!(nodes[1], 1);
 901         let htlc_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 902         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates.update_fail_htlcs[0]);
 903         commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates.commitment_signed, false);
 904         // nodes[0] shouldn't generate any events here, while it just got a payment failure completion
 905         // it had already considered the payment fulfilled, and now they just got free money.
 906         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 907 }
 908
 909 #[test]
 910 fn get_ldk_payment_preimage() {
 911         // Ensure that `ChannelManager::get_payment_preimage` can successfully be used to claim a payment.
 912         let chanmon_cfgs = create_chanmon_cfgs(2);
 913         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 914         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 915         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 916         create_announced_chan_between_nodes(&nodes, 0, 1);
 917
 918         let amt_msat = 60_000;
 919         let expiry_secs = 60 * 60;
 920         let (payment_hash, payment_secret) = nodes[1].node.create_inbound_payment(Some(amt_msat), expiry_secs, None).unwrap();
 921
 922         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
 923                 .with_features(nodes[1].node.invoice_features());
 924         let scorer = test_utils::TestScorer::with_penalty(0);
 925         let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet);
 926         let random_seed_bytes = keys_manager.get_secure_random_bytes();
 927         let route = get_route(
 928                 &nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(),
 929                 Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()),
 930                 amt_msat, TEST_FINAL_CLTV, nodes[0].logger, &scorer, &random_seed_bytes).unwrap();
 931         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 932         check_added_monitors!(nodes[0], 1);
 933
 934         // Make sure to use `get_payment_preimage`
 935         let payment_preimage = nodes[1].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
 936         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 937         assert_eq!(events.len(), 1);
 938         pass_along_path(&nodes[0], &[&nodes[1]], amt_msat, payment_hash, Some(payment_secret), events.pop().unwrap(), true, Some(payment_preimage));
 939         claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, payment_preimage);
 940 }
 941
 942 #[test]
 943 fn sent_probe_is_probe_of_sending_node() {
 944         let chanmon_cfgs = create_chanmon_cfgs(3);
 945         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 946         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
 947         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 948
 949         create_announced_chan_between_nodes(&nodes, 0, 1);
 950         create_announced_chan_between_nodes(&nodes, 1, 2);
 951
 952         // First check we refuse to build a single-hop probe
 953         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[1], 100_000);
 954         assert!(nodes[0].node.send_probe(route.paths[0].clone()).is_err());
 955
 956         // Then build an actual two-hop probing path
 957         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
 958
 959         match nodes[0].node.send_probe(route.paths[0].clone()) {
 960                 Ok((payment_hash, payment_id)) => {
 961                         assert!(nodes[0].node.payment_is_probe(&payment_hash, &payment_id));
 962                         assert!(!nodes[1].node.payment_is_probe(&payment_hash, &payment_id));
 963                         assert!(!nodes[2].node.payment_is_probe(&payment_hash, &payment_id));
 964                 },
 965                 _ => panic!(),
 966         }
 967
 968         get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 969         check_added_monitors!(nodes[0], 1);
 970 }
 971
 972 #[test]
 973 fn successful_probe_yields_event() {
 974         let chanmon_cfgs = create_chanmon_cfgs(3);
 975         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 976         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
 977         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 978
 979         create_announced_chan_between_nodes(&nodes, 0, 1);
 980         create_announced_chan_between_nodes(&nodes, 1, 2);
 981
 982         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
 983
 984         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
 985
 986         // node[0] -- update_add_htlcs -> node[1]
 987         check_added_monitors!(nodes[0], 1);
 988         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 989         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
 990         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
 991         check_added_monitors!(nodes[1], 0);
 992         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
 993         expect_pending_htlcs_forwardable!(nodes[1]);
 994
 995         // node[1] -- update_add_htlcs -> node[2]
 996         check_added_monitors!(nodes[1], 1);
 997         let updates = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 998         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
 999         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &probe_event.msgs[0]);
1000         check_added_monitors!(nodes[2], 0);
1001         commitment_signed_dance!(nodes[2], nodes[1], probe_event.commitment_msg, true, true);
1002
1003         // node[1] <- update_fail_htlcs -- node[2]
1004         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1005         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1006         check_added_monitors!(nodes[1], 0);
1007         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, true);
1008
1009         // node[0] <- update_fail_htlcs -- node[1]
1010         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1011         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1012         check_added_monitors!(nodes[0], 0);
1013         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1014
1015         let mut events = nodes[0].node.get_and_clear_pending_events();
1016         assert_eq!(events.len(), 1);
1017         match events.drain(..).next().unwrap() {
1018                 crate::util::events::Event::ProbeSuccessful { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1019                         assert_eq!(payment_id, ev_pid);
1020                         assert_eq!(payment_hash, ev_ph);
1021                 },
1022                 _ => panic!(),
1023         };
1024 }
1025
1026 #[test]
1027 fn failed_probe_yields_event() {
1028         let chanmon_cfgs = create_chanmon_cfgs(3);
1029         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1030         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1031         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1032
1033         create_announced_chan_between_nodes(&nodes, 0, 1);
1034         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 100000, 90000000);
1035
1036         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1037
1038         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 9_998_000, 42);
1039
1040         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1041
1042         // node[0] -- update_add_htlcs -> node[1]
1043         check_added_monitors!(nodes[0], 1);
1044         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1045         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1046         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1047         check_added_monitors!(nodes[1], 0);
1048         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1049         expect_pending_htlcs_forwardable!(nodes[1]);
1050
1051         // node[0] <- update_fail_htlcs -- node[1]
1052         check_added_monitors!(nodes[1], 1);
1053         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1054         // Skip the PendingHTLCsForwardable event
1055         let _events = nodes[1].node.get_and_clear_pending_events();
1056         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1057         check_added_monitors!(nodes[0], 0);
1058         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1059
1060         let mut events = nodes[0].node.get_and_clear_pending_events();
1061         assert_eq!(events.len(), 1);
1062         match events.drain(..).next().unwrap() {
1063                 crate::util::events::Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1064                         assert_eq!(payment_id, ev_pid);
1065                         assert_eq!(payment_hash, ev_ph);
1066                 },
1067                 _ => panic!(),
1068         };
1069 }
1070
1071 #[test]
1072 fn onchain_failed_probe_yields_event() {
1073         // Tests that an attempt to probe over a channel that is eventaully closed results in a failure
1074         // event.
1075         let chanmon_cfgs = create_chanmon_cfgs(3);
1076         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1077         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1078         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1079
1080         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1081         create_announced_chan_between_nodes(&nodes, 1, 2);
1082
1083         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1084
1085         // Send a dust HTLC, which will be treated as if it timed out once the channel hits the chain.
1086         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 1_000, 42);
1087         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1088
1089         // node[0] -- update_add_htlcs -> node[1]
1090         check_added_monitors!(nodes[0], 1);
1091         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1092         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1093         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1094         check_added_monitors!(nodes[1], 0);
1095         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1096         expect_pending_htlcs_forwardable!(nodes[1]);
1097
1098         check_added_monitors!(nodes[1], 1);
1099         let _ = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1100
1101         // Don't bother forwarding the HTLC onwards and just confirm the force-close transaction on
1102         // Node A, which after 6 confirmations should result in a probe failure event.
1103         let bs_txn = get_local_commitment_txn!(nodes[1], chan_id);
1104         confirm_transaction(&nodes[0], &bs_txn[0]);
1105         check_closed_broadcast!(&nodes[0], true);
1106         check_added_monitors!(nodes[0], 1);
1107
1108         let mut events = nodes[0].node.get_and_clear_pending_events();
1109         assert_eq!(events.len(), 2);
1110         let mut found_probe_failed = false;
1111         for event in events.drain(..) {
1112                 match event {
1113                         Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1114                                 assert_eq!(payment_id, ev_pid);
1115                                 assert_eq!(payment_hash, ev_ph);
1116                                 found_probe_failed = true;
1117                         },
1118                         Event::ChannelClosed { .. } => {},
1119                         _ => panic!(),
1120                 }
1121         }
1122         assert!(found_probe_failed);
1123 }
1124
1125 #[test]
1126 fn claimed_send_payment_idempotent() {
1127         // Tests that `send_payment` (and friends) are (reasonably) idempotent.
1128         let chanmon_cfgs = create_chanmon_cfgs(2);
1129         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1130         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1131         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1132
1133         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1134
1135         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1136         let (first_payment_preimage, _, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1137
1138         macro_rules! check_send_rejected {
1139                 () => {
1140                         // If we try to resend a new payment with a different payment_hash but with the same
1141                         // payment_id, it should be rejected.
1142                         let send_result = nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id);
1143                         match send_result {
1144                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1145                                 _ => panic!("Unexpected send result: {:?}", send_result),
1146                         }
1147
1148                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1149                         // also be rejected.
1150                         let send_result = nodes[0].node.send_spontaneous_payment(&route, None, payment_id);
1151                         match send_result {
1152                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1153                                 _ => panic!("Unexpected send result: {:?}", send_result),
1154                         }
1155                 }
1156         }
1157
1158         check_send_rejected!();
1159
1160         // Claim the payment backwards, but note that the PaymentSent event is still pending and has
1161         // not been seen by the user. At this point, from the user perspective nothing has changed, so
1162         // we must remain just as idempotent as we were before.
1163         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, first_payment_preimage);
1164
1165         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1166                 nodes[0].node.timer_tick_occurred();
1167         }
1168
1169         check_send_rejected!();
1170
1171         // Once the user sees and handles the `PaymentSent` event, we expect them to no longer call
1172         // `send_payment`, and our idempotency guarantees are off - they should have atomically marked
1173         // the payment complete. However, they could have called `send_payment` while the event was
1174         // being processed, leading to a race in our idempotency guarantees. Thus, even immediately
1175         // after the event is handled a duplicate payment should sitll be rejected.
1176         expect_payment_sent!(&nodes[0], first_payment_preimage, Some(0));
1177         check_send_rejected!();
1178
1179         // If relatively little time has passed, a duplicate payment should still fail.
1180         nodes[0].node.timer_tick_occurred();
1181         check_send_rejected!();
1182
1183         // However, after some time has passed (at least more than the one timer tick above), a
1184         // duplicate payment should go through, as ChannelManager should no longer have any remaining
1185         // references to the old payment data.
1186         for _ in 0..IDEMPOTENCY_TIMEOUT_TICKS {
1187                 nodes[0].node.timer_tick_occurred();
1188         }
1189
1190         nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id).unwrap();
1191         check_added_monitors!(nodes[0], 1);
1192         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1193         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1194 }
1195
1196 #[test]
1197 fn abandoned_send_payment_idempotent() {
1198         // Tests that `send_payment` (and friends) allow duplicate PaymentIds immediately after
1199         // abandon_payment.
1200         let chanmon_cfgs = create_chanmon_cfgs(2);
1201         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1202         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1203         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1204
1205         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1206
1207         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1208         let (_, first_payment_hash, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1209
1210         macro_rules! check_send_rejected {
1211                 () => {
1212                         // If we try to resend a new payment with a different payment_hash but with the same
1213                         // payment_id, it should be rejected.
1214                         let send_result = nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id);
1215                         match send_result {
1216                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1217                                 _ => panic!("Unexpected send result: {:?}", send_result),
1218                         }
1219
1220                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1221                         // also be rejected.
1222                         let send_result = nodes[0].node.send_spontaneous_payment(&route, None, payment_id);
1223                         match send_result {
1224                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1225                                 _ => panic!("Unexpected send result: {:?}", send_result),
1226                         }
1227                 }
1228         }
1229
1230         check_send_rejected!();
1231
1232         nodes[1].node.fail_htlc_backwards(&first_payment_hash);
1233         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash: first_payment_hash }]);
1234
1235         pass_failed_payment_back_no_abandon(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash);
1236         check_send_rejected!();
1237
1238         // Until we abandon the payment, no matter how many timer ticks pass, we still cannot reuse the
1239         // PaymentId.
1240         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1241                 nodes[0].node.timer_tick_occurred();
1242         }
1243         check_send_rejected!();
1244
1245         nodes[0].node.abandon_payment(payment_id);
1246         get_event!(nodes[0], Event::PaymentFailed);
1247
1248         // However, we can reuse the PaymentId immediately after we `abandon_payment`.
1249         nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id).unwrap();
1250         check_added_monitors!(nodes[0], 1);
1251         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1252         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1253 }
1254
1255 #[derive(PartialEq)]
1256 enum InterceptTest {
1257         Forward,
1258         Fail,
1259         Timeout,
1260 }
1261
1262 #[test]
1263 fn test_trivial_inflight_htlc_tracking(){
1264         // In this test, we test three scenarios:
1265         // (1) Sending + claiming a payment successfully should return `None` when querying InFlightHtlcs
1266         // (2) Sending a payment without claiming it should return the payment's value (500000) when querying InFlightHtlcs
1267         // (3) After we claim the payment sent in (2), InFlightHtlcs should return `None` for the query.
1268         let chanmon_cfgs = create_chanmon_cfgs(3);
1269         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1270         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1271         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1272
1273         let (_, _, chan_1_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
1274         let (_, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
1275
1276         // Send and claim the payment. Inflight HTLCs should be empty.
1277         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 500000);
1278         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
1279         check_added_monitors!(nodes[0], 1);
1280         pass_along_route(&nodes[0], &[&vec!(&nodes[1], &nodes[2])[..]], 500000, payment_hash, payment_secret);
1281         claim_payment(&nodes[0], &vec!(&nodes[1], &nodes[2])[..], payment_preimage);
1282         {
1283                 let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1284
1285                 let mut node_0_per_peer_lock;
1286                 let mut node_0_peer_state_lock;
1287                 let mut node_1_per_peer_lock;
1288                 let mut node_1_peer_state_lock;
1289                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1290                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1291
1292                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1293                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1294                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1295                         channel_1.get_short_channel_id().unwrap()
1296                 );
1297                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1298                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1299                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1300                         channel_2.get_short_channel_id().unwrap()
1301                 );
1302
1303                 assert_eq!(chan_1_used_liquidity, None);
1304                 assert_eq!(chan_2_used_liquidity, None);
1305         }
1306         let pending_payments = nodes[0].node.list_recent_payments();
1307         assert_eq!(pending_payments.len(), 1);
1308         assert_eq!(pending_payments[0], RecentPaymentDetails::Fulfilled { payment_hash: Some(payment_hash) });
1309
1310         // Remove fulfilled payment
1311         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1312                 nodes[0].node.timer_tick_occurred();
1313         }
1314
1315         // Send the payment, but do not claim it. Our inflight HTLCs should contain the pending payment.
1316         let (payment_preimage, payment_hash,  _) = route_payment(&nodes[0], &vec!(&nodes[1], &nodes[2])[..], 500000);
1317         {
1318                 let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1319
1320                 let mut node_0_per_peer_lock;
1321                 let mut node_0_peer_state_lock;
1322                 let mut node_1_per_peer_lock;
1323                 let mut node_1_peer_state_lock;
1324                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1325                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1326
1327                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1328                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1329                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1330                         channel_1.get_short_channel_id().unwrap()
1331                 );
1332                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1333                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1334                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1335                         channel_2.get_short_channel_id().unwrap()
1336                 );
1337
1338                 // First hop accounts for expected 1000 msat fee
1339                 assert_eq!(chan_1_used_liquidity, Some(501000));
1340                 assert_eq!(chan_2_used_liquidity, Some(500000));
1341         }
1342         let pending_payments = nodes[0].node.list_recent_payments();
1343         assert_eq!(pending_payments.len(), 1);
1344         assert_eq!(pending_payments[0], RecentPaymentDetails::Pending { payment_hash, total_msat: 500000 });
1345
1346         // Now, let's claim the payment. This should result in the used liquidity to return `None`.
1347         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
1348
1349         // Remove fulfilled payment
1350         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1351                 nodes[0].node.timer_tick_occurred();
1352         }
1353
1354         {
1355                 let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1356
1357                 let mut node_0_per_peer_lock;
1358                 let mut node_0_peer_state_lock;
1359                 let mut node_1_per_peer_lock;
1360                 let mut node_1_peer_state_lock;
1361                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1362                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1363
1364                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1365                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1366                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1367                         channel_1.get_short_channel_id().unwrap()
1368                 );
1369                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1370                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1371                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1372                         channel_2.get_short_channel_id().unwrap()
1373                 );
1374
1375                 assert_eq!(chan_1_used_liquidity, None);
1376                 assert_eq!(chan_2_used_liquidity, None);
1377         }
1378
1379         let pending_payments = nodes[0].node.list_recent_payments();
1380         assert_eq!(pending_payments.len(), 0);
1381 }
1382
1383 #[test]
1384 fn test_holding_cell_inflight_htlcs() {
1385         let chanmon_cfgs = create_chanmon_cfgs(2);
1386         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1387         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1388         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1389         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1390
1391         let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
1392         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1393
1394         // Queue up two payments - one will be delivered right away, one immediately goes into the
1395         // holding cell as nodes[0] is AwaitingRAA.
1396         {
1397                 nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
1398                 check_added_monitors!(nodes[0], 1);
1399                 nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
1400                 check_added_monitors!(nodes[0], 0);
1401         }
1402
1403         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1404
1405         {
1406                 let mut node_0_per_peer_lock;
1407                 let mut node_0_peer_state_lock;
1408                 let channel =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, channel_id);
1409
1410                 let used_liquidity = inflight_htlcs.used_liquidity_msat(
1411                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1412                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1413                         channel.get_short_channel_id().unwrap()
1414                 );
1415
1416                 assert_eq!(used_liquidity, Some(2000000));
1417         }
1418
1419         // Clear pending events so test doesn't throw a "Had excess message on node..." error
1420         nodes[0].node.get_and_clear_pending_msg_events();
1421 }
1422
1423 #[test]
1424 fn intercepted_payment() {
1425         // Test that detecting an intercept scid on payment forward will signal LDK to generate an
1426         // intercept event, which the LSP can then use to either (a) open a JIT channel to forward the
1427         // payment or (b) fail the payment.
1428         do_test_intercepted_payment(InterceptTest::Forward);
1429         do_test_intercepted_payment(InterceptTest::Fail);
1430         // Make sure that intercepted payments will be automatically failed back if too many blocks pass.
1431         do_test_intercepted_payment(InterceptTest::Timeout);
1432 }
1433
1434 fn do_test_intercepted_payment(test: InterceptTest) {
1435         let chanmon_cfgs = create_chanmon_cfgs(3);
1436         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1437
1438         let mut zero_conf_chan_config = test_default_channel_config();
1439         zero_conf_chan_config.manually_accept_inbound_channels = true;
1440         let mut intercept_forwards_config = test_default_channel_config();
1441         intercept_forwards_config.accept_intercept_htlcs = true;
1442         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(zero_conf_chan_config)]);
1443
1444         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1445         let scorer = test_utils::TestScorer::with_penalty(0);
1446         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
1447
1448         let _ = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1449
1450         let amt_msat = 100_000;
1451         let intercept_scid = nodes[1].node.get_intercept_scid();
1452         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1453                 .with_route_hints(vec![
1454                         RouteHint(vec![RouteHintHop {
1455                                 src_node_id: nodes[1].node.get_our_node_id(),
1456                                 short_channel_id: intercept_scid,
1457                                 fees: RoutingFees {
1458                                         base_msat: 1000,
1459                                         proportional_millionths: 0,
1460                                 },
1461                                 cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
1462                                 htlc_minimum_msat: None,
1463                                 htlc_maximum_msat: None,
1464                         }])
1465                 ])
1466                 .with_features(nodes[2].node.invoice_features());
1467         let route_params = RouteParameters {
1468                 payment_params,
1469                 final_value_msat: amt_msat,
1470                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
1471         };
1472         let route = get_route(
1473                 &nodes[0].node.get_our_node_id(), &route_params.payment_params,
1474                 &nodes[0].network_graph.read_only(), None, route_params.final_value_msat,
1475                 route_params.final_cltv_expiry_delta, nodes[0].logger, &scorer, &random_seed_bytes
1476         ).unwrap();
1477
1478         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
1479         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
1480         let payment_event = {
1481                 {
1482                         let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
1483                         assert_eq!(added_monitors.len(), 1);
1484                         added_monitors.clear();
1485                 }
1486                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1487                 assert_eq!(events.len(), 1);
1488                 SendEvent::from_event(events.remove(0))
1489         };
1490         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1491         commitment_signed_dance!(nodes[1], nodes[0], &payment_event.commitment_msg, false, true);
1492
1493         // Check that we generate the PaymentIntercepted event when an intercept forward is detected.
1494         let events = nodes[1].node.get_and_clear_pending_events();
1495         assert_eq!(events.len(), 1);
1496         let (intercept_id, expected_outbound_amount_msat) = match events[0] {
1497                 crate::util::events::Event::HTLCIntercepted {
1498                         intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, inbound_amount_msat, requested_next_hop_scid: short_channel_id
1499                 } => {
1500                         assert_eq!(pmt_hash, payment_hash);
1501                         assert_eq!(inbound_amount_msat, route.get_total_amount() + route.get_total_fees());
1502                         assert_eq!(short_channel_id, intercept_scid);
1503                         (intercept_id, expected_outbound_amount_msat)
1504                 },
1505                 _ => panic!()
1506         };
1507
1508         // Check for unknown channel id error.
1509         let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &[42; 32], nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1510         assert_eq!(unknown_chan_id_err , APIError::ChannelUnavailable  { err: format!("Channel with id {} not found for the passed counterparty node_id {}", log_bytes!([42; 32]), nodes[2].node.get_our_node_id()) });
1511
1512         if test == InterceptTest::Fail {
1513                 // Ensure we can fail the intercepted payment back.
1514                 nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap();
1515                 expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::UnknownNextHop { requested_forward_scid: intercept_scid }]);
1516                 nodes[1].node.process_pending_htlc_forwards();
1517                 let update_fail = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1518                 check_added_monitors!(&nodes[1], 1);
1519                 assert!(update_fail.update_fail_htlcs.len() == 1);
1520                 let fail_msg = update_fail.update_fail_htlcs[0].clone();
1521                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1522                 commitment_signed_dance!(nodes[0], nodes[1], update_fail.commitment_signed, false);
1523
1524                 // Ensure the payment fails with the expected error.
1525                 let fail_conditions = PaymentFailedConditions::new()
1526                         .blamed_scid(intercept_scid)
1527                         .blamed_chan_closed(true)
1528                         .expected_htlc_error_data(0x4000 | 10, &[]);
1529                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
1530         } else if test == InterceptTest::Forward {
1531                 // Check that we'll fail as expected when sending to a channel that isn't in `ChannelReady` yet.
1532                 let temp_chan_id = nodes[1].node.create_channel(nodes[2].node.get_our_node_id(), 100_000, 0, 42, None).unwrap();
1533                 let unusable_chan_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &temp_chan_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1534                 assert_eq!(unusable_chan_err , APIError::ChannelUnavailable { err: format!("Channel with id {} not fully established", log_bytes!(temp_chan_id)) });
1535                 assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
1536
1537                 // Open the just-in-time channel so the payment can then be forwarded.
1538                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1539
1540                 // Finally, forward the intercepted payment through and claim it.
1541                 nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap();
1542                 expect_pending_htlcs_forwardable!(nodes[1]);
1543
1544                 let payment_event = {
1545                         {
1546                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
1547                                 assert_eq!(added_monitors.len(), 1);
1548                                 added_monitors.clear();
1549                         }
1550                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
1551                         assert_eq!(events.len(), 1);
1552                         SendEvent::from_event(events.remove(0))
1553                 };
1554                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1555                 commitment_signed_dance!(nodes[2], nodes[1], &payment_event.commitment_msg, false, true);
1556                 expect_pending_htlcs_forwardable!(nodes[2]);
1557
1558                 let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1559                 expect_payment_claimable!(&nodes[2], payment_hash, payment_secret, amt_msat, Some(payment_preimage), nodes[2].node.get_our_node_id());
1560                 do_claim_payment_along_route(&nodes[0], &vec!(&vec!(&nodes[1], &nodes[2])[..]), false, payment_preimage);
1561                 let events = nodes[0].node.get_and_clear_pending_events();
1562                 assert_eq!(events.len(), 2);
1563                 match events[0] {
1564                         Event::PaymentSent { payment_preimage: ref ev_preimage, payment_hash: ref ev_hash, ref fee_paid_msat, .. } => {
1565                                 assert_eq!(payment_preimage, *ev_preimage);
1566                                 assert_eq!(payment_hash, *ev_hash);
1567                                 assert_eq!(fee_paid_msat, &Some(1000));
1568                         },
1569                         _ => panic!("Unexpected event")
1570                 }
1571                 match events[1] {
1572                         Event::PaymentPathSuccessful { payment_hash: hash, .. } => {
1573                                 assert_eq!(hash, Some(payment_hash));
1574                         },
1575                         _ => panic!("Unexpected event")
1576                 }
1577         } else if test == InterceptTest::Timeout {
1578                 let mut block = Block {
1579                         header: BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 },
1580                         txdata: vec![],
1581                 };
1582                 connect_block(&nodes[0], &block);
1583                 connect_block(&nodes[1], &block);
1584                 for _ in 0..TEST_FINAL_CLTV {
1585                         block.header.prev_blockhash = block.block_hash();
1586                         connect_block(&nodes[0], &block);
1587                         connect_block(&nodes[1], &block);
1588                 }
1589                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::InvalidForward { requested_forward_scid: intercept_scid }]);
1590                 check_added_monitors!(nodes[1], 1);
1591                 let htlc_timeout_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1592                 assert!(htlc_timeout_updates.update_add_htlcs.is_empty());
1593                 assert_eq!(htlc_timeout_updates.update_fail_htlcs.len(), 1);
1594                 assert!(htlc_timeout_updates.update_fail_malformed_htlcs.is_empty());
1595                 assert!(htlc_timeout_updates.update_fee.is_none());
1596
1597                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_timeout_updates.update_fail_htlcs[0]);
1598                 commitment_signed_dance!(nodes[0], nodes[1], htlc_timeout_updates.commitment_signed, false);
1599                 expect_payment_failed!(nodes[0], payment_hash, false, 0x2000 | 2, []);
1600
1601                 // Check for unknown intercept id error.
1602                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1603                 let unknown_intercept_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1604                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1605                 let unknown_intercept_id_err = nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap_err();
1606                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1607         }
1608 }
1609
1610 #[derive(PartialEq)]
1611 enum AutoRetry {
1612         Success,
1613         FailAttempts,
1614         FailTimeout,
1615         FailOnRestart,
1616 }
1617
1618 #[test]
1619 fn automatic_retries() {
1620         do_automatic_retries(AutoRetry::Success);
1621         do_automatic_retries(AutoRetry::FailAttempts);
1622         do_automatic_retries(AutoRetry::FailTimeout);
1623         do_automatic_retries(AutoRetry::FailOnRestart);
1624 }
1625 fn do_automatic_retries(test: AutoRetry) {
1626         // Test basic automatic payment retries in ChannelManager. See individual `test` variant comments
1627         // below.
1628         let chanmon_cfgs = create_chanmon_cfgs(3);
1629         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1630         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1631
1632         let persister;
1633         let new_chain_monitor;
1634         let node_0_deserialized;
1635
1636         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1637         let channel_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1638         let channel_id_2 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1639
1640         // Marshall data to send the payment
1641         #[cfg(feature = "std")]
1642         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1643         #[cfg(not(feature = "std"))]
1644         let payment_expiry_secs = 60 * 60;
1645         let amt_msat = 1000;
1646         let mut invoice_features = InvoiceFeatures::empty();
1647         invoice_features.set_variable_length_onion_required();
1648         invoice_features.set_payment_secret_required();
1649         invoice_features.set_basic_mpp_optional();
1650         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1651                 .with_expiry_time(payment_expiry_secs as u64)
1652                 .with_features(invoice_features);
1653         let route_params = RouteParameters {
1654                 payment_params,
1655                 final_value_msat: amt_msat,
1656                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
1657         };
1658         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
1659
1660         macro_rules! pass_failed_attempt_with_retry_along_path {
1661                 ($failing_channel_id: expr, $expect_pending_htlcs_forwardable: expr) => {
1662                         // Send a payment attempt that fails due to lack of liquidity on the second hop
1663                         check_added_monitors!(nodes[0], 1);
1664                         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1665                         let mut update_add = update_0.update_add_htlcs[0].clone();
1666                         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
1667                         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
1668                         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
1669                         nodes[1].node.process_pending_htlc_forwards();
1670                         expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1],
1671                                 vec![HTLCDestination::NextHopChannel {
1672                                         node_id: Some(nodes[2].node.get_our_node_id()),
1673                                         channel_id: $failing_channel_id,
1674                                 }]);
1675                         nodes[1].node.process_pending_htlc_forwards();
1676                         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1677                         check_added_monitors!(&nodes[1], 1);
1678                         assert!(update_1.update_fail_htlcs.len() == 1);
1679                         let fail_msg = update_1.update_fail_htlcs[0].clone();
1680
1681                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1682                         commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
1683
1684                         // Ensure the attempt fails and a new PendingHTLCsForwardable event is generated for the retry
1685                         let mut events = nodes[0].node.get_and_clear_pending_events();
1686                         match events[0] {
1687                                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
1688                                         assert_eq!(payment_hash, ev_payment_hash);
1689                                         assert_eq!(payment_failed_permanently, false);
1690                                 },
1691                                 _ => panic!("Unexpected event"),
1692                         }
1693                         if $expect_pending_htlcs_forwardable {
1694                                 assert_eq!(events.len(), 2);
1695                                 match events[1] {
1696                                         Event::PendingHTLCsForwardable { .. } => {},
1697                                         _ => panic!("Unexpected event"),
1698                                 }
1699                         } else { assert_eq!(events.len(), 1) }
1700                 }
1701         }
1702
1703         if test == AutoRetry::Success {
1704                 // Test that we can succeed on the first retry.
1705                 nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1706                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1707
1708                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
1709                 // attempt, since the initial second hop channel will be excluded from pathfinding
1710                 create_announced_chan_between_nodes(&nodes, 1, 2);
1711
1712                 // We retry payments in `process_pending_htlc_forwards`
1713                 nodes[0].node.process_pending_htlc_forwards();
1714                 check_added_monitors!(nodes[0], 1);
1715                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1716                 assert_eq!(msg_events.len(), 1);
1717                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, Some(payment_secret), msg_events.pop().unwrap(), true, None);
1718                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
1719         } else if test == AutoRetry::FailAttempts {
1720                 // Ensure ChannelManager will not retry a payment if it has run out of payment attempts.
1721                 nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1722                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1723
1724                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
1725                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
1726                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1727
1728                 // We retry payments in `process_pending_htlc_forwards`
1729                 nodes[0].node.process_pending_htlc_forwards();
1730                 pass_failed_attempt_with_retry_along_path!(channel_id_3, false);
1731
1732                 // Ensure we won't retry a second time.
1733                 nodes[0].node.process_pending_htlc_forwards();
1734                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1735                 assert_eq!(msg_events.len(), 0);
1736
1737                 nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
1738                 let events = nodes[0].node.get_and_clear_pending_events();
1739                 assert_eq!(events.len(), 1);
1740                 match events[0] {
1741                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
1742                                 assert_eq!(payment_hash, *ev_payment_hash);
1743                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1744                         },
1745                         _ => panic!("Unexpected event"),
1746                 }
1747         } else if test == AutoRetry::FailTimeout {
1748                 #[cfg(not(feature = "no-std"))] {
1749                         // Ensure ChannelManager will not retry a payment if it times out due to Retry::Timeout.
1750                         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
1751                         pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1752
1753                         // Advance the time so the second attempt fails due to timeout.
1754                         SinceEpoch::advance(Duration::from_secs(61));
1755
1756                         // Make sure we don't retry again.
1757                         nodes[0].node.process_pending_htlc_forwards();
1758                         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1759                         assert_eq!(msg_events.len(), 0);
1760
1761                         nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
1762                         let mut events = nodes[0].node.get_and_clear_pending_events();
1763                         assert_eq!(events.len(), 1);
1764                         match events[0] {
1765                                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
1766                                         assert_eq!(payment_hash, *ev_payment_hash);
1767                                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1768                                 },
1769                                 _ => panic!("Unexpected event"),
1770                         }
1771                 }
1772         } else if test == AutoRetry::FailOnRestart {
1773                 // Ensure ChannelManager will not retry a payment after restart, even if there were retry
1774                 // attempts remaining prior to restart.
1775                 nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
1776                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1777
1778                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
1779                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
1780                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1781
1782                 // Ensure the first retry attempt fails, with 1 retry attempt remaining
1783                 nodes[0].node.process_pending_htlc_forwards();
1784                 pass_failed_attempt_with_retry_along_path!(channel_id_3, true);
1785
1786                 // Restart the node and ensure that ChannelManager does not use its remaining retry attempt
1787                 let node_encoded = nodes[0].node.encode();
1788                 let chan_1_monitor_serialized = get_monitor!(nodes[0], channel_id_1).encode();
1789                 reload_node!(nodes[0], node_encoded, &[&chan_1_monitor_serialized], persister, new_chain_monitor, node_0_deserialized);
1790
1791                 // Make sure we don't retry again.
1792                 nodes[0].node.process_pending_htlc_forwards();
1793                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1794                 assert_eq!(msg_events.len(), 0);
1795
1796                 nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
1797                 let mut events = nodes[0].node.get_and_clear_pending_events();
1798                 assert_eq!(events.len(), 1);
1799                 match events[0] {
1800                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
1801                                 assert_eq!(payment_hash, *ev_payment_hash);
1802                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1803                         },
1804                         _ => panic!("Unexpected event"),
1805                 }
1806         }
1807 }
1808
1809 #[test]
1810 fn auto_retry_partial_failure() {
1811         // Test that we'll retry appropriately on send partial failure and retry partial failure.
1812         let chanmon_cfgs = create_chanmon_cfgs(2);
1813         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1814         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1815         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1816
1817         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1818         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1819         let chan_3_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1820
1821         // Marshall data to send the payment
1822         let amt_msat = 20_000;
1823         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
1824         #[cfg(feature = "std")]
1825         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1826         #[cfg(not(feature = "std"))]
1827         let payment_expiry_secs = 60 * 60;
1828         let mut invoice_features = InvoiceFeatures::empty();
1829         invoice_features.set_variable_length_onion_required();
1830         invoice_features.set_payment_secret_required();
1831         invoice_features.set_basic_mpp_optional();
1832         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
1833                 .with_expiry_time(payment_expiry_secs as u64)
1834                 .with_features(invoice_features);
1835         let route_params = RouteParameters {
1836                 payment_params,
1837                 final_value_msat: amt_msat,
1838                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
1839         };
1840
1841         // Ensure the first monitor update (for the initial send path1 over chan_1) succeeds, but the
1842         // second (for the initial send path2 over chan_2) fails.
1843         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1844         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
1845         // Ensure third monitor update (for the retry1's path1 over chan_1) succeeds, but the fourth (for
1846         // the retry1's path2 over chan_3) fails, and monitor updates succeed after that.
1847         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1848         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
1849         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1850
1851         // Configure the initial send, retry1 and retry2's paths.
1852         let send_route = Route {
1853                 paths: vec![
1854                         vec![RouteHop {
1855                                 pubkey: nodes[1].node.get_our_node_id(),
1856                                 node_features: nodes[1].node.node_features(),
1857                                 short_channel_id: chan_1_id,
1858                                 channel_features: nodes[1].node.channel_features(),
1859                                 fee_msat: amt_msat / 2,
1860                                 cltv_expiry_delta: 100,
1861                         }],
1862                         vec![RouteHop {
1863                                 pubkey: nodes[1].node.get_our_node_id(),
1864                                 node_features: nodes[1].node.node_features(),
1865                                 short_channel_id: chan_2_id,
1866                                 channel_features: nodes[1].node.channel_features(),
1867                                 fee_msat: amt_msat / 2,
1868                                 cltv_expiry_delta: 100,
1869                         }],
1870                 ],
1871                 payment_params: Some(route_params.payment_params.clone()),
1872         };
1873         let retry_1_route = Route {
1874                 paths: vec![
1875                         vec![RouteHop {
1876                                 pubkey: nodes[1].node.get_our_node_id(),
1877                                 node_features: nodes[1].node.node_features(),
1878                                 short_channel_id: chan_1_id,
1879                                 channel_features: nodes[1].node.channel_features(),
1880                                 fee_msat: amt_msat / 4,
1881                                 cltv_expiry_delta: 100,
1882                         }],
1883                         vec![RouteHop {
1884                                 pubkey: nodes[1].node.get_our_node_id(),
1885                                 node_features: nodes[1].node.node_features(),
1886                                 short_channel_id: chan_3_id,
1887                                 channel_features: nodes[1].node.channel_features(),
1888                                 fee_msat: amt_msat / 4,
1889                                 cltv_expiry_delta: 100,
1890                         }],
1891                 ],
1892                 payment_params: Some(route_params.payment_params.clone()),
1893         };
1894         let retry_2_route = Route {
1895                 paths: vec![
1896                         vec![RouteHop {
1897                                 pubkey: nodes[1].node.get_our_node_id(),
1898                                 node_features: nodes[1].node.node_features(),
1899                                 short_channel_id: chan_1_id,
1900                                 channel_features: nodes[1].node.channel_features(),
1901                                 fee_msat: amt_msat / 4,
1902                                 cltv_expiry_delta: 100,
1903                         }],
1904                 ],
1905                 payment_params: Some(route_params.payment_params.clone()),
1906         };
1907         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
1908         nodes[0].router.expect_find_route(RouteParameters {
1909                         payment_params: route_params.payment_params.clone(),
1910                         final_value_msat: amt_msat / 2, final_cltv_expiry_delta: TEST_FINAL_CLTV
1911                 }, Ok(retry_1_route));
1912         nodes[0].router.expect_find_route(RouteParameters {
1913                         payment_params: route_params.payment_params.clone(),
1914                         final_value_msat: amt_msat / 4, final_cltv_expiry_delta: TEST_FINAL_CLTV
1915                 }, Ok(retry_2_route));
1916
1917         // Send a payment that will partially fail on send, then partially fail on retry, then succeed.
1918         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap();
1919         let closed_chan_events = nodes[0].node.get_and_clear_pending_events();
1920         assert_eq!(closed_chan_events.len(), 2);
1921         match closed_chan_events[0] {
1922                 Event::ChannelClosed { .. } => {},
1923                 _ => panic!("Unexpected event"),
1924         }
1925         match closed_chan_events[1] {
1926                 Event::ChannelClosed { .. } => {},
1927                 _ => panic!("Unexpected event"),
1928         }
1929
1930         // Pass the first part of the payment along the path.
1931         check_added_monitors!(nodes[0], 5); // three outbound channel updates succeeded, two permanently failed
1932         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1933
1934         // First message is the first update_add, remaining messages are broadcasting channel updates and
1935         // errors for the permfailed channels
1936         assert_eq!(msg_events.len(), 5);
1937         let mut payment_event = SendEvent::from_event(msg_events.remove(0));
1938
1939         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1940         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1941         check_added_monitors!(nodes[1], 1);
1942         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1943
1944         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
1945         check_added_monitors!(nodes[0], 1);
1946         let as_second_htlc_updates = SendEvent::from_node(&nodes[0]);
1947
1948         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
1949         check_added_monitors!(nodes[0], 1);
1950         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1951
1952         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
1953         check_added_monitors!(nodes[1], 1);
1954
1955         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[0]);
1956         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[1]);
1957         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.commitment_msg);
1958         check_added_monitors!(nodes[1], 1);
1959         let (bs_second_raa, bs_second_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1960
1961         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
1962         check_added_monitors!(nodes[0], 1);
1963
1964         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_cs);
1965         check_added_monitors!(nodes[0], 1);
1966         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1967
1968         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
1969         check_added_monitors!(nodes[1], 1);
1970
1971         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
1972         nodes[1].node.process_pending_htlc_forwards();
1973         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, amt_msat);
1974         nodes[1].node.claim_funds(payment_preimage);
1975         expect_payment_claimed!(nodes[1], payment_hash, amt_msat);
1976         let bs_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1977         assert_eq!(bs_claim_update.update_fulfill_htlcs.len(), 1);
1978
1979         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_claim_update.update_fulfill_htlcs[0]);
1980         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_claim_update.commitment_signed);
1981         check_added_monitors!(nodes[0], 1);
1982         let (as_third_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1983
1984         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
1985         check_added_monitors!(nodes[1], 4);
1986         let bs_second_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1987
1988         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
1989         check_added_monitors!(nodes[1], 1);
1990         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1991
1992         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
1993         check_added_monitors!(nodes[0], 1);
1994
1995         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[0]);
1996         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[1]);
1997         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.commitment_signed);
1998         check_added_monitors!(nodes[0], 1);
1999         let (as_fourth_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2000
2001         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_fourth_raa);
2002         check_added_monitors!(nodes[1], 1);
2003
2004         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2005         check_added_monitors!(nodes[1], 1);
2006         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2007
2008         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2009         check_added_monitors!(nodes[0], 1);
2010         expect_payment_sent!(nodes[0], payment_preimage);
2011 }
2012
2013 #[test]
2014 fn auto_retry_zero_attempts_send_error() {
2015         let chanmon_cfgs = create_chanmon_cfgs(2);
2016         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2017         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2018         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2019
2020         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2021         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2022
2023         // Marshall data to send the payment
2024         let amt_msat = 20_000;
2025         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2026         #[cfg(feature = "std")]
2027         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2028         #[cfg(not(feature = "std"))]
2029         let payment_expiry_secs = 60 * 60;
2030         let mut invoice_features = InvoiceFeatures::empty();
2031         invoice_features.set_variable_length_onion_required();
2032         invoice_features.set_payment_secret_required();
2033         invoice_features.set_basic_mpp_optional();
2034         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2035                 .with_expiry_time(payment_expiry_secs as u64)
2036                 .with_features(invoice_features);
2037         let route_params = RouteParameters {
2038                 payment_params,
2039                 final_value_msat: amt_msat,
2040                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2041         };
2042
2043         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2044         let err = nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap_err();
2045         if let PaymentSendFailure::AllFailedResendSafe(_) = err {
2046         } else { panic!("Unexpected error"); }
2047         assert_eq!(nodes[0].node.get_and_clear_pending_msg_events().len(), 2); // channel close messages
2048         assert_eq!(nodes[0].node.get_and_clear_pending_events().len(), 1); // channel close event
2049         check_added_monitors!(nodes[0], 2);
2050 }
2051
2052 #[test]
2053 fn fails_paying_after_rejected_by_payee() {
2054         let chanmon_cfgs = create_chanmon_cfgs(2);
2055         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2056         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2057         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2058
2059         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2060
2061         // Marshall data to send the payment
2062         let amt_msat = 20_000;
2063         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2064         #[cfg(feature = "std")]
2065         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2066         #[cfg(not(feature = "std"))]
2067         let payment_expiry_secs = 60 * 60;
2068         let mut invoice_features = InvoiceFeatures::empty();
2069         invoice_features.set_variable_length_onion_required();
2070         invoice_features.set_payment_secret_required();
2071         invoice_features.set_basic_mpp_optional();
2072         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2073                 .with_expiry_time(payment_expiry_secs as u64)
2074                 .with_features(invoice_features);
2075         let route_params = RouteParameters {
2076                 payment_params,
2077                 final_value_msat: amt_msat,
2078                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2079         };
2080
2081         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2082         check_added_monitors!(nodes[0], 1);
2083         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
2084         assert_eq!(events.len(), 1);
2085         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
2086         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2087         check_added_monitors!(nodes[1], 0);
2088         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
2089         expect_pending_htlcs_forwardable!(nodes[1]);
2090         expect_payment_claimable!(&nodes[1], payment_hash, payment_secret, amt_msat);
2091
2092         nodes[1].node.fail_htlc_backwards(&payment_hash);
2093         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash }]);
2094         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, payment_hash);
2095 }
2096
2097 #[test]
2098 fn retry_multi_path_single_failed_payment() {
2099         // Tests that we can/will retry after a single path of an MPP payment failed immediately
2100         let chanmon_cfgs = create_chanmon_cfgs(2);
2101         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2102         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2103         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2104
2105         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2106         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2107
2108         let amt_msat = 100_010_000;
2109
2110         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2111         #[cfg(feature = "std")]
2112         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2113         #[cfg(not(feature = "std"))]
2114         let payment_expiry_secs = 60 * 60;
2115         let mut invoice_features = InvoiceFeatures::empty();
2116         invoice_features.set_variable_length_onion_required();
2117         invoice_features.set_payment_secret_required();
2118         invoice_features.set_basic_mpp_optional();
2119         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2120                 .with_expiry_time(payment_expiry_secs as u64)
2121                 .with_features(invoice_features);
2122         let route_params = RouteParameters {
2123                 payment_params: payment_params.clone(),
2124                 final_value_msat: amt_msat,
2125                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2126         };
2127
2128         let chans = nodes[0].node.list_usable_channels();
2129         let mut route = Route {
2130                 paths: vec![
2131                         vec![RouteHop {
2132                                 pubkey: nodes[1].node.get_our_node_id(),
2133                                 node_features: nodes[1].node.node_features(),
2134                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2135                                 channel_features: nodes[1].node.channel_features(),
2136                                 fee_msat: 10_000,
2137                                 cltv_expiry_delta: 100,
2138                         }],
2139                         vec![RouteHop {
2140                                 pubkey: nodes[1].node.get_our_node_id(),
2141                                 node_features: nodes[1].node.node_features(),
2142                                 short_channel_id: chans[1].short_channel_id.unwrap(),
2143                                 channel_features: nodes[1].node.channel_features(),
2144                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2145                                 cltv_expiry_delta: 100,
2146                         }],
2147                 ],
2148                 payment_params: Some(payment_params),
2149         };
2150         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2151         // On retry, split the payment across both channels.
2152         route.paths[0][0].fee_msat = 50_000_001;
2153         route.paths[1][0].fee_msat = 50_000_000;
2154         nodes[0].router.expect_find_route(RouteParameters {
2155                         payment_params: route.payment_params.clone().unwrap(),
2156                         // Note that the second request here requests the amount we originally failed to send,
2157                         // not the amount remaining on the full payment, which should be changed.
2158                         final_value_msat: 100_000_001, final_cltv_expiry_delta: TEST_FINAL_CLTV
2159                 }, Ok(route.clone()));
2160
2161         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2162         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2163         assert_eq!(htlc_msgs.len(), 2);
2164         check_added_monitors!(nodes[0], 2);
2165 }
2166
2167 #[test]
2168 fn immediate_retry_on_failure() {
2169         // Tests that we can/will retry immediately after a failure
2170         let chanmon_cfgs = create_chanmon_cfgs(2);
2171         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2172         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2173         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2174
2175         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2176         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2177
2178         let amt_msat = 100_000_001;
2179         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2180         #[cfg(feature = "std")]
2181         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2182         #[cfg(not(feature = "std"))]
2183         let payment_expiry_secs = 60 * 60;
2184         let mut invoice_features = InvoiceFeatures::empty();
2185         invoice_features.set_variable_length_onion_required();
2186         invoice_features.set_payment_secret_required();
2187         invoice_features.set_basic_mpp_optional();
2188         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2189                 .with_expiry_time(payment_expiry_secs as u64)
2190                 .with_features(invoice_features);
2191         let route_params = RouteParameters {
2192                 payment_params,
2193                 final_value_msat: amt_msat,
2194                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2195         };
2196
2197         let chans = nodes[0].node.list_usable_channels();
2198         let mut route = Route {
2199                 paths: vec![
2200                         vec![RouteHop {
2201                                 pubkey: nodes[1].node.get_our_node_id(),
2202                                 node_features: nodes[1].node.node_features(),
2203                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2204                                 channel_features: nodes[1].node.channel_features(),
2205                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2206                                 cltv_expiry_delta: 100,
2207                         }],
2208                 ],
2209                 payment_params: Some(PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)),
2210         };
2211         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2212         // On retry, split the payment across both channels.
2213         route.paths.push(route.paths[0].clone());
2214         route.paths[0][0].short_channel_id = chans[1].short_channel_id.unwrap();
2215         route.paths[0][0].fee_msat = 50_000_000;
2216         route.paths[1][0].fee_msat = 50_000_001;
2217         nodes[0].router.expect_find_route(RouteParameters {
2218                         payment_params: route_params.payment_params.clone(),
2219                         final_value_msat: amt_msat, final_cltv_expiry_delta: TEST_FINAL_CLTV
2220                 }, Ok(route.clone()));
2221
2222         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2223         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2224         assert_eq!(htlc_msgs.len(), 2);
2225         check_added_monitors!(nodes[0], 2);
2226 }
2227
2228 #[test]
2229 fn no_extra_retries_on_back_to_back_fail() {
2230         // In a previous release, we had a race where we may exceed the payment retry count if we
2231         // get two failures in a row with the second having `all_paths_failed` set.
2232         // Generally, when we give up trying to retry a payment, we don't know for sure what the
2233         // current state of the ChannelManager event queue is. Specifically, we cannot be sure that
2234         // there are not multiple additional `PaymentPathFailed` or even `PaymentSent` events
2235         // pending which we will see later. Thus, when we previously removed the retry tracking map
2236         // entry after a `all_paths_failed` `PaymentPathFailed` event, we may have dropped the
2237         // retry entry even though more events for the same payment were still pending. This led to
2238         // us retrying a payment again even though we'd already given up on it.
2239         //
2240         // We now have a separate event - `PaymentFailed` which indicates no HTLCs remain and which
2241         // is used to remove the payment retry counter entries instead. This tests for the specific
2242         // excess-retry case while also testing `PaymentFailed` generation.
2243
2244         let chanmon_cfgs = create_chanmon_cfgs(3);
2245         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2246         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2247         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2248
2249         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2250         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2251
2252         let amt_msat = 200_000_000;
2253         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2254         #[cfg(feature = "std")]
2255         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2256         #[cfg(not(feature = "std"))]
2257         let payment_expiry_secs = 60 * 60;
2258         let mut invoice_features = InvoiceFeatures::empty();
2259         invoice_features.set_variable_length_onion_required();
2260         invoice_features.set_payment_secret_required();
2261         invoice_features.set_basic_mpp_optional();
2262         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2263                 .with_expiry_time(payment_expiry_secs as u64)
2264                 .with_features(invoice_features);
2265         let route_params = RouteParameters {
2266                 payment_params,
2267                 final_value_msat: amt_msat,
2268                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2269         };
2270
2271         let mut route = Route {
2272                 paths: vec![
2273                         vec![RouteHop {
2274                                 pubkey: nodes[1].node.get_our_node_id(),
2275                                 node_features: nodes[1].node.node_features(),
2276                                 short_channel_id: chan_1_scid,
2277                                 channel_features: nodes[1].node.channel_features(),
2278                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2279                                 cltv_expiry_delta: 100,
2280                         }, RouteHop {
2281                                 pubkey: nodes[2].node.get_our_node_id(),
2282                                 node_features: nodes[2].node.node_features(),
2283                                 short_channel_id: chan_2_scid,
2284                                 channel_features: nodes[2].node.channel_features(),
2285                                 fee_msat: 100_000_000,
2286                                 cltv_expiry_delta: 100,
2287                         }],
2288                         vec![RouteHop {
2289                                 pubkey: nodes[1].node.get_our_node_id(),
2290                                 node_features: nodes[1].node.node_features(),
2291                                 short_channel_id: chan_1_scid,
2292                                 channel_features: nodes[1].node.channel_features(),
2293                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2294                                 cltv_expiry_delta: 100,
2295                         }, RouteHop {
2296                                 pubkey: nodes[2].node.get_our_node_id(),
2297                                 node_features: nodes[2].node.node_features(),
2298                                 short_channel_id: chan_2_scid,
2299                                 channel_features: nodes[2].node.channel_features(),
2300                                 fee_msat: 100_000_000,
2301                                 cltv_expiry_delta: 100,
2302                         }]
2303                 ],
2304                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2305         };
2306         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2307         let mut second_payment_params = route_params.payment_params.clone();
2308         second_payment_params.previously_failed_channels = vec![chan_2_scid, chan_2_scid];
2309         // On retry, we'll only return one path
2310         route.paths.remove(1);
2311         route.paths[0][1].fee_msat = amt_msat;
2312         nodes[0].router.expect_find_route(RouteParameters {
2313                         payment_params: second_payment_params,
2314                         final_value_msat: amt_msat, final_cltv_expiry_delta: TEST_FINAL_CLTV,
2315                 }, Ok(route.clone()));
2316
2317         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2318         let htlc_updates = SendEvent::from_node(&nodes[0]);
2319         check_added_monitors!(nodes[0], 1);
2320         assert_eq!(htlc_updates.msgs.len(), 1);
2321
2322         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2323         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2324         check_added_monitors!(nodes[1], 1);
2325         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2326
2327         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2328         check_added_monitors!(nodes[0], 1);
2329         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2330
2331         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2332         check_added_monitors!(nodes[0], 1);
2333         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2334
2335         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2336         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2337         check_added_monitors!(nodes[1], 1);
2338         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2339
2340         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2341         check_added_monitors!(nodes[1], 1);
2342         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2343
2344         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2345         check_added_monitors!(nodes[0], 1);
2346
2347         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2348         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2349         check_added_monitors!(nodes[0], 1);
2350         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2351
2352         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2353         check_added_monitors!(nodes[1], 1);
2354         let bs_second_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2355
2356         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2357         check_added_monitors!(nodes[1], 1);
2358         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2359
2360         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.update_fail_htlcs[0]);
2361         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.commitment_signed);
2362         check_added_monitors!(nodes[0], 1);
2363
2364         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2365         check_added_monitors!(nodes[0], 1);
2366         let (as_third_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2367
2368         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2369         check_added_monitors!(nodes[1], 1);
2370         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2371         check_added_monitors!(nodes[1], 1);
2372         let bs_fourth_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2373
2374         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_fourth_raa);
2375         check_added_monitors!(nodes[0], 1);
2376
2377         // At this point A has sent two HTLCs which both failed due to lack of fee. It now has two
2378         // pending `PaymentPathFailed` events, one with `all_paths_failed` unset, and the second
2379         // with it set.
2380         //
2381         // Previously, we retried payments in an event consumer, which would retry each
2382         // `PaymentPathFailed` individually. In that setup, we had retried the payment in response to
2383         // the first `PaymentPathFailed`, then seen the second `PaymentPathFailed` with
2384         // `all_paths_failed` set and assumed the payment was completely failed. We ultimately fixed it
2385         // by adding the `PaymentFailed` event.
2386         //
2387         // Because we now retry payments as a batch, we simply return a single-path route in the
2388         // second, batched, request, have that fail, then complete the payment via `abandon_payment`.
2389         let mut events = nodes[0].node.get_and_clear_pending_events();
2390         assert_eq!(events.len(), 4);
2391         match events[0] {
2392                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2393                         assert_eq!(payment_hash, ev_payment_hash);
2394                         assert_eq!(payment_failed_permanently, false);
2395                 },
2396                 _ => panic!("Unexpected event"),
2397         }
2398         match events[1] {
2399                 Event::PendingHTLCsForwardable { .. } => {},
2400                 _ => panic!("Unexpected event"),
2401         }
2402         match events[2] {
2403                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2404                         assert_eq!(payment_hash, ev_payment_hash);
2405                         assert_eq!(payment_failed_permanently, false);
2406                 },
2407                 _ => panic!("Unexpected event"),
2408         }
2409         match events[3] {
2410                 Event::PendingHTLCsForwardable { .. } => {},
2411                 _ => panic!("Unexpected event"),
2412         }
2413
2414         nodes[0].node.process_pending_htlc_forwards();
2415         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2416         check_added_monitors!(nodes[0], 1);
2417
2418         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2419         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2420         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2421         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2422         commitment_signed_dance!(nodes[0], nodes[1], &bs_fail_update.commitment_signed, false, true);
2423
2424         let mut events = nodes[0].node.get_and_clear_pending_events();
2425         assert_eq!(events.len(), 1);
2426         match events[0] {
2427                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2428                         assert_eq!(payment_hash, ev_payment_hash);
2429                         assert_eq!(payment_failed_permanently, false);
2430                 },
2431                 _ => panic!("Unexpected event"),
2432         }
2433         nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
2434         events = nodes[0].node.get_and_clear_pending_events();
2435         assert_eq!(events.len(), 1);
2436         match events[0] {
2437                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
2438                         assert_eq!(payment_hash, *ev_payment_hash);
2439                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2440                 },
2441                 _ => panic!("Unexpected event"),
2442         }
2443 }
2444
2445 #[test]
2446 fn test_simple_partial_retry() {
2447         // In the first version of the in-`ChannelManager` payment retries, retries were sent for the
2448         // full amount of the payment, rather than only the missing amount. Here we simply test for
2449         // this by sending a payment with two parts, failing one, and retrying the second. Note that
2450         // `TestRouter` will check that the `RouteParameters` (which contain the amount) matches the
2451         // request.
2452         let chanmon_cfgs = create_chanmon_cfgs(3);
2453         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2454         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2455         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2456
2457         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2458         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2459
2460         let amt_msat = 200_000_000;
2461         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
2462         #[cfg(feature = "std")]
2463         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2464         #[cfg(not(feature = "std"))]
2465         let payment_expiry_secs = 60 * 60;
2466         let mut invoice_features = InvoiceFeatures::empty();
2467         invoice_features.set_variable_length_onion_required();
2468         invoice_features.set_payment_secret_required();
2469         invoice_features.set_basic_mpp_optional();
2470         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2471                 .with_expiry_time(payment_expiry_secs as u64)
2472                 .with_features(invoice_features);
2473         let route_params = RouteParameters {
2474                 payment_params,
2475                 final_value_msat: amt_msat,
2476                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2477         };
2478
2479         let mut route = Route {
2480                 paths: vec![
2481                         vec![RouteHop {
2482                                 pubkey: nodes[1].node.get_our_node_id(),
2483                                 node_features: nodes[1].node.node_features(),
2484                                 short_channel_id: chan_1_scid,
2485                                 channel_features: nodes[1].node.channel_features(),
2486                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2487                                 cltv_expiry_delta: 100,
2488                         }, RouteHop {
2489                                 pubkey: nodes[2].node.get_our_node_id(),
2490                                 node_features: nodes[2].node.node_features(),
2491                                 short_channel_id: chan_2_scid,
2492                                 channel_features: nodes[2].node.channel_features(),
2493                                 fee_msat: 100_000_000,
2494                                 cltv_expiry_delta: 100,
2495                         }],
2496                         vec![RouteHop {
2497                                 pubkey: nodes[1].node.get_our_node_id(),
2498                                 node_features: nodes[1].node.node_features(),
2499                                 short_channel_id: chan_1_scid,
2500                                 channel_features: nodes[1].node.channel_features(),
2501                                 fee_msat: 100_000,
2502                                 cltv_expiry_delta: 100,
2503                         }, RouteHop {
2504                                 pubkey: nodes[2].node.get_our_node_id(),
2505                                 node_features: nodes[2].node.node_features(),
2506                                 short_channel_id: chan_2_scid,
2507                                 channel_features: nodes[2].node.channel_features(),
2508                                 fee_msat: 100_000_000,
2509                                 cltv_expiry_delta: 100,
2510                         }]
2511                 ],
2512                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2513         };
2514         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2515         let mut second_payment_params = route_params.payment_params.clone();
2516         second_payment_params.previously_failed_channels = vec![chan_2_scid];
2517         // On retry, we'll only be asked for one path (or 100k sats)
2518         route.paths.remove(0);
2519         nodes[0].router.expect_find_route(RouteParameters {
2520                         payment_params: second_payment_params,
2521                         final_value_msat: amt_msat / 2, final_cltv_expiry_delta: TEST_FINAL_CLTV,
2522                 }, Ok(route.clone()));
2523
2524         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2525         let htlc_updates = SendEvent::from_node(&nodes[0]);
2526         check_added_monitors!(nodes[0], 1);
2527         assert_eq!(htlc_updates.msgs.len(), 1);
2528
2529         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2530         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2531         check_added_monitors!(nodes[1], 1);
2532         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2533
2534         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2535         check_added_monitors!(nodes[0], 1);
2536         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2537
2538         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2539         check_added_monitors!(nodes[0], 1);
2540         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2541
2542         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2543         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2544         check_added_monitors!(nodes[1], 1);
2545         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2546
2547         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2548         check_added_monitors!(nodes[1], 1);
2549         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2550
2551         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2552         check_added_monitors!(nodes[0], 1);
2553
2554         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2555         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2556         check_added_monitors!(nodes[0], 1);
2557         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2558
2559         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2560         check_added_monitors!(nodes[1], 1);
2561
2562         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2563         check_added_monitors!(nodes[1], 1);
2564
2565         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2566
2567         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2568         check_added_monitors!(nodes[0], 1);
2569
2570         let mut events = nodes[0].node.get_and_clear_pending_events();
2571         assert_eq!(events.len(), 2);
2572         match events[0] {
2573                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2574                         assert_eq!(payment_hash, ev_payment_hash);
2575                         assert_eq!(payment_failed_permanently, false);
2576                 },
2577                 _ => panic!("Unexpected event"),
2578         }
2579         match events[1] {
2580                 Event::PendingHTLCsForwardable { .. } => {},
2581                 _ => panic!("Unexpected event"),
2582         }
2583
2584         nodes[0].node.process_pending_htlc_forwards();
2585         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2586         check_added_monitors!(nodes[0], 1);
2587
2588         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2589         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2590
2591         expect_pending_htlcs_forwardable!(nodes[1]);
2592         check_added_monitors!(nodes[1], 1);
2593
2594         let bs_forward_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
2595         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[0]);
2596         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[1]);
2597         commitment_signed_dance!(nodes[2], nodes[1], &bs_forward_update.commitment_signed, false);
2598
2599         expect_pending_htlcs_forwardable!(nodes[2]);
2600         expect_payment_claimable!(nodes[2], payment_hash, payment_secret, amt_msat);
2601 }