_ Git - rust-lightning/blob - lightning/src/ln/payment_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Tests that test the payment retry logic in ChannelManager, including various edge-cases around
  11 //! serialization ordering between ChannelManager/ChannelMonitors and ensuring we can still retry
  12 //! payments thereafter.
  13
  14 use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Listen, Watch};
  15 use crate::chain::channelmonitor::{ANTI_REORG_DELAY, LATENCY_GRACE_PERIOD_BLOCKS};
  16 use crate::chain::keysinterface::EntropySource;
  17 use crate::chain::transaction::OutPoint;
  18 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
  19 use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, ChannelManager, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS, RecentPaymentDetails};
  20 use crate::ln::features::InvoiceFeatures;
  21 use crate::ln::msgs;
  22 use crate::ln::msgs::ChannelMessageHandler;
  23 use crate::ln::outbound_payment::Retry;
  24 use crate::routing::gossip::RoutingFees;
  25 use crate::routing::router::{get_route, PaymentParameters, Route, RouteHint, RouteHintHop, RouteHop, RouteParameters};
  26 use crate::util::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
  27 use crate::util::test_utils;
  28 use crate::util::errors::APIError;
  29 use crate::util::ser::Writeable;
  30
  31 use bitcoin::{Block, BlockHeader, TxMerkleNode};
  32 use bitcoin::hashes::Hash;
  33 use bitcoin::network::constants::Network;
  34
  35 use crate::prelude::*;
  36
  37 use crate::ln::functional_test_utils::*;
  38 use crate::routing::gossip::NodeId;
  39 #[cfg(feature = "std")]
  40 use {
  41         crate::util::time::tests::SinceEpoch,
  42         std::time::{SystemTime, Duration}
  43 };
  44
  45 #[test]
  46 fn retry_single_path_payment() {
  47         let chanmon_cfgs = create_chanmon_cfgs(3);
  48         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
  49         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
  50         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
  51
  52         let _chan_0 = create_announced_chan_between_nodes(&nodes, 0, 1);
  53         let chan_1 = create_announced_chan_between_nodes(&nodes, 2, 1);
  54         // Rebalance to find a route
  55         send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000);
  56
  57         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 100_000);
  58
  59         // Rebalance so that the first hop fails.
  60         send_payment(&nodes[1], &vec!(&nodes[2])[..], 2_000_000);
  61
  62         // Make sure the payment fails on the first hop.
  63         let payment_id = PaymentId(payment_hash.0);
  64         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
  65         check_added_monitors!(nodes[0], 1);
  66         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
  67         assert_eq!(events.len(), 1);
  68         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
  69         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
  70         check_added_monitors!(nodes[1], 0);
  71         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  72         expect_pending_htlcs_forwardable!(nodes[1]);
  73         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_1.2 }]);
  74         let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
  75         assert!(htlc_updates.update_add_htlcs.is_empty());
  76         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
  77         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
  78         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
  79         check_added_monitors!(nodes[1], 1);
  80         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
  81         commitment_signed_dance!(nodes[0], nodes[1], htlc_updates.commitment_signed, false);
  82         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
  83
  84         // Rebalance the channel so the retry succeeds.
  85         send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000);
  86
  87         // Mine two blocks (we expire retries after 3, so this will check that we don't expire early)
  88         connect_blocks(&nodes[0], 2);
  89
  90         // Retry the payment and make sure it succeeds.
  91         nodes[0].node.retry_payment(&route, payment_id).unwrap();
  92         check_added_monitors!(nodes[0], 1);
  93         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
  94         assert_eq!(events.len(), 1);
  95         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 100_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
  96         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
  97 }
  98
  99 #[test]
 100 fn mpp_failure() {
 101         let chanmon_cfgs = create_chanmon_cfgs(4);
 102         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 103         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 104         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 105
 106         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
 107         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
 108         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
 109         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3).0.contents.short_channel_id;
 110
 111         let (mut route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
 112         let path = route.paths[0].clone();
 113         route.paths.push(path);
 114         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
 115         route.paths[0][0].short_channel_id = chan_1_id;
 116         route.paths[0][1].short_channel_id = chan_3_id;
 117         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
 118         route.paths[1][0].short_channel_id = chan_2_id;
 119         route.paths[1][1].short_channel_id = chan_4_id;
 120         send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
 121         fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
 122 }
 123
 124 #[test]
 125 fn mpp_retry() {
 126         let chanmon_cfgs = create_chanmon_cfgs(4);
 127         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 128         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 129         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 130
 131         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 132         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 133         let (chan_3_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 134         let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes(&nodes, 3, 2);
 135         // Rebalance
 136         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 137
 138         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 1_000_000);
 139         let path = route.paths[0].clone();
 140         route.paths.push(path);
 141         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
 142         route.paths[0][0].short_channel_id = chan_1_update.contents.short_channel_id;
 143         route.paths[0][1].short_channel_id = chan_3_update.contents.short_channel_id;
 144         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
 145         route.paths[1][0].short_channel_id = chan_2_update.contents.short_channel_id;
 146         route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id;
 147
 148         // Initiate the MPP payment.
 149         let payment_id = PaymentId(payment_hash.0);
 150         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
 151         check_added_monitors!(nodes[0], 2); // one monitor per path
 152         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 153         assert_eq!(events.len(), 2);
 154
 155         // Pass half of the payment along the success path.
 156         let (success_path_msgs, mut events) = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &events);
 157         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), success_path_msgs, false, None);
 158
 159         // Add the HTLC along the first hop.
 160         let (fail_path_msgs_1, _events) = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &events);
 161         let (update_add, commitment_signed) = match fail_path_msgs_1 {
 162                 MessageSendEvent::UpdateHTLCs { node_id: _, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 163                         assert_eq!(update_add_htlcs.len(), 1);
 164                         assert!(update_fail_htlcs.is_empty());
 165                         assert!(update_fulfill_htlcs.is_empty());
 166                         assert!(update_fail_malformed_htlcs.is_empty());
 167                         assert!(update_fee.is_none());
 168                         (update_add_htlcs[0].clone(), commitment_signed.clone())
 169                 },
 170                 _ => panic!("Unexpected event"),
 171         };
 172         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
 173         commitment_signed_dance!(nodes[2], nodes[0], commitment_signed, false);
 174
 175         // Attempt to forward the payment and complete the 2nd path's failure.
 176         expect_pending_htlcs_forwardable!(&nodes[2]);
 177         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id }]);
 178         let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 179         assert!(htlc_updates.update_add_htlcs.is_empty());
 180         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 181         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 182         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 183         check_added_monitors!(nodes[2], 1);
 184         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 185         commitment_signed_dance!(nodes[0], nodes[2], htlc_updates.commitment_signed, false);
 186         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 187
 188         // Rebalance the channel so the second half of the payment can succeed.
 189         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 190
 191         // Make sure it errors as expected given a too-large amount.
 192         if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, payment_id) {
 193                 assert!(err.contains("over total_payment_amt_msat"));
 194         } else { panic!("Unexpected error"); }
 195
 196         // Make sure it errors as expected given the wrong payment_id.
 197         if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, PaymentId([0; 32])) {
 198                 assert!(err.contains("not found"));
 199         } else { panic!("Unexpected error"); }
 200
 201         // Retry the second half of the payment and make sure it succeeds.
 202         let mut path = route.clone();
 203         path.paths.remove(0);
 204         nodes[0].node.retry_payment(&path, payment_id).unwrap();
 205         check_added_monitors!(nodes[0], 1);
 206         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 207         assert_eq!(events.len(), 1);
 208         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 209         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 210 }
 211
 212 fn do_mpp_receive_timeout(send_partial_mpp: bool) {
 213         let chanmon_cfgs = create_chanmon_cfgs(4);
 214         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 215         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 216         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 217
 218         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 219         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 220         let (chan_3_update, _, chan_3_id, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 221         let (chan_4_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 222
 223         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 100_000);
 224         let path = route.paths[0].clone();
 225         route.paths.push(path);
 226         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
 227         route.paths[0][0].short_channel_id = chan_1_update.contents.short_channel_id;
 228         route.paths[0][1].short_channel_id = chan_3_update.contents.short_channel_id;
 229         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
 230         route.paths[1][0].short_channel_id = chan_2_update.contents.short_channel_id;
 231         route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id;
 232
 233         // Initiate the MPP payment.
 234         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 235         check_added_monitors!(nodes[0], 2); // one monitor per path
 236         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 237         assert_eq!(events.len(), 2);
 238
 239         // Pass half of the payment along the first path.
 240         let (node_1_msgs, mut events) = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &events);
 241         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_1_msgs, false, None);
 242
 243         if send_partial_mpp {
 244                 // Time out the partial MPP
 245                 for _ in 0..MPP_TIMEOUT_TICKS {
 246                         nodes[3].node.timer_tick_occurred();
 247                 }
 248
 249                 // Failed HTLC from node 3 -> 1
 250                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 251                 let htlc_fail_updates_3_1 = get_htlc_update_msgs!(nodes[3], nodes[1].node.get_our_node_id());
 252                 assert_eq!(htlc_fail_updates_3_1.update_fail_htlcs.len(), 1);
 253                 nodes[1].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &htlc_fail_updates_3_1.update_fail_htlcs[0]);
 254                 check_added_monitors!(nodes[3], 1);
 255                 commitment_signed_dance!(nodes[1], nodes[3], htlc_fail_updates_3_1.commitment_signed, false);
 256
 257                 // Failed HTLC from node 1 -> 0
 258                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_3_id }]);
 259                 let htlc_fail_updates_1_0 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 260                 assert_eq!(htlc_fail_updates_1_0.update_fail_htlcs.len(), 1);
 261                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates_1_0.update_fail_htlcs[0]);
 262                 check_added_monitors!(nodes[1], 1);
 263                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates_1_0.commitment_signed, false);
 264
 265                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain().expected_htlc_error_data(23, &[][..]));
 266         } else {
 267                 // Pass half of the payment along the second path.
 268                 let (node_2_msgs, _events) = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &events);
 269                 pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_2_msgs, true, None);
 270
 271                 // Even after MPP_TIMEOUT_TICKS we should not timeout the MPP if we have all the parts
 272                 for _ in 0..MPP_TIMEOUT_TICKS {
 273                         nodes[3].node.timer_tick_occurred();
 274                 }
 275
 276                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 277         }
 278 }
 279
 280 #[test]
 281 fn mpp_receive_timeout() {
 282         do_mpp_receive_timeout(true);
 283         do_mpp_receive_timeout(false);
 284 }
 285
 286 #[test]
 287 fn retry_expired_payment() {
 288         let chanmon_cfgs = create_chanmon_cfgs(3);
 289         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 290         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 291         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 292
 293         let _chan_0 = create_announced_chan_between_nodes(&nodes, 0, 1);
 294         let chan_1 = create_announced_chan_between_nodes(&nodes, 2, 1);
 295         // Rebalance to find a route
 296         send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000);
 297
 298         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 100_000);
 299
 300         // Rebalance so that the first hop fails.
 301         send_payment(&nodes[1], &vec!(&nodes[2])[..], 2_000_000);
 302
 303         // Make sure the payment fails on the first hop.
 304         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 305         check_added_monitors!(nodes[0], 1);
 306         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 307         assert_eq!(events.len(), 1);
 308         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
 309         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 310         check_added_monitors!(nodes[1], 0);
 311         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 312         expect_pending_htlcs_forwardable!(nodes[1]);
 313         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_1.2 }]);
 314         let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 315         assert!(htlc_updates.update_add_htlcs.is_empty());
 316         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 317         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 318         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 319         check_added_monitors!(nodes[1], 1);
 320         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 321         commitment_signed_dance!(nodes[0], nodes[1], htlc_updates.commitment_signed, false);
 322         expect_payment_failed!(nodes[0], payment_hash, false);
 323
 324         // Mine blocks so the payment will have expired.
 325         connect_blocks(&nodes[0], 3);
 326
 327         // Retry the payment and make sure it errors as expected.
 328         if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, PaymentId(payment_hash.0)) {
 329                 assert!(err.contains("not found"));
 330         } else {
 331                 panic!("Unexpected error");
 332         }
 333 }
 334
 335 #[test]
 336 fn no_pending_leak_on_initial_send_failure() {
 337         // In an earlier version of our payment tracking, we'd have a retry entry even when the initial
 338         // HTLC for payment failed to send due to local channel errors (e.g. peer disconnected). In this
 339         // case, the user wouldn't have a PaymentId to retry the payment with, but we'd think we have a
 340         // pending payment forever and never time it out.
 341         // Here we test exactly that - retrying a payment when a peer was disconnected on the first
 342         // try, and then check that no pending payment is being tracked.
 343         let chanmon_cfgs = create_chanmon_cfgs(2);
 344         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 345         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 346         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 347
 348         create_announced_chan_between_nodes(&nodes, 0, 1);
 349
 350         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 351
 352         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 353         nodes[1].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 354
 355         unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)),
 356                 true, APIError::ChannelUnavailable { ref err },
 357                 assert_eq!(err, "Peer for first hop currently disconnected/pending monitor update!"));
 358
 359         assert!(!nodes[0].node.has_pending_payments());
 360 }
 361
 362 fn do_retry_with_no_persist(confirm_before_reload: bool) {
 363         // If we send a pending payment and `send_payment` returns success, we should always either
 364         // return a payment failure event or a payment success event, and on failure the payment should
 365         // be retryable.
 366         //
 367         // In order to do so when the ChannelManager isn't immediately persisted (which is normal - its
 368         // always persisted asynchronously), the ChannelManager has to reload some payment data from
 369         // ChannelMonitor(s) in some cases. This tests that reloading.
 370         //
 371         // `confirm_before_reload` confirms the channel-closing commitment transaction on-chain prior
 372         // to reloading the ChannelManager, increasing test coverage in ChannelMonitor HTLC tracking
 373         // which has separate codepaths for "commitment transaction already confirmed" and not.
 374         let chanmon_cfgs = create_chanmon_cfgs(3);
 375         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 376         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 377         let persister: test_utils::TestPersister;
 378         let new_chain_monitor: test_utils::TestChainMonitor;
 379         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 380         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 381
 382         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 383         let (_, _, chan_id_2, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
 384
 385         // Serialize the ChannelManager prior to sending payments
 386         let nodes_0_serialized = nodes[0].node.encode();
 387
 388         // Send two payments - one which will get to nodes[2] and will be claimed, one which we'll time
 389         // out and retry.
 390         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 391         let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000);
 392         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 393         check_added_monitors!(nodes[0], 1);
 394
 395         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 396         assert_eq!(events.len(), 1);
 397         let payment_event = SendEvent::from_event(events.pop().unwrap());
 398         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 399
 400         // We relay the payment to nodes[1] while its disconnected from nodes[2], causing the payment
 401         // to be returned immediately to nodes[0], without having nodes[2] fail the inbound payment
 402         // which would prevent retry.
 403         nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id(), false);
 404         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 405
 406         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 407         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true);
 408         // nodes[1] now immediately fails the HTLC as the next-hop channel is disconnected
 409         let _ = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 410
 411         reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 412
 413         let as_commitment_tx = get_local_commitment_txn!(nodes[0], chan_id)[0].clone();
 414         if confirm_before_reload {
 415                 mine_transaction(&nodes[0], &as_commitment_tx);
 416                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 417         }
 418
 419         // The ChannelMonitor should always be the latest version, as we're required to persist it
 420         // during the `commitment_signed_dance!()`.
 421         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 422         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 423
 424         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 425         // force-close the channel.
 426         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 427         assert!(nodes[0].node.list_channels().is_empty());
 428         assert!(nodes[0].node.has_pending_payments());
 429         let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 430         assert_eq!(as_broadcasted_txn.len(), 1);
 431         assert_eq!(as_broadcasted_txn[0], as_commitment_tx);
 432
 433         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 434         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
 435         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 436
 437         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 438         // error, as the channel has hit the chain.
 439         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
 440         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 441         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 442         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 443         assert_eq!(as_err.len(), 1);
 444         match as_err[0] {
 445                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 446                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 447                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 448                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()) });
 449                         check_added_monitors!(nodes[1], 1);
 450                         assert_eq!(nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 451                 },
 452                 _ => panic!("Unexpected event"),
 453         }
 454         check_closed_broadcast!(nodes[1], false);
 455
 456         // Now claim the first payment, which should allow nodes[1] to claim the payment on-chain when
 457         // we close in a moment.
 458         nodes[2].node.claim_funds(payment_preimage_1);
 459         check_added_monitors!(nodes[2], 1);
 460         expect_payment_claimed!(nodes[2], payment_hash_1, 1_000_000);
 461
 462         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 463         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 464         check_added_monitors!(nodes[1], 1);
 465         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 466         expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], None, false, false);
 467
 468         if confirm_before_reload {
 469                 let best_block = nodes[0].blocks.lock().unwrap().last().unwrap().clone();
 470                 nodes[0].node.best_block_updated(&best_block.0.header, best_block.1);
 471         }
 472
 473         // Create a new channel on which to retry the payment before we fail the payment via the
 474         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 475         // connecting several blocks while creating the channel (implying time has passed).
 476         create_announced_chan_between_nodes(&nodes, 0, 1);
 477         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 478
 479         mine_transaction(&nodes[1], &as_commitment_tx);
 480         let bs_htlc_claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 481         assert_eq!(bs_htlc_claim_txn.len(), 1);
 482         check_spends!(bs_htlc_claim_txn[0], as_commitment_tx);
 483
 484         if !confirm_before_reload {
 485                 mine_transaction(&nodes[0], &as_commitment_tx);
 486         }
 487         mine_transaction(&nodes[0], &bs_htlc_claim_txn[0]);
 488         expect_payment_sent!(nodes[0], payment_preimage_1);
 489         connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
 490         let as_htlc_timeout_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 491         assert_eq!(as_htlc_timeout_txn.len(), 2);
 492         let (first_htlc_timeout_tx, second_htlc_timeout_tx) = (&as_htlc_timeout_txn[0], &as_htlc_timeout_txn[1]);
 493         check_spends!(first_htlc_timeout_tx, as_commitment_tx);
 494         check_spends!(second_htlc_timeout_tx, as_commitment_tx);
 495         if first_htlc_timeout_tx.input[0].previous_output == bs_htlc_claim_txn[0].input[0].previous_output {
 496                 confirm_transaction(&nodes[0], &second_htlc_timeout_tx);
 497         } else {
 498                 confirm_transaction(&nodes[0], &first_htlc_timeout_tx);
 499         }
 500         nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 501         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 502
 503         // Finally, retry the payment (which was reloaded from the ChannelMonitor when nodes[0] was
 504         // reloaded) via a route over the new channel, which work without issue and eventually be
 505         // received and claimed at the recipient just like any other payment.
 506         let (mut new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 507
 508         // Update the fee on the middle hop to ensure PaymentSent events have the correct (retried) fee
 509         // and not the original fee. We also update node[1]'s relevant config as
 510         // do_claim_payment_along_route expects us to never overpay.
 511         {
 512                 let per_peer_state = nodes[1].node.per_peer_state.read().unwrap();
 513                 let mut peer_state = per_peer_state.get(&nodes[2].node.get_our_node_id())
 514                         .unwrap().lock().unwrap();
 515                 let mut channel = peer_state.channel_by_id.get_mut(&chan_id_2).unwrap();
 516                 let mut new_config = channel.config();
 517                 new_config.forwarding_fee_base_msat += 100_000;
 518                 channel.update_config(&new_config);
 519                 new_route.paths[0][0].fee_msat += 100_000;
 520         }
 521
 522         // Force expiration of the channel's previous config.
 523         for _ in 0..EXPIRE_PREV_CONFIG_TICKS {
 524                 nodes[1].node.timer_tick_occurred();
 525         }
 526
 527         assert!(nodes[0].node.retry_payment(&new_route, payment_id_1).is_err()); // Shouldn't be allowed to retry a fulfilled payment
 528         nodes[0].node.retry_payment(&new_route, PaymentId(payment_hash.0)).unwrap();
 529         check_added_monitors!(nodes[0], 1);
 530         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 531         assert_eq!(events.len(), 1);
 532         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 533         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
 534         expect_payment_sent!(nodes[0], payment_preimage, Some(new_route.paths[0][0].fee_msat));
 535 }
 536
 537 #[test]
 538 fn retry_with_no_persist() {
 539         do_retry_with_no_persist(true);
 540         do_retry_with_no_persist(false);
 541 }
 542
 543 fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 544         // Test that an off-chain completed payment is not retryable on restart. This was previously
 545         // broken for dust payments, but we test for both dust and non-dust payments.
 546         //
 547         // `use_dust` switches to using a dust HTLC, which results in the HTLC not having an on-chain
 548         // output at all.
 549         let chanmon_cfgs = create_chanmon_cfgs(3);
 550         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 551
 552         let mut manually_accept_config = test_default_channel_config();
 553         manually_accept_config.manually_accept_inbound_channels = true;
 554
 555         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(manually_accept_config), None]);
 556
 557         let first_persister: test_utils::TestPersister;
 558         let first_new_chain_monitor: test_utils::TestChainMonitor;
 559         let first_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 560         let second_persister: test_utils::TestPersister;
 561         let second_new_chain_monitor: test_utils::TestChainMonitor;
 562         let second_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 563         let third_persister: test_utils::TestPersister;
 564         let third_new_chain_monitor: test_utils::TestChainMonitor;
 565         let third_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 566
 567         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 568
 569         // Because we set nodes[1] to manually accept channels, just open a 0-conf channel.
 570         let (funding_tx, chan_id) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 571         confirm_transaction(&nodes[0], &funding_tx);
 572         confirm_transaction(&nodes[1], &funding_tx);
 573         // Ignore the announcement_signatures messages
 574         nodes[0].node.get_and_clear_pending_msg_events();
 575         nodes[1].node.get_and_clear_pending_msg_events();
 576         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 577
 578         // Serialize the ChannelManager prior to sending payments
 579         let mut nodes_0_serialized = nodes[0].node.encode();
 580
 581         let route = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 }).0;
 582         let (payment_preimage, payment_hash, payment_secret, payment_id) = send_along_route(&nodes[0], route, &[&nodes[1], &nodes[2]], if use_dust { 1_000 } else { 1_000_000 });
 583
 584         // The ChannelMonitor should always be the latest version, as we're required to persist it
 585         // during the `commitment_signed_dance!()`.
 586         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 587
 588         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized], first_persister, first_new_chain_monitor, first_nodes_0_deserialized);
 589         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 590
 591         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 592         // force-close the channel.
 593         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 594         assert!(nodes[0].node.list_channels().is_empty());
 595         assert!(nodes[0].node.has_pending_payments());
 596         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 597
 598         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
 599         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 600
 601         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 602         // error, as the channel has hit the chain.
 603         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
 604         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 605         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 606         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 607         assert_eq!(as_err.len(), 1);
 608         let bs_commitment_tx;
 609         match as_err[0] {
 610                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 611                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 612                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 613                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()) });
 614                         check_added_monitors!(nodes[1], 1);
 615                         bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 616                 },
 617                 _ => panic!("Unexpected event"),
 618         }
 619         check_closed_broadcast!(nodes[1], false);
 620
 621         // Now fail back the payment from nodes[2] to nodes[1]. This doesn't really matter as the
 622         // previous hop channel is already on-chain, but it makes nodes[2] willing to see additional
 623         // incoming HTLCs with the same payment hash later.
 624         nodes[2].node.fail_htlc_backwards(&payment_hash);
 625         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
 626         check_added_monitors!(nodes[2], 1);
 627
 628         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 629         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fail_htlcs[0]);
 630         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 631         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1],
 632                 [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 633
 634         // Connect the HTLC-Timeout transaction, timing out the HTLC on both nodes (but not confirming
 635         // the HTLC-Timeout transaction beyond 1 conf). For dust HTLCs, the HTLC is considered resolved
 636         // after the commitment transaction, so always connect the commitment transaction.
 637         mine_transaction(&nodes[0], &bs_commitment_tx[0]);
 638         mine_transaction(&nodes[1], &bs_commitment_tx[0]);
 639         if !use_dust {
 640                 connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1 + (MIN_CLTV_EXPIRY_DELTA as u32));
 641                 connect_blocks(&nodes[1], TEST_FINAL_CLTV - 1 + (MIN_CLTV_EXPIRY_DELTA as u32));
 642                 let as_htlc_timeout = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 643                 check_spends!(as_htlc_timeout[0], bs_commitment_tx[0]);
 644                 assert_eq!(as_htlc_timeout.len(), 1);
 645
 646                 mine_transaction(&nodes[0], &as_htlc_timeout[0]);
 647                 // nodes[0] may rebroadcast (or RBF-bump) its HTLC-Timeout, so wipe the announced set.
 648                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 649                 mine_transaction(&nodes[1], &as_htlc_timeout[0]);
 650         }
 651
 652         // Create a new channel on which to retry the payment before we fail the payment via the
 653         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 654         // connecting several blocks while creating the channel (implying time has passed).
 655         // We do this with a zero-conf channel to avoid connecting blocks as a side-effect.
 656         let (_, chan_id_3) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 657         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 658
 659         // If we attempt to retry prior to the HTLC-Timeout (or commitment transaction, for dust HTLCs)
 660         // confirming, we will fail as it's considered still-pending...
 661         let (new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 });
 662         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_err());
 663         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 664
 665         // After ANTI_REORG_DELAY confirmations, the HTLC should be failed and we can try the payment
 666         // again. We serialize the node first as we'll then test retrying the HTLC after a restart
 667         // (which should also still work).
 668         connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
 669         connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 670         // We set mpp_parts_remain to avoid having abandon_payment called
 671         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 672
 673         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 674         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 675         nodes_0_serialized = nodes[0].node.encode();
 676
 677         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_ok());
 678         assert!(!nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 679
 680         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], second_persister, second_new_chain_monitor, second_nodes_0_deserialized);
 681         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 682
 683         reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 684
 685         // Now resend the payment, delivering the HTLC and actually claiming it this time. This ensures
 686         // the payment is not (spuriously) listed as still pending.
 687         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_ok());
 688         check_added_monitors!(nodes[0], 1);
 689         pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], if use_dust { 1_000 } else { 1_000_000 }, payment_hash, payment_secret);
 690         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
 691
 692         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_err());
 693         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 694
 695         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 696         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 697         nodes_0_serialized = nodes[0].node.encode();
 698
 699         // Ensure that after reload we cannot retry the payment.
 700         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], third_persister, third_new_chain_monitor, third_nodes_0_deserialized);
 701         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 702
 703         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 704
 705         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_err());
 706         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 707 }
 708
 709 #[test]
 710 fn test_completed_payment_not_retryable_on_reload() {
 711         do_test_completed_payment_not_retryable_on_reload(true);
 712         do_test_completed_payment_not_retryable_on_reload(false);
 713 }
 714
 715
 716 fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, confirm_commitment_tx: bool, payment_timeout: bool) {
 717         // When a Channel is closed, any outbound HTLCs which were relayed through it are simply
 718         // dropped when the Channel is. From there, the ChannelManager relies on the ChannelMonitor
 719         // having a copy of the relevant fail-/claim-back data and processes the HTLC fail/claim when
 720         // the ChannelMonitor tells it to.
 721         //
 722         // If, due to an on-chain event, an HTLC is failed/claimed, we should avoid providing the
 723         // ChannelManager the HTLC event until after the monitor is re-persisted. This should prevent a
 724         // duplicate HTLC fail/claim (e.g. via a PaymentPathFailed event).
 725         let chanmon_cfgs = create_chanmon_cfgs(2);
 726         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 727         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 728         let persister: test_utils::TestPersister;
 729         let new_chain_monitor: test_utils::TestChainMonitor;
 730         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 731         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 732
 733         let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1);
 734
 735         // Route a payment, but force-close the channel before the HTLC fulfill message arrives at
 736         // nodes[0].
 737         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 10_000_000);
 738         nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap();
 739         check_closed_broadcast!(nodes[0], true);
 740         check_added_monitors!(nodes[0], 1);
 741         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 742
 743         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 744         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 745
 746         // Connect blocks until the CLTV timeout is up so that we get an HTLC-Timeout transaction
 747         connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
 748         let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 749         assert_eq!(node_txn.len(), 3);
 750         assert_eq!(node_txn[0], node_txn[1]);
 751         check_spends!(node_txn[1], funding_tx);
 752         check_spends!(node_txn[2], node_txn[1]);
 753         let timeout_txn = vec![node_txn[2].clone()];
 754
 755         nodes[1].node.claim_funds(payment_preimage);
 756         check_added_monitors!(nodes[1], 1);
 757         expect_payment_claimed!(nodes[1], payment_hash, 10_000_000);
 758
 759         let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 };
 760         connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[1].clone()]});
 761         check_closed_broadcast!(nodes[1], true);
 762         check_added_monitors!(nodes[1], 1);
 763         check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 764         let claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 765         assert_eq!(claim_txn.len(), 1);
 766         check_spends!(claim_txn[0], node_txn[1]);
 767
 768         header.prev_blockhash = nodes[0].best_block_hash();
 769         connect_block(&nodes[0], &Block { header, txdata: vec![node_txn[1].clone()]});
 770
 771         if confirm_commitment_tx {
 772                 connect_blocks(&nodes[0], BREAKDOWN_TIMEOUT as u32 - 1);
 773         }
 774
 775         header.prev_blockhash = nodes[0].best_block_hash();
 776         let claim_block = Block { header, txdata: if payment_timeout { timeout_txn } else { vec![claim_txn[0].clone()] } };
 777
 778         if payment_timeout {
 779                 assert!(confirm_commitment_tx); // Otherwise we're spending below our CSV!
 780                 connect_block(&nodes[0], &claim_block);
 781                 connect_blocks(&nodes[0], ANTI_REORG_DELAY - 2);
 782         }
 783
 784         // Now connect the HTLC claim transaction with the ChainMonitor-generated ChannelMonitor update
 785         // returning InProgress. This should cause the claim event to never make its way to the
 786         // ChannelManager.
 787         chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap().clear();
 788         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
 789
 790         if payment_timeout {
 791                 connect_blocks(&nodes[0], 1);
 792         } else {
 793                 connect_block(&nodes[0], &claim_block);
 794         }
 795
 796         let funding_txo = OutPoint { txid: funding_tx.txid(), index: 0 };
 797         let mon_updates: Vec<_> = chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap()
 798                 .get_mut(&funding_txo).unwrap().drain().collect();
 799         // If we are using chain::Confirm instead of chain::Listen, we will get the same update twice.
 800         // If we're testing connection idempotency we may get substantially more.
 801         assert!(mon_updates.len() >= 1);
 802         assert!(nodes[0].chain_monitor.release_pending_monitor_events().is_empty());
 803         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 804
 805         // If we persist the ChannelManager here, we should get the PaymentSent event after
 806         // deserialization.
 807         let mut chan_manager_serialized = Vec::new();
 808         if !persist_manager_post_event {
 809                 chan_manager_serialized = nodes[0].node.encode();
 810         }
 811
 812         // Now persist the ChannelMonitor and inform the ChainMonitor that we're done, generating the
 813         // payment sent event.
 814         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
 815         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 816         for update in mon_updates {
 817                 nodes[0].chain_monitor.chain_monitor.channel_monitor_updated(funding_txo, update).unwrap();
 818         }
 819         if payment_timeout {
 820                 expect_payment_failed!(nodes[0], payment_hash, false);
 821         } else {
 822                 expect_payment_sent!(nodes[0], payment_preimage);
 823         }
 824
 825         // If we persist the ChannelManager after we get the PaymentSent event, we shouldn't get it
 826         // twice.
 827         if persist_manager_post_event {
 828                 chan_manager_serialized = nodes[0].node.encode();
 829         }
 830
 831         // Now reload nodes[0]...
 832         reload_node!(nodes[0], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 833
 834         if persist_manager_post_event {
 835                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 836         } else if payment_timeout {
 837                 expect_payment_failed!(nodes[0], payment_hash, false);
 838         } else {
 839                 expect_payment_sent!(nodes[0], payment_preimage);
 840         }
 841
 842         // Note that if we re-connect the block which exposed nodes[0] to the payment preimage (but
 843         // which the current ChannelMonitor has not seen), the ChannelManager's de-duplication of
 844         // payment events should kick in, leaving us with no pending events here.
 845         let height = nodes[0].blocks.lock().unwrap().len() as u32 - 1;
 846         nodes[0].chain_monitor.chain_monitor.block_connected(&claim_block, height);
 847         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 848 }
 849
 850 #[test]
 851 fn test_dup_htlc_onchain_fails_on_reload() {
 852         do_test_dup_htlc_onchain_fails_on_reload(true, true, true);
 853         do_test_dup_htlc_onchain_fails_on_reload(true, true, false);
 854         do_test_dup_htlc_onchain_fails_on_reload(true, false, false);
 855         do_test_dup_htlc_onchain_fails_on_reload(false, true, true);
 856         do_test_dup_htlc_onchain_fails_on_reload(false, true, false);
 857         do_test_dup_htlc_onchain_fails_on_reload(false, false, false);
 858 }
 859
 860 #[test]
 861 fn test_fulfill_restart_failure() {
 862         // When we receive an update_fulfill_htlc message, we immediately consider the HTLC fully
 863         // fulfilled. At this point, the peer can reconnect and decide to either fulfill the HTLC
 864         // again, or fail it, giving us free money.
 865         //
 866         // Of course probably they won't fail it and give us free money, but because we have code to
 867         // handle it, we should test the logic for it anyway. We do that here.
 868         let chanmon_cfgs = create_chanmon_cfgs(2);
 869         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 870         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 871         let persister: test_utils::TestPersister;
 872         let new_chain_monitor: test_utils::TestChainMonitor;
 873         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 874         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 875
 876         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 877         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 100_000);
 878
 879         // The simplest way to get a failure after a fulfill is to reload nodes[1] from a state
 880         // pre-fulfill, which we do by serializing it here.
 881         let chan_manager_serialized = nodes[1].node.encode();
 882         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id).encode();
 883
 884         nodes[1].node.claim_funds(payment_preimage);
 885         check_added_monitors!(nodes[1], 1);
 886         expect_payment_claimed!(nodes[1], payment_hash, 100_000);
 887
 888         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 889         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 890         expect_payment_sent_without_paths!(nodes[0], payment_preimage);
 891
 892         // Now reload nodes[1]...
 893         reload_node!(nodes[1], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 894
 895         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 896         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 897
 898         nodes[1].node.fail_htlc_backwards(&payment_hash);
 899         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]);
 900         check_added_monitors!(nodes[1], 1);
 901         let htlc_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 902         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates.update_fail_htlcs[0]);
 903         commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates.commitment_signed, false);
 904         // nodes[0] shouldn't generate any events here, while it just got a payment failure completion
 905         // it had already considered the payment fulfilled, and now they just got free money.
 906         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 907 }
 908
 909 #[test]
 910 fn get_ldk_payment_preimage() {
 911         // Ensure that `ChannelManager::get_payment_preimage` can successfully be used to claim a payment.
 912         let chanmon_cfgs = create_chanmon_cfgs(2);
 913         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 914         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 915         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 916         create_announced_chan_between_nodes(&nodes, 0, 1);
 917
 918         let amt_msat = 60_000;
 919         let expiry_secs = 60 * 60;
 920         let (payment_hash, payment_secret) = nodes[1].node.create_inbound_payment(Some(amt_msat), expiry_secs, None).unwrap();
 921
 922         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
 923                 .with_features(nodes[1].node.invoice_features());
 924         let scorer = test_utils::TestScorer::with_penalty(0);
 925         let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet);
 926         let random_seed_bytes = keys_manager.get_secure_random_bytes();
 927         let route = get_route(
 928                 &nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(),
 929                 Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()),
 930                 amt_msat, TEST_FINAL_CLTV, nodes[0].logger, &scorer, &random_seed_bytes).unwrap();
 931         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 932         check_added_monitors!(nodes[0], 1);
 933
 934         // Make sure to use `get_payment_preimage`
 935         let payment_preimage = nodes[1].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
 936         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 937         assert_eq!(events.len(), 1);
 938         pass_along_path(&nodes[0], &[&nodes[1]], amt_msat, payment_hash, Some(payment_secret), events.pop().unwrap(), true, Some(payment_preimage));
 939         claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, payment_preimage);
 940 }
 941
 942 #[test]
 943 fn sent_probe_is_probe_of_sending_node() {
 944         let chanmon_cfgs = create_chanmon_cfgs(3);
 945         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 946         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
 947         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 948
 949         create_announced_chan_between_nodes(&nodes, 0, 1);
 950         create_announced_chan_between_nodes(&nodes, 1, 2);
 951
 952         // First check we refuse to build a single-hop probe
 953         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[1], 100_000);
 954         assert!(nodes[0].node.send_probe(route.paths[0].clone()).is_err());
 955
 956         // Then build an actual two-hop probing path
 957         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
 958
 959         match nodes[0].node.send_probe(route.paths[0].clone()) {
 960                 Ok((payment_hash, payment_id)) => {
 961                         assert!(nodes[0].node.payment_is_probe(&payment_hash, &payment_id));
 962                         assert!(!nodes[1].node.payment_is_probe(&payment_hash, &payment_id));
 963                         assert!(!nodes[2].node.payment_is_probe(&payment_hash, &payment_id));
 964                 },
 965                 _ => panic!(),
 966         }
 967
 968         get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 969         check_added_monitors!(nodes[0], 1);
 970 }
 971
 972 #[test]
 973 fn successful_probe_yields_event() {
 974         let chanmon_cfgs = create_chanmon_cfgs(3);
 975         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 976         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
 977         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 978
 979         create_announced_chan_between_nodes(&nodes, 0, 1);
 980         create_announced_chan_between_nodes(&nodes, 1, 2);
 981
 982         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
 983
 984         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
 985
 986         // node[0] -- update_add_htlcs -> node[1]
 987         check_added_monitors!(nodes[0], 1);
 988         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 989         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
 990         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
 991         check_added_monitors!(nodes[1], 0);
 992         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
 993         expect_pending_htlcs_forwardable!(nodes[1]);
 994
 995         // node[1] -- update_add_htlcs -> node[2]
 996         check_added_monitors!(nodes[1], 1);
 997         let updates = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 998         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
 999         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &probe_event.msgs[0]);
1000         check_added_monitors!(nodes[2], 0);
1001         commitment_signed_dance!(nodes[2], nodes[1], probe_event.commitment_msg, true, true);
1002
1003         // node[1] <- update_fail_htlcs -- node[2]
1004         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1005         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1006         check_added_monitors!(nodes[1], 0);
1007         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, true);
1008
1009         // node[0] <- update_fail_htlcs -- node[1]
1010         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1011         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1012         check_added_monitors!(nodes[0], 0);
1013         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1014
1015         let mut events = nodes[0].node.get_and_clear_pending_events();
1016         assert_eq!(events.len(), 1);
1017         match events.drain(..).next().unwrap() {
1018                 crate::util::events::Event::ProbeSuccessful { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1019                         assert_eq!(payment_id, ev_pid);
1020                         assert_eq!(payment_hash, ev_ph);
1021                 },
1022                 _ => panic!(),
1023         };
1024 }
1025
1026 #[test]
1027 fn failed_probe_yields_event() {
1028         let chanmon_cfgs = create_chanmon_cfgs(3);
1029         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1030         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1031         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1032
1033         create_announced_chan_between_nodes(&nodes, 0, 1);
1034         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 100000, 90000000);
1035
1036         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1037
1038         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 9_998_000, 42);
1039
1040         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1041
1042         // node[0] -- update_add_htlcs -> node[1]
1043         check_added_monitors!(nodes[0], 1);
1044         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1045         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1046         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1047         check_added_monitors!(nodes[1], 0);
1048         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1049         expect_pending_htlcs_forwardable!(nodes[1]);
1050
1051         // node[0] <- update_fail_htlcs -- node[1]
1052         check_added_monitors!(nodes[1], 1);
1053         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1054         // Skip the PendingHTLCsForwardable event
1055         let _events = nodes[1].node.get_and_clear_pending_events();
1056         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1057         check_added_monitors!(nodes[0], 0);
1058         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1059
1060         let mut events = nodes[0].node.get_and_clear_pending_events();
1061         assert_eq!(events.len(), 1);
1062         match events.drain(..).next().unwrap() {
1063                 crate::util::events::Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1064                         assert_eq!(payment_id, ev_pid);
1065                         assert_eq!(payment_hash, ev_ph);
1066                 },
1067                 _ => panic!(),
1068         };
1069 }
1070
1071 #[test]
1072 fn onchain_failed_probe_yields_event() {
1073         // Tests that an attempt to probe over a channel that is eventaully closed results in a failure
1074         // event.
1075         let chanmon_cfgs = create_chanmon_cfgs(3);
1076         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1077         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1078         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1079
1080         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1081         create_announced_chan_between_nodes(&nodes, 1, 2);
1082
1083         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1084
1085         // Send a dust HTLC, which will be treated as if it timed out once the channel hits the chain.
1086         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 1_000, 42);
1087         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1088
1089         // node[0] -- update_add_htlcs -> node[1]
1090         check_added_monitors!(nodes[0], 1);
1091         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1092         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1093         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1094         check_added_monitors!(nodes[1], 0);
1095         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1096         expect_pending_htlcs_forwardable!(nodes[1]);
1097
1098         check_added_monitors!(nodes[1], 1);
1099         let _ = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1100
1101         // Don't bother forwarding the HTLC onwards and just confirm the force-close transaction on
1102         // Node A, which after 6 confirmations should result in a probe failure event.
1103         let bs_txn = get_local_commitment_txn!(nodes[1], chan_id);
1104         confirm_transaction(&nodes[0], &bs_txn[0]);
1105         check_closed_broadcast!(&nodes[0], true);
1106         check_added_monitors!(nodes[0], 1);
1107
1108         let mut events = nodes[0].node.get_and_clear_pending_events();
1109         assert_eq!(events.len(), 2);
1110         let mut found_probe_failed = false;
1111         for event in events.drain(..) {
1112                 match event {
1113                         Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1114                                 assert_eq!(payment_id, ev_pid);
1115                                 assert_eq!(payment_hash, ev_ph);
1116                                 found_probe_failed = true;
1117                         },
1118                         Event::ChannelClosed { .. } => {},
1119                         _ => panic!(),
1120                 }
1121         }
1122         assert!(found_probe_failed);
1123 }
1124
1125 #[test]
1126 fn claimed_send_payment_idempotent() {
1127         // Tests that `send_payment` (and friends) are (reasonably) idempotent.
1128         let chanmon_cfgs = create_chanmon_cfgs(2);
1129         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1130         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1131         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1132
1133         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1134
1135         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1136         let (first_payment_preimage, _, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1137
1138         macro_rules! check_send_rejected {
1139                 () => {
1140                         // If we try to resend a new payment with a different payment_hash but with the same
1141                         // payment_id, it should be rejected.
1142                         let send_result = nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id);
1143                         match send_result {
1144                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1145                                 _ => panic!("Unexpected send result: {:?}", send_result),
1146                         }
1147
1148                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1149                         // also be rejected.
1150                         let send_result = nodes[0].node.send_spontaneous_payment(&route, None, payment_id);
1151                         match send_result {
1152                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1153                                 _ => panic!("Unexpected send result: {:?}", send_result),
1154                         }
1155                 }
1156         }
1157
1158         check_send_rejected!();
1159
1160         // Claim the payment backwards, but note that the PaymentSent event is still pending and has
1161         // not been seen by the user. At this point, from the user perspective nothing has changed, so
1162         // we must remain just as idempotent as we were before.
1163         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, first_payment_preimage);
1164
1165         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1166                 nodes[0].node.timer_tick_occurred();
1167         }
1168
1169         check_send_rejected!();
1170
1171         // Once the user sees and handles the `PaymentSent` event, we expect them to no longer call
1172         // `send_payment`, and our idempotency guarantees are off - they should have atomically marked
1173         // the payment complete. However, they could have called `send_payment` while the event was
1174         // being processed, leading to a race in our idempotency guarantees. Thus, even immediately
1175         // after the event is handled a duplicate payment should sitll be rejected.
1176         expect_payment_sent!(&nodes[0], first_payment_preimage, Some(0));
1177         check_send_rejected!();
1178
1179         // If relatively little time has passed, a duplicate payment should still fail.
1180         nodes[0].node.timer_tick_occurred();
1181         check_send_rejected!();
1182
1183         // However, after some time has passed (at least more than the one timer tick above), a
1184         // duplicate payment should go through, as ChannelManager should no longer have any remaining
1185         // references to the old payment data.
1186         for _ in 0..IDEMPOTENCY_TIMEOUT_TICKS {
1187                 nodes[0].node.timer_tick_occurred();
1188         }
1189
1190         nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id).unwrap();
1191         check_added_monitors!(nodes[0], 1);
1192         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1193         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1194 }
1195
1196 #[test]
1197 fn abandoned_send_payment_idempotent() {
1198         // Tests that `send_payment` (and friends) allow duplicate PaymentIds immediately after
1199         // abandon_payment.
1200         let chanmon_cfgs = create_chanmon_cfgs(2);
1201         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1202         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1203         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1204
1205         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1206
1207         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1208         let (_, first_payment_hash, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1209
1210         macro_rules! check_send_rejected {
1211                 () => {
1212                         // If we try to resend a new payment with a different payment_hash but with the same
1213                         // payment_id, it should be rejected.
1214                         let send_result = nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id);
1215                         match send_result {
1216                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1217                                 _ => panic!("Unexpected send result: {:?}", send_result),
1218                         }
1219
1220                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1221                         // also be rejected.
1222                         let send_result = nodes[0].node.send_spontaneous_payment(&route, None, payment_id);
1223                         match send_result {
1224                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1225                                 _ => panic!("Unexpected send result: {:?}", send_result),
1226                         }
1227                 }
1228         }
1229
1230         check_send_rejected!();
1231
1232         nodes[1].node.fail_htlc_backwards(&first_payment_hash);
1233         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash: first_payment_hash }]);
1234
1235         pass_failed_payment_back_no_abandon(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash);
1236         check_send_rejected!();
1237
1238         // Until we abandon the payment, no matter how many timer ticks pass, we still cannot reuse the
1239         // PaymentId.
1240         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1241                 nodes[0].node.timer_tick_occurred();
1242         }
1243         check_send_rejected!();
1244
1245         nodes[0].node.abandon_payment(payment_id);
1246         get_event!(nodes[0], Event::PaymentFailed);
1247
1248         // However, we can reuse the PaymentId immediately after we `abandon_payment`.
1249         nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id).unwrap();
1250         check_added_monitors!(nodes[0], 1);
1251         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1252         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1253 }
1254
1255 #[derive(PartialEq)]
1256 enum InterceptTest {
1257         Forward,
1258         Fail,
1259         Timeout,
1260 }
1261
1262 #[test]
1263 fn test_trivial_inflight_htlc_tracking(){
1264         // In this test, we test three scenarios:
1265         // (1) Sending + claiming a payment successfully should return `None` when querying InFlightHtlcs
1266         // (2) Sending a payment without claiming it should return the payment's value (500000) when querying InFlightHtlcs
1267         // (3) After we claim the payment sent in (2), InFlightHtlcs should return `None` for the query.
1268         let chanmon_cfgs = create_chanmon_cfgs(3);
1269         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1270         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1271         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1272
1273         let (_, _, chan_1_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
1274         let (_, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
1275
1276         // Send and claim the payment. Inflight HTLCs should be empty.
1277         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 500000);
1278         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
1279         check_added_monitors!(nodes[0], 1);
1280         pass_along_route(&nodes[0], &[&vec!(&nodes[1], &nodes[2])[..]], 500000, payment_hash, payment_secret);
1281         claim_payment(&nodes[0], &vec!(&nodes[1], &nodes[2])[..], payment_preimage);
1282         {
1283                 let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1284
1285                 let mut node_0_per_peer_lock;
1286                 let mut node_0_peer_state_lock;
1287                 let mut node_1_per_peer_lock;
1288                 let mut node_1_peer_state_lock;
1289                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1290                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1291
1292                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1293                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1294                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1295                         channel_1.get_short_channel_id().unwrap()
1296                 );
1297                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1298                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1299                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1300                         channel_2.get_short_channel_id().unwrap()
1301                 );
1302
1303                 assert_eq!(chan_1_used_liquidity, None);
1304                 assert_eq!(chan_2_used_liquidity, None);
1305         }
1306         let pending_payments = nodes[0].node.list_recent_payments();
1307         assert_eq!(pending_payments.len(), 1);
1308         assert_eq!(pending_payments[0], RecentPaymentDetails::Fulfilled { payment_hash: Some(payment_hash) });
1309
1310         // Remove fulfilled payment
1311         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1312                 nodes[0].node.timer_tick_occurred();
1313         }
1314
1315         // Send the payment, but do not claim it. Our inflight HTLCs should contain the pending payment.
1316         let (payment_preimage, payment_hash,  _) = route_payment(&nodes[0], &vec!(&nodes[1], &nodes[2])[..], 500000);
1317         {
1318                 let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1319
1320                 let mut node_0_per_peer_lock;
1321                 let mut node_0_peer_state_lock;
1322                 let mut node_1_per_peer_lock;
1323                 let mut node_1_peer_state_lock;
1324                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1325                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1326
1327                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1328                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1329                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1330                         channel_1.get_short_channel_id().unwrap()
1331                 );
1332                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1333                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1334                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1335                         channel_2.get_short_channel_id().unwrap()
1336                 );
1337
1338                 // First hop accounts for expected 1000 msat fee
1339                 assert_eq!(chan_1_used_liquidity, Some(501000));
1340                 assert_eq!(chan_2_used_liquidity, Some(500000));
1341         }
1342         let pending_payments = nodes[0].node.list_recent_payments();
1343         assert_eq!(pending_payments.len(), 1);
1344         assert_eq!(pending_payments[0], RecentPaymentDetails::Pending { payment_hash, total_msat: 500000 });
1345
1346         // Now, let's claim the payment. This should result in the used liquidity to return `None`.
1347         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
1348
1349         // Remove fulfilled payment
1350         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1351                 nodes[0].node.timer_tick_occurred();
1352         }
1353
1354         {
1355                 let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1356
1357                 let mut node_0_per_peer_lock;
1358                 let mut node_0_peer_state_lock;
1359                 let mut node_1_per_peer_lock;
1360                 let mut node_1_peer_state_lock;
1361                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1362                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1363
1364                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1365                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1366                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1367                         channel_1.get_short_channel_id().unwrap()
1368                 );
1369                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1370                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1371                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1372                         channel_2.get_short_channel_id().unwrap()
1373                 );
1374
1375                 assert_eq!(chan_1_used_liquidity, None);
1376                 assert_eq!(chan_2_used_liquidity, None);
1377         }
1378
1379         let pending_payments = nodes[0].node.list_recent_payments();
1380         assert_eq!(pending_payments.len(), 0);
1381 }
1382
1383 #[test]
1384 fn test_holding_cell_inflight_htlcs() {
1385         let chanmon_cfgs = create_chanmon_cfgs(2);
1386         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1387         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1388         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1389         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1390
1391         let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
1392         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1393
1394         // Queue up two payments - one will be delivered right away, one immediately goes into the
1395         // holding cell as nodes[0] is AwaitingRAA.
1396         {
1397                 nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
1398                 check_added_monitors!(nodes[0], 1);
1399                 nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
1400                 check_added_monitors!(nodes[0], 0);
1401         }
1402
1403         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1404
1405         {
1406                 let mut node_0_per_peer_lock;
1407                 let mut node_0_peer_state_lock;
1408                 let channel =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, channel_id);
1409
1410                 let used_liquidity = inflight_htlcs.used_liquidity_msat(
1411                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1412                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1413                         channel.get_short_channel_id().unwrap()
1414                 );
1415
1416                 assert_eq!(used_liquidity, Some(2000000));
1417         }
1418
1419         // Clear pending events so test doesn't throw a "Had excess message on node..." error
1420         nodes[0].node.get_and_clear_pending_msg_events();
1421 }
1422
1423 #[test]
1424 fn intercepted_payment() {
1425         // Test that detecting an intercept scid on payment forward will signal LDK to generate an
1426         // intercept event, which the LSP can then use to either (a) open a JIT channel to forward the
1427         // payment or (b) fail the payment.
1428         do_test_intercepted_payment(InterceptTest::Forward);
1429         do_test_intercepted_payment(InterceptTest::Fail);
1430         // Make sure that intercepted payments will be automatically failed back if too many blocks pass.
1431         do_test_intercepted_payment(InterceptTest::Timeout);
1432 }
1433
1434 fn do_test_intercepted_payment(test: InterceptTest) {
1435         let chanmon_cfgs = create_chanmon_cfgs(3);
1436         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1437
1438         let mut zero_conf_chan_config = test_default_channel_config();
1439         zero_conf_chan_config.manually_accept_inbound_channels = true;
1440         let mut intercept_forwards_config = test_default_channel_config();
1441         intercept_forwards_config.accept_intercept_htlcs = true;
1442         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(zero_conf_chan_config)]);
1443
1444         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1445         let scorer = test_utils::TestScorer::with_penalty(0);
1446         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
1447
1448         let _ = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1449
1450         let amt_msat = 100_000;
1451         let intercept_scid = nodes[1].node.get_intercept_scid();
1452         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1453                 .with_route_hints(vec![
1454                         RouteHint(vec![RouteHintHop {
1455                                 src_node_id: nodes[1].node.get_our_node_id(),
1456                                 short_channel_id: intercept_scid,
1457                                 fees: RoutingFees {
1458                                         base_msat: 1000,
1459                                         proportional_millionths: 0,
1460                                 },
1461                                 cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
1462                                 htlc_minimum_msat: None,
1463                                 htlc_maximum_msat: None,
1464                         }])
1465                 ])
1466                 .with_features(nodes[2].node.invoice_features());
1467         let route_params = RouteParameters {
1468                 payment_params,
1469                 final_value_msat: amt_msat,
1470                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
1471         };
1472         let route = get_route(
1473                 &nodes[0].node.get_our_node_id(), &route_params.payment_params,
1474                 &nodes[0].network_graph.read_only(), None, route_params.final_value_msat,
1475                 route_params.final_cltv_expiry_delta, nodes[0].logger, &scorer, &random_seed_bytes
1476         ).unwrap();
1477
1478         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
1479         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
1480         let payment_event = {
1481                 {
1482                         let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
1483                         assert_eq!(added_monitors.len(), 1);
1484                         added_monitors.clear();
1485                 }
1486                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1487                 assert_eq!(events.len(), 1);
1488                 SendEvent::from_event(events.remove(0))
1489         };
1490         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1491         commitment_signed_dance!(nodes[1], nodes[0], &payment_event.commitment_msg, false, true);
1492
1493         // Check that we generate the PaymentIntercepted event when an intercept forward is detected.
1494         let events = nodes[1].node.get_and_clear_pending_events();
1495         assert_eq!(events.len(), 1);
1496         let (intercept_id, expected_outbound_amount_msat) = match events[0] {
1497                 crate::util::events::Event::HTLCIntercepted {
1498                         intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, inbound_amount_msat, requested_next_hop_scid: short_channel_id
1499                 } => {
1500                         assert_eq!(pmt_hash, payment_hash);
1501                         assert_eq!(inbound_amount_msat, route.get_total_amount() + route.get_total_fees());
1502                         assert_eq!(short_channel_id, intercept_scid);
1503                         (intercept_id, expected_outbound_amount_msat)
1504                 },
1505                 _ => panic!()
1506         };
1507
1508         // Check for unknown channel id error.
1509         let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &[42; 32], nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1510         assert_eq!(unknown_chan_id_err , APIError::ChannelUnavailable  { err: format!("Channel with id {} not found for the passed counterparty node_id {}", log_bytes!([42; 32]), nodes[2].node.get_our_node_id()) });
1511
1512         if test == InterceptTest::Fail {
1513                 // Ensure we can fail the intercepted payment back.
1514                 nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap();
1515                 expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::UnknownNextHop { requested_forward_scid: intercept_scid }]);
1516                 nodes[1].node.process_pending_htlc_forwards();
1517                 let update_fail = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1518                 check_added_monitors!(&nodes[1], 1);
1519                 assert!(update_fail.update_fail_htlcs.len() == 1);
1520                 let fail_msg = update_fail.update_fail_htlcs[0].clone();
1521                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1522                 commitment_signed_dance!(nodes[0], nodes[1], update_fail.commitment_signed, false);
1523
1524                 // Ensure the payment fails with the expected error.
1525                 let fail_conditions = PaymentFailedConditions::new()
1526                         .blamed_scid(intercept_scid)
1527                         .blamed_chan_closed(true)
1528                         .expected_htlc_error_data(0x4000 | 10, &[]);
1529                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
1530         } else if test == InterceptTest::Forward {
1531                 // Check that we'll fail as expected when sending to a channel that isn't in `ChannelReady` yet.
1532                 let temp_chan_id = nodes[1].node.create_channel(nodes[2].node.get_our_node_id(), 100_000, 0, 42, None).unwrap();
1533                 let unusable_chan_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &temp_chan_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1534                 assert_eq!(unusable_chan_err , APIError::ChannelUnavailable { err: format!("Channel with id {} not fully established", log_bytes!(temp_chan_id)) });
1535                 assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
1536
1537                 // Open the just-in-time channel so the payment can then be forwarded.
1538                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1539
1540                 // Finally, forward the intercepted payment through and claim it.
1541                 nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap();
1542                 expect_pending_htlcs_forwardable!(nodes[1]);
1543
1544                 let payment_event = {
1545                         {
1546                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
1547                                 assert_eq!(added_monitors.len(), 1);
1548                                 added_monitors.clear();
1549                         }
1550                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
1551                         assert_eq!(events.len(), 1);
1552                         SendEvent::from_event(events.remove(0))
1553                 };
1554                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1555                 commitment_signed_dance!(nodes[2], nodes[1], &payment_event.commitment_msg, false, true);
1556                 expect_pending_htlcs_forwardable!(nodes[2]);
1557
1558                 let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1559                 expect_payment_claimable!(&nodes[2], payment_hash, payment_secret, amt_msat, Some(payment_preimage), nodes[2].node.get_our_node_id());
1560                 do_claim_payment_along_route(&nodes[0], &vec!(&vec!(&nodes[1], &nodes[2])[..]), false, payment_preimage);
1561                 let events = nodes[0].node.get_and_clear_pending_events();
1562                 assert_eq!(events.len(), 2);
1563                 match events[0] {
1564                         Event::PaymentSent { payment_preimage: ref ev_preimage, payment_hash: ref ev_hash, ref fee_paid_msat, .. } => {
1565                                 assert_eq!(payment_preimage, *ev_preimage);
1566                                 assert_eq!(payment_hash, *ev_hash);
1567                                 assert_eq!(fee_paid_msat, &Some(1000));
1568                         },
1569                         _ => panic!("Unexpected event")
1570                 }
1571                 match events[1] {
1572                         Event::PaymentPathSuccessful { payment_hash: hash, .. } => {
1573                                 assert_eq!(hash, Some(payment_hash));
1574                         },
1575                         _ => panic!("Unexpected event")
1576                 }
1577         } else if test == InterceptTest::Timeout {
1578                 let mut block = Block {
1579                         header: BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 },
1580                         txdata: vec![],
1581                 };
1582                 connect_block(&nodes[0], &block);
1583                 connect_block(&nodes[1], &block);
1584                 for _ in 0..TEST_FINAL_CLTV {
1585                         block.header.prev_blockhash = block.block_hash();
1586                         connect_block(&nodes[0], &block);
1587                         connect_block(&nodes[1], &block);
1588                 }
1589                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::InvalidForward { requested_forward_scid: intercept_scid }]);
1590                 check_added_monitors!(nodes[1], 1);
1591                 let htlc_timeout_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1592                 assert!(htlc_timeout_updates.update_add_htlcs.is_empty());
1593                 assert_eq!(htlc_timeout_updates.update_fail_htlcs.len(), 1);
1594                 assert!(htlc_timeout_updates.update_fail_malformed_htlcs.is_empty());
1595                 assert!(htlc_timeout_updates.update_fee.is_none());
1596
1597                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_timeout_updates.update_fail_htlcs[0]);
1598                 commitment_signed_dance!(nodes[0], nodes[1], htlc_timeout_updates.commitment_signed, false);
1599                 expect_payment_failed!(nodes[0], payment_hash, false, 0x2000 | 2, []);
1600
1601                 // Check for unknown intercept id error.
1602                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1603                 let unknown_intercept_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1604                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1605                 let unknown_intercept_id_err = nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap_err();
1606                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1607         }
1608 }
1609
1610 #[derive(PartialEq)]
1611 enum AutoRetry {
1612         Success,
1613         Spontaneous,
1614         FailAttempts,
1615         FailTimeout,
1616         FailOnRestart,
1617 }
1618
1619 #[test]
1620 fn automatic_retries() {
1621         do_automatic_retries(AutoRetry::Success);
1622         do_automatic_retries(AutoRetry::Spontaneous);
1623         do_automatic_retries(AutoRetry::FailAttempts);
1624         do_automatic_retries(AutoRetry::FailTimeout);
1625         do_automatic_retries(AutoRetry::FailOnRestart);
1626 }
1627 fn do_automatic_retries(test: AutoRetry) {
1628         // Test basic automatic payment retries in ChannelManager. See individual `test` variant comments
1629         // below.
1630         let chanmon_cfgs = create_chanmon_cfgs(3);
1631         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1632         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1633
1634         let persister;
1635         let new_chain_monitor;
1636         let node_0_deserialized;
1637
1638         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1639         let channel_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1640         let channel_id_2 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1641
1642         // Marshall data to send the payment
1643         #[cfg(feature = "std")]
1644         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1645         #[cfg(not(feature = "std"))]
1646         let payment_expiry_secs = 60 * 60;
1647         let amt_msat = 1000;
1648         let mut invoice_features = InvoiceFeatures::empty();
1649         invoice_features.set_variable_length_onion_required();
1650         invoice_features.set_payment_secret_required();
1651         invoice_features.set_basic_mpp_optional();
1652         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1653                 .with_expiry_time(payment_expiry_secs as u64)
1654                 .with_features(invoice_features);
1655         let route_params = RouteParameters {
1656                 payment_params,
1657                 final_value_msat: amt_msat,
1658                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
1659         };
1660         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
1661
1662         macro_rules! pass_failed_attempt_with_retry_along_path {
1663                 ($failing_channel_id: expr, $expect_pending_htlcs_forwardable: expr) => {
1664                         // Send a payment attempt that fails due to lack of liquidity on the second hop
1665                         check_added_monitors!(nodes[0], 1);
1666                         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1667                         let mut update_add = update_0.update_add_htlcs[0].clone();
1668                         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
1669                         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
1670                         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
1671                         nodes[1].node.process_pending_htlc_forwards();
1672                         expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1],
1673                                 vec![HTLCDestination::NextHopChannel {
1674                                         node_id: Some(nodes[2].node.get_our_node_id()),
1675                                         channel_id: $failing_channel_id,
1676                                 }]);
1677                         nodes[1].node.process_pending_htlc_forwards();
1678                         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1679                         check_added_monitors!(&nodes[1], 1);
1680                         assert!(update_1.update_fail_htlcs.len() == 1);
1681                         let fail_msg = update_1.update_fail_htlcs[0].clone();
1682
1683                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1684                         commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
1685
1686                         // Ensure the attempt fails and a new PendingHTLCsForwardable event is generated for the retry
1687                         let mut events = nodes[0].node.get_and_clear_pending_events();
1688                         match events[0] {
1689                                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
1690                                         assert_eq!(payment_hash, ev_payment_hash);
1691                                         assert_eq!(payment_failed_permanently, false);
1692                                 },
1693                                 _ => panic!("Unexpected event"),
1694                         }
1695                         if $expect_pending_htlcs_forwardable {
1696                                 assert_eq!(events.len(), 2);
1697                                 match events[1] {
1698                                         Event::PendingHTLCsForwardable { .. } => {},
1699                                         _ => panic!("Unexpected event"),
1700                                 }
1701                         } else { assert_eq!(events.len(), 1) }
1702                 }
1703         }
1704
1705         if test == AutoRetry::Success {
1706                 // Test that we can succeed on the first retry.
1707                 nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1708                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1709
1710                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
1711                 // attempt, since the initial second hop channel will be excluded from pathfinding
1712                 create_announced_chan_between_nodes(&nodes, 1, 2);
1713
1714                 // We retry payments in `process_pending_htlc_forwards`
1715                 nodes[0].node.process_pending_htlc_forwards();
1716                 check_added_monitors!(nodes[0], 1);
1717                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1718                 assert_eq!(msg_events.len(), 1);
1719                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, Some(payment_secret), msg_events.pop().unwrap(), true, None);
1720                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
1721         } else if test == AutoRetry::Spontaneous {
1722                 nodes[0].node.send_spontaneous_payment_with_retry(Some(payment_preimage), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1723                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1724
1725                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
1726                 // attempt, since the initial second hop channel will be excluded from pathfinding
1727                 create_announced_chan_between_nodes(&nodes, 1, 2);
1728
1729                 // We retry payments in `process_pending_htlc_forwards`
1730                 nodes[0].node.process_pending_htlc_forwards();
1731                 check_added_monitors!(nodes[0], 1);
1732                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1733                 assert_eq!(msg_events.len(), 1);
1734                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, None, msg_events.pop().unwrap(), true, Some(payment_preimage));
1735                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
1736         } else if test == AutoRetry::FailAttempts {
1737                 // Ensure ChannelManager will not retry a payment if it has run out of payment attempts.
1738                 nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1739                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1740
1741                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
1742                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
1743                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1744
1745                 // We retry payments in `process_pending_htlc_forwards`
1746                 nodes[0].node.process_pending_htlc_forwards();
1747                 pass_failed_attempt_with_retry_along_path!(channel_id_3, false);
1748
1749                 // Ensure we won't retry a second time.
1750                 nodes[0].node.process_pending_htlc_forwards();
1751                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1752                 assert_eq!(msg_events.len(), 0);
1753
1754                 nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
1755                 let events = nodes[0].node.get_and_clear_pending_events();
1756                 assert_eq!(events.len(), 1);
1757                 match events[0] {
1758                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
1759                                 assert_eq!(payment_hash, *ev_payment_hash);
1760                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1761                         },
1762                         _ => panic!("Unexpected event"),
1763                 }
1764         } else if test == AutoRetry::FailTimeout {
1765                 #[cfg(not(feature = "no-std"))] {
1766                         // Ensure ChannelManager will not retry a payment if it times out due to Retry::Timeout.
1767                         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
1768                         pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1769
1770                         // Advance the time so the second attempt fails due to timeout.
1771                         SinceEpoch::advance(Duration::from_secs(61));
1772
1773                         // Make sure we don't retry again.
1774                         nodes[0].node.process_pending_htlc_forwards();
1775                         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1776                         assert_eq!(msg_events.len(), 0);
1777
1778                         nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
1779                         let mut events = nodes[0].node.get_and_clear_pending_events();
1780                         assert_eq!(events.len(), 1);
1781                         match events[0] {
1782                                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
1783                                         assert_eq!(payment_hash, *ev_payment_hash);
1784                                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1785                                 },
1786                                 _ => panic!("Unexpected event"),
1787                         }
1788                 }
1789         } else if test == AutoRetry::FailOnRestart {
1790                 // Ensure ChannelManager will not retry a payment after restart, even if there were retry
1791                 // attempts remaining prior to restart.
1792                 nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
1793                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1794
1795                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
1796                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
1797                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1798
1799                 // Ensure the first retry attempt fails, with 1 retry attempt remaining
1800                 nodes[0].node.process_pending_htlc_forwards();
1801                 pass_failed_attempt_with_retry_along_path!(channel_id_3, true);
1802
1803                 // Restart the node and ensure that ChannelManager does not use its remaining retry attempt
1804                 let node_encoded = nodes[0].node.encode();
1805                 let chan_1_monitor_serialized = get_monitor!(nodes[0], channel_id_1).encode();
1806                 reload_node!(nodes[0], node_encoded, &[&chan_1_monitor_serialized], persister, new_chain_monitor, node_0_deserialized);
1807
1808                 // Make sure we don't retry again.
1809                 nodes[0].node.process_pending_htlc_forwards();
1810                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1811                 assert_eq!(msg_events.len(), 0);
1812
1813                 nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
1814                 let mut events = nodes[0].node.get_and_clear_pending_events();
1815                 assert_eq!(events.len(), 1);
1816                 match events[0] {
1817                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
1818                                 assert_eq!(payment_hash, *ev_payment_hash);
1819                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1820                         },
1821                         _ => panic!("Unexpected event"),
1822                 }
1823         }
1824 }
1825
1826 #[test]
1827 fn auto_retry_partial_failure() {
1828         // Test that we'll retry appropriately on send partial failure and retry partial failure.
1829         let chanmon_cfgs = create_chanmon_cfgs(2);
1830         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1831         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1832         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1833
1834         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1835         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1836         let chan_3_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1837
1838         // Marshall data to send the payment
1839         let amt_msat = 20_000;
1840         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
1841         #[cfg(feature = "std")]
1842         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1843         #[cfg(not(feature = "std"))]
1844         let payment_expiry_secs = 60 * 60;
1845         let mut invoice_features = InvoiceFeatures::empty();
1846         invoice_features.set_variable_length_onion_required();
1847         invoice_features.set_payment_secret_required();
1848         invoice_features.set_basic_mpp_optional();
1849         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
1850                 .with_expiry_time(payment_expiry_secs as u64)
1851                 .with_features(invoice_features);
1852         let route_params = RouteParameters {
1853                 payment_params,
1854                 final_value_msat: amt_msat,
1855                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
1856         };
1857
1858         // Ensure the first monitor update (for the initial send path1 over chan_1) succeeds, but the
1859         // second (for the initial send path2 over chan_2) fails.
1860         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1861         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
1862         // Ensure third monitor update (for the retry1's path1 over chan_1) succeeds, but the fourth (for
1863         // the retry1's path2 over chan_3) fails, and monitor updates succeed after that.
1864         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1865         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
1866         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1867
1868         // Configure the initial send, retry1 and retry2's paths.
1869         let send_route = Route {
1870                 paths: vec![
1871                         vec![RouteHop {
1872                                 pubkey: nodes[1].node.get_our_node_id(),
1873                                 node_features: nodes[1].node.node_features(),
1874                                 short_channel_id: chan_1_id,
1875                                 channel_features: nodes[1].node.channel_features(),
1876                                 fee_msat: amt_msat / 2,
1877                                 cltv_expiry_delta: 100,
1878                         }],
1879                         vec![RouteHop {
1880                                 pubkey: nodes[1].node.get_our_node_id(),
1881                                 node_features: nodes[1].node.node_features(),
1882                                 short_channel_id: chan_2_id,
1883                                 channel_features: nodes[1].node.channel_features(),
1884                                 fee_msat: amt_msat / 2,
1885                                 cltv_expiry_delta: 100,
1886                         }],
1887                 ],
1888                 payment_params: Some(route_params.payment_params.clone()),
1889         };
1890         let retry_1_route = Route {
1891                 paths: vec![
1892                         vec![RouteHop {
1893                                 pubkey: nodes[1].node.get_our_node_id(),
1894                                 node_features: nodes[1].node.node_features(),
1895                                 short_channel_id: chan_1_id,
1896                                 channel_features: nodes[1].node.channel_features(),
1897                                 fee_msat: amt_msat / 4,
1898                                 cltv_expiry_delta: 100,
1899                         }],
1900                         vec![RouteHop {
1901                                 pubkey: nodes[1].node.get_our_node_id(),
1902                                 node_features: nodes[1].node.node_features(),
1903                                 short_channel_id: chan_3_id,
1904                                 channel_features: nodes[1].node.channel_features(),
1905                                 fee_msat: amt_msat / 4,
1906                                 cltv_expiry_delta: 100,
1907                         }],
1908                 ],
1909                 payment_params: Some(route_params.payment_params.clone()),
1910         };
1911         let retry_2_route = Route {
1912                 paths: vec![
1913                         vec![RouteHop {
1914                                 pubkey: nodes[1].node.get_our_node_id(),
1915                                 node_features: nodes[1].node.node_features(),
1916                                 short_channel_id: chan_1_id,
1917                                 channel_features: nodes[1].node.channel_features(),
1918                                 fee_msat: amt_msat / 4,
1919                                 cltv_expiry_delta: 100,
1920                         }],
1921                 ],
1922                 payment_params: Some(route_params.payment_params.clone()),
1923         };
1924         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
1925         nodes[0].router.expect_find_route(RouteParameters {
1926                         payment_params: route_params.payment_params.clone(),
1927                         final_value_msat: amt_msat / 2, final_cltv_expiry_delta: TEST_FINAL_CLTV
1928                 }, Ok(retry_1_route));
1929         nodes[0].router.expect_find_route(RouteParameters {
1930                         payment_params: route_params.payment_params.clone(),
1931                         final_value_msat: amt_msat / 4, final_cltv_expiry_delta: TEST_FINAL_CLTV
1932                 }, Ok(retry_2_route));
1933
1934         // Send a payment that will partially fail on send, then partially fail on retry, then succeed.
1935         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap();
1936         let closed_chan_events = nodes[0].node.get_and_clear_pending_events();
1937         assert_eq!(closed_chan_events.len(), 2);
1938         match closed_chan_events[0] {
1939                 Event::ChannelClosed { .. } => {},
1940                 _ => panic!("Unexpected event"),
1941         }
1942         match closed_chan_events[1] {
1943                 Event::ChannelClosed { .. } => {},
1944                 _ => panic!("Unexpected event"),
1945         }
1946
1947         // Pass the first part of the payment along the path.
1948         check_added_monitors!(nodes[0], 5); // three outbound channel updates succeeded, two permanently failed
1949         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1950
1951         // First message is the first update_add, remaining messages are broadcasting channel updates and
1952         // errors for the permfailed channels
1953         assert_eq!(msg_events.len(), 5);
1954         let mut payment_event = SendEvent::from_event(msg_events.remove(0));
1955
1956         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1957         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1958         check_added_monitors!(nodes[1], 1);
1959         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1960
1961         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
1962         check_added_monitors!(nodes[0], 1);
1963         let as_second_htlc_updates = SendEvent::from_node(&nodes[0]);
1964
1965         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
1966         check_added_monitors!(nodes[0], 1);
1967         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1968
1969         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
1970         check_added_monitors!(nodes[1], 1);
1971
1972         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[0]);
1973         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[1]);
1974         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.commitment_msg);
1975         check_added_monitors!(nodes[1], 1);
1976         let (bs_second_raa, bs_second_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1977
1978         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
1979         check_added_monitors!(nodes[0], 1);
1980
1981         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_cs);
1982         check_added_monitors!(nodes[0], 1);
1983         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1984
1985         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
1986         check_added_monitors!(nodes[1], 1);
1987
1988         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
1989         nodes[1].node.process_pending_htlc_forwards();
1990         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, amt_msat);
1991         nodes[1].node.claim_funds(payment_preimage);
1992         expect_payment_claimed!(nodes[1], payment_hash, amt_msat);
1993         let bs_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1994         assert_eq!(bs_claim_update.update_fulfill_htlcs.len(), 1);
1995
1996         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_claim_update.update_fulfill_htlcs[0]);
1997         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_claim_update.commitment_signed);
1998         check_added_monitors!(nodes[0], 1);
1999         let (as_third_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2000
2001         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2002         check_added_monitors!(nodes[1], 4);
2003         let bs_second_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2004
2005         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2006         check_added_monitors!(nodes[1], 1);
2007         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2008
2009         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2010         check_added_monitors!(nodes[0], 1);
2011
2012         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[0]);
2013         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[1]);
2014         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.commitment_signed);
2015         check_added_monitors!(nodes[0], 1);
2016         let (as_fourth_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2017
2018         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_fourth_raa);
2019         check_added_monitors!(nodes[1], 1);
2020
2021         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2022         check_added_monitors!(nodes[1], 1);
2023         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2024
2025         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2026         check_added_monitors!(nodes[0], 1);
2027         expect_payment_sent!(nodes[0], payment_preimage);
2028 }
2029
2030 #[test]
2031 fn auto_retry_zero_attempts_send_error() {
2032         let chanmon_cfgs = create_chanmon_cfgs(2);
2033         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2034         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2035         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2036
2037         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2038         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2039
2040         // Marshall data to send the payment
2041         let amt_msat = 20_000;
2042         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2043         #[cfg(feature = "std")]
2044         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2045         #[cfg(not(feature = "std"))]
2046         let payment_expiry_secs = 60 * 60;
2047         let mut invoice_features = InvoiceFeatures::empty();
2048         invoice_features.set_variable_length_onion_required();
2049         invoice_features.set_payment_secret_required();
2050         invoice_features.set_basic_mpp_optional();
2051         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2052                 .with_expiry_time(payment_expiry_secs as u64)
2053                 .with_features(invoice_features);
2054         let route_params = RouteParameters {
2055                 payment_params,
2056                 final_value_msat: amt_msat,
2057                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2058         };
2059
2060         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2061         let err = nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap_err();
2062         if let PaymentSendFailure::AllFailedResendSafe(_) = err {
2063         } else { panic!("Unexpected error"); }
2064         assert_eq!(nodes[0].node.get_and_clear_pending_msg_events().len(), 2); // channel close messages
2065         assert_eq!(nodes[0].node.get_and_clear_pending_events().len(), 1); // channel close event
2066         check_added_monitors!(nodes[0], 2);
2067 }
2068
2069 #[test]
2070 fn fails_paying_after_rejected_by_payee() {
2071         let chanmon_cfgs = create_chanmon_cfgs(2);
2072         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2073         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2074         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2075
2076         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2077
2078         // Marshall data to send the payment
2079         let amt_msat = 20_000;
2080         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2081         #[cfg(feature = "std")]
2082         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2083         #[cfg(not(feature = "std"))]
2084         let payment_expiry_secs = 60 * 60;
2085         let mut invoice_features = InvoiceFeatures::empty();
2086         invoice_features.set_variable_length_onion_required();
2087         invoice_features.set_payment_secret_required();
2088         invoice_features.set_basic_mpp_optional();
2089         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2090                 .with_expiry_time(payment_expiry_secs as u64)
2091                 .with_features(invoice_features);
2092         let route_params = RouteParameters {
2093                 payment_params,
2094                 final_value_msat: amt_msat,
2095                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2096         };
2097
2098         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2099         check_added_monitors!(nodes[0], 1);
2100         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
2101         assert_eq!(events.len(), 1);
2102         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
2103         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2104         check_added_monitors!(nodes[1], 0);
2105         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
2106         expect_pending_htlcs_forwardable!(nodes[1]);
2107         expect_payment_claimable!(&nodes[1], payment_hash, payment_secret, amt_msat);
2108
2109         nodes[1].node.fail_htlc_backwards(&payment_hash);
2110         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash }]);
2111         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, payment_hash);
2112 }
2113
2114 #[test]
2115 fn retry_multi_path_single_failed_payment() {
2116         // Tests that we can/will retry after a single path of an MPP payment failed immediately
2117         let chanmon_cfgs = create_chanmon_cfgs(2);
2118         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2119         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2120         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2121
2122         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2123         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2124
2125         let amt_msat = 100_010_000;
2126
2127         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2128         #[cfg(feature = "std")]
2129         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2130         #[cfg(not(feature = "std"))]
2131         let payment_expiry_secs = 60 * 60;
2132         let mut invoice_features = InvoiceFeatures::empty();
2133         invoice_features.set_variable_length_onion_required();
2134         invoice_features.set_payment_secret_required();
2135         invoice_features.set_basic_mpp_optional();
2136         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2137                 .with_expiry_time(payment_expiry_secs as u64)
2138                 .with_features(invoice_features);
2139         let route_params = RouteParameters {
2140                 payment_params: payment_params.clone(),
2141                 final_value_msat: amt_msat,
2142                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2143         };
2144
2145         let chans = nodes[0].node.list_usable_channels();
2146         let mut route = Route {
2147                 paths: vec![
2148                         vec![RouteHop {
2149                                 pubkey: nodes[1].node.get_our_node_id(),
2150                                 node_features: nodes[1].node.node_features(),
2151                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2152                                 channel_features: nodes[1].node.channel_features(),
2153                                 fee_msat: 10_000,
2154                                 cltv_expiry_delta: 100,
2155                         }],
2156                         vec![RouteHop {
2157                                 pubkey: nodes[1].node.get_our_node_id(),
2158                                 node_features: nodes[1].node.node_features(),
2159                                 short_channel_id: chans[1].short_channel_id.unwrap(),
2160                                 channel_features: nodes[1].node.channel_features(),
2161                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2162                                 cltv_expiry_delta: 100,
2163                         }],
2164                 ],
2165                 payment_params: Some(payment_params),
2166         };
2167         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2168         // On retry, split the payment across both channels.
2169         route.paths[0][0].fee_msat = 50_000_001;
2170         route.paths[1][0].fee_msat = 50_000_000;
2171         nodes[0].router.expect_find_route(RouteParameters {
2172                         payment_params: route.payment_params.clone().unwrap(),
2173                         // Note that the second request here requests the amount we originally failed to send,
2174                         // not the amount remaining on the full payment, which should be changed.
2175                         final_value_msat: 100_000_001, final_cltv_expiry_delta: TEST_FINAL_CLTV
2176                 }, Ok(route.clone()));
2177
2178         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2179         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2180         assert_eq!(htlc_msgs.len(), 2);
2181         check_added_monitors!(nodes[0], 2);
2182 }
2183
2184 #[test]
2185 fn immediate_retry_on_failure() {
2186         // Tests that we can/will retry immediately after a failure
2187         let chanmon_cfgs = create_chanmon_cfgs(2);
2188         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2189         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2190         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2191
2192         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2193         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2194
2195         let amt_msat = 100_000_001;
2196         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2197         #[cfg(feature = "std")]
2198         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2199         #[cfg(not(feature = "std"))]
2200         let payment_expiry_secs = 60 * 60;
2201         let mut invoice_features = InvoiceFeatures::empty();
2202         invoice_features.set_variable_length_onion_required();
2203         invoice_features.set_payment_secret_required();
2204         invoice_features.set_basic_mpp_optional();
2205         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2206                 .with_expiry_time(payment_expiry_secs as u64)
2207                 .with_features(invoice_features);
2208         let route_params = RouteParameters {
2209                 payment_params,
2210                 final_value_msat: amt_msat,
2211                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2212         };
2213
2214         let chans = nodes[0].node.list_usable_channels();
2215         let mut route = Route {
2216                 paths: vec![
2217                         vec![RouteHop {
2218                                 pubkey: nodes[1].node.get_our_node_id(),
2219                                 node_features: nodes[1].node.node_features(),
2220                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2221                                 channel_features: nodes[1].node.channel_features(),
2222                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2223                                 cltv_expiry_delta: 100,
2224                         }],
2225                 ],
2226                 payment_params: Some(PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)),
2227         };
2228         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2229         // On retry, split the payment across both channels.
2230         route.paths.push(route.paths[0].clone());
2231         route.paths[0][0].short_channel_id = chans[1].short_channel_id.unwrap();
2232         route.paths[0][0].fee_msat = 50_000_000;
2233         route.paths[1][0].fee_msat = 50_000_001;
2234         nodes[0].router.expect_find_route(RouteParameters {
2235                         payment_params: route_params.payment_params.clone(),
2236                         final_value_msat: amt_msat, final_cltv_expiry_delta: TEST_FINAL_CLTV
2237                 }, Ok(route.clone()));
2238
2239         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2240         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2241         assert_eq!(htlc_msgs.len(), 2);
2242         check_added_monitors!(nodes[0], 2);
2243 }
2244
2245 #[test]
2246 fn no_extra_retries_on_back_to_back_fail() {
2247         // In a previous release, we had a race where we may exceed the payment retry count if we
2248         // get two failures in a row with the second having `all_paths_failed` set.
2249         // Generally, when we give up trying to retry a payment, we don't know for sure what the
2250         // current state of the ChannelManager event queue is. Specifically, we cannot be sure that
2251         // there are not multiple additional `PaymentPathFailed` or even `PaymentSent` events
2252         // pending which we will see later. Thus, when we previously removed the retry tracking map
2253         // entry after a `all_paths_failed` `PaymentPathFailed` event, we may have dropped the
2254         // retry entry even though more events for the same payment were still pending. This led to
2255         // us retrying a payment again even though we'd already given up on it.
2256         //
2257         // We now have a separate event - `PaymentFailed` which indicates no HTLCs remain and which
2258         // is used to remove the payment retry counter entries instead. This tests for the specific
2259         // excess-retry case while also testing `PaymentFailed` generation.
2260
2261         let chanmon_cfgs = create_chanmon_cfgs(3);
2262         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2263         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2264         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2265
2266         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2267         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2268
2269         let amt_msat = 200_000_000;
2270         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2271         #[cfg(feature = "std")]
2272         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2273         #[cfg(not(feature = "std"))]
2274         let payment_expiry_secs = 60 * 60;
2275         let mut invoice_features = InvoiceFeatures::empty();
2276         invoice_features.set_variable_length_onion_required();
2277         invoice_features.set_payment_secret_required();
2278         invoice_features.set_basic_mpp_optional();
2279         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2280                 .with_expiry_time(payment_expiry_secs as u64)
2281                 .with_features(invoice_features);
2282         let route_params = RouteParameters {
2283                 payment_params,
2284                 final_value_msat: amt_msat,
2285                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2286         };
2287
2288         let mut route = Route {
2289                 paths: vec![
2290                         vec![RouteHop {
2291                                 pubkey: nodes[1].node.get_our_node_id(),
2292                                 node_features: nodes[1].node.node_features(),
2293                                 short_channel_id: chan_1_scid,
2294                                 channel_features: nodes[1].node.channel_features(),
2295                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2296                                 cltv_expiry_delta: 100,
2297                         }, RouteHop {
2298                                 pubkey: nodes[2].node.get_our_node_id(),
2299                                 node_features: nodes[2].node.node_features(),
2300                                 short_channel_id: chan_2_scid,
2301                                 channel_features: nodes[2].node.channel_features(),
2302                                 fee_msat: 100_000_000,
2303                                 cltv_expiry_delta: 100,
2304                         }],
2305                         vec![RouteHop {
2306                                 pubkey: nodes[1].node.get_our_node_id(),
2307                                 node_features: nodes[1].node.node_features(),
2308                                 short_channel_id: chan_1_scid,
2309                                 channel_features: nodes[1].node.channel_features(),
2310                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2311                                 cltv_expiry_delta: 100,
2312                         }, RouteHop {
2313                                 pubkey: nodes[2].node.get_our_node_id(),
2314                                 node_features: nodes[2].node.node_features(),
2315                                 short_channel_id: chan_2_scid,
2316                                 channel_features: nodes[2].node.channel_features(),
2317                                 fee_msat: 100_000_000,
2318                                 cltv_expiry_delta: 100,
2319                         }]
2320                 ],
2321                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2322         };
2323         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2324         let mut second_payment_params = route_params.payment_params.clone();
2325         second_payment_params.previously_failed_channels = vec![chan_2_scid, chan_2_scid];
2326         // On retry, we'll only return one path
2327         route.paths.remove(1);
2328         route.paths[0][1].fee_msat = amt_msat;
2329         nodes[0].router.expect_find_route(RouteParameters {
2330                         payment_params: second_payment_params,
2331                         final_value_msat: amt_msat, final_cltv_expiry_delta: TEST_FINAL_CLTV,
2332                 }, Ok(route.clone()));
2333
2334         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2335         let htlc_updates = SendEvent::from_node(&nodes[0]);
2336         check_added_monitors!(nodes[0], 1);
2337         assert_eq!(htlc_updates.msgs.len(), 1);
2338
2339         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2340         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2341         check_added_monitors!(nodes[1], 1);
2342         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2343
2344         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2345         check_added_monitors!(nodes[0], 1);
2346         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2347
2348         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2349         check_added_monitors!(nodes[0], 1);
2350         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2351
2352         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2353         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2354         check_added_monitors!(nodes[1], 1);
2355         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2356
2357         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2358         check_added_monitors!(nodes[1], 1);
2359         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2360
2361         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2362         check_added_monitors!(nodes[0], 1);
2363
2364         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2365         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2366         check_added_monitors!(nodes[0], 1);
2367         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2368
2369         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2370         check_added_monitors!(nodes[1], 1);
2371         let bs_second_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2372
2373         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2374         check_added_monitors!(nodes[1], 1);
2375         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2376
2377         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.update_fail_htlcs[0]);
2378         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.commitment_signed);
2379         check_added_monitors!(nodes[0], 1);
2380
2381         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2382         check_added_monitors!(nodes[0], 1);
2383         let (as_third_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2384
2385         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2386         check_added_monitors!(nodes[1], 1);
2387         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2388         check_added_monitors!(nodes[1], 1);
2389         let bs_fourth_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2390
2391         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_fourth_raa);
2392         check_added_monitors!(nodes[0], 1);
2393
2394         // At this point A has sent two HTLCs which both failed due to lack of fee. It now has two
2395         // pending `PaymentPathFailed` events, one with `all_paths_failed` unset, and the second
2396         // with it set.
2397         //
2398         // Previously, we retried payments in an event consumer, which would retry each
2399         // `PaymentPathFailed` individually. In that setup, we had retried the payment in response to
2400         // the first `PaymentPathFailed`, then seen the second `PaymentPathFailed` with
2401         // `all_paths_failed` set and assumed the payment was completely failed. We ultimately fixed it
2402         // by adding the `PaymentFailed` event.
2403         //
2404         // Because we now retry payments as a batch, we simply return a single-path route in the
2405         // second, batched, request, have that fail, then complete the payment via `abandon_payment`.
2406         let mut events = nodes[0].node.get_and_clear_pending_events();
2407         assert_eq!(events.len(), 4);
2408         match events[0] {
2409                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2410                         assert_eq!(payment_hash, ev_payment_hash);
2411                         assert_eq!(payment_failed_permanently, false);
2412                 },
2413                 _ => panic!("Unexpected event"),
2414         }
2415         match events[1] {
2416                 Event::PendingHTLCsForwardable { .. } => {},
2417                 _ => panic!("Unexpected event"),
2418         }
2419         match events[2] {
2420                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2421                         assert_eq!(payment_hash, ev_payment_hash);
2422                         assert_eq!(payment_failed_permanently, false);
2423                 },
2424                 _ => panic!("Unexpected event"),
2425         }
2426         match events[3] {
2427                 Event::PendingHTLCsForwardable { .. } => {},
2428                 _ => panic!("Unexpected event"),
2429         }
2430
2431         nodes[0].node.process_pending_htlc_forwards();
2432         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2433         check_added_monitors!(nodes[0], 1);
2434
2435         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2436         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2437         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2438         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2439         commitment_signed_dance!(nodes[0], nodes[1], &bs_fail_update.commitment_signed, false, true);
2440
2441         let mut events = nodes[0].node.get_and_clear_pending_events();
2442         assert_eq!(events.len(), 1);
2443         match events[0] {
2444                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2445                         assert_eq!(payment_hash, ev_payment_hash);
2446                         assert_eq!(payment_failed_permanently, false);
2447                 },
2448                 _ => panic!("Unexpected event"),
2449         }
2450         nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
2451         events = nodes[0].node.get_and_clear_pending_events();
2452         assert_eq!(events.len(), 1);
2453         match events[0] {
2454                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
2455                         assert_eq!(payment_hash, *ev_payment_hash);
2456                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2457                 },
2458                 _ => panic!("Unexpected event"),
2459         }
2460 }
2461
2462 #[test]
2463 fn test_simple_partial_retry() {
2464         // In the first version of the in-`ChannelManager` payment retries, retries were sent for the
2465         // full amount of the payment, rather than only the missing amount. Here we simply test for
2466         // this by sending a payment with two parts, failing one, and retrying the second. Note that
2467         // `TestRouter` will check that the `RouteParameters` (which contain the amount) matches the
2468         // request.
2469         let chanmon_cfgs = create_chanmon_cfgs(3);
2470         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2471         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2472         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2473
2474         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2475         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2476
2477         let amt_msat = 200_000_000;
2478         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
2479         #[cfg(feature = "std")]
2480         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2481         #[cfg(not(feature = "std"))]
2482         let payment_expiry_secs = 60 * 60;
2483         let mut invoice_features = InvoiceFeatures::empty();
2484         invoice_features.set_variable_length_onion_required();
2485         invoice_features.set_payment_secret_required();
2486         invoice_features.set_basic_mpp_optional();
2487         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2488                 .with_expiry_time(payment_expiry_secs as u64)
2489                 .with_features(invoice_features);
2490         let route_params = RouteParameters {
2491                 payment_params,
2492                 final_value_msat: amt_msat,
2493                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2494         };
2495
2496         let mut route = Route {
2497                 paths: vec![
2498                         vec![RouteHop {
2499                                 pubkey: nodes[1].node.get_our_node_id(),
2500                                 node_features: nodes[1].node.node_features(),
2501                                 short_channel_id: chan_1_scid,
2502                                 channel_features: nodes[1].node.channel_features(),
2503                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2504                                 cltv_expiry_delta: 100,
2505                         }, RouteHop {
2506                                 pubkey: nodes[2].node.get_our_node_id(),
2507                                 node_features: nodes[2].node.node_features(),
2508                                 short_channel_id: chan_2_scid,
2509                                 channel_features: nodes[2].node.channel_features(),
2510                                 fee_msat: 100_000_000,
2511                                 cltv_expiry_delta: 100,
2512                         }],
2513                         vec![RouteHop {
2514                                 pubkey: nodes[1].node.get_our_node_id(),
2515                                 node_features: nodes[1].node.node_features(),
2516                                 short_channel_id: chan_1_scid,
2517                                 channel_features: nodes[1].node.channel_features(),
2518                                 fee_msat: 100_000,
2519                                 cltv_expiry_delta: 100,
2520                         }, RouteHop {
2521                                 pubkey: nodes[2].node.get_our_node_id(),
2522                                 node_features: nodes[2].node.node_features(),
2523                                 short_channel_id: chan_2_scid,
2524                                 channel_features: nodes[2].node.channel_features(),
2525                                 fee_msat: 100_000_000,
2526                                 cltv_expiry_delta: 100,
2527                         }]
2528                 ],
2529                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2530         };
2531         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2532         let mut second_payment_params = route_params.payment_params.clone();
2533         second_payment_params.previously_failed_channels = vec![chan_2_scid];
2534         // On retry, we'll only be asked for one path (or 100k sats)
2535         route.paths.remove(0);
2536         nodes[0].router.expect_find_route(RouteParameters {
2537                         payment_params: second_payment_params,
2538                         final_value_msat: amt_msat / 2, final_cltv_expiry_delta: TEST_FINAL_CLTV,
2539                 }, Ok(route.clone()));
2540
2541         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2542         let htlc_updates = SendEvent::from_node(&nodes[0]);
2543         check_added_monitors!(nodes[0], 1);
2544         assert_eq!(htlc_updates.msgs.len(), 1);
2545
2546         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2547         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2548         check_added_monitors!(nodes[1], 1);
2549         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2550
2551         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2552         check_added_monitors!(nodes[0], 1);
2553         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2554
2555         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2556         check_added_monitors!(nodes[0], 1);
2557         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2558
2559         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2560         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2561         check_added_monitors!(nodes[1], 1);
2562         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2563
2564         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2565         check_added_monitors!(nodes[1], 1);
2566         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2567
2568         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2569         check_added_monitors!(nodes[0], 1);
2570
2571         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2572         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2573         check_added_monitors!(nodes[0], 1);
2574         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2575
2576         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2577         check_added_monitors!(nodes[1], 1);
2578
2579         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2580         check_added_monitors!(nodes[1], 1);
2581
2582         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2583
2584         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2585         check_added_monitors!(nodes[0], 1);
2586
2587         let mut events = nodes[0].node.get_and_clear_pending_events();
2588         assert_eq!(events.len(), 2);
2589         match events[0] {
2590                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2591                         assert_eq!(payment_hash, ev_payment_hash);
2592                         assert_eq!(payment_failed_permanently, false);
2593                 },
2594                 _ => panic!("Unexpected event"),
2595         }
2596         match events[1] {
2597                 Event::PendingHTLCsForwardable { .. } => {},
2598                 _ => panic!("Unexpected event"),
2599         }
2600
2601         nodes[0].node.process_pending_htlc_forwards();
2602         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2603         check_added_monitors!(nodes[0], 1);
2604
2605         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2606         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2607
2608         expect_pending_htlcs_forwardable!(nodes[1]);
2609         check_added_monitors!(nodes[1], 1);
2610
2611         let bs_forward_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
2612         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[0]);
2613         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[1]);
2614         commitment_signed_dance!(nodes[2], nodes[1], &bs_forward_update.commitment_signed, false);
2615
2616         expect_pending_htlcs_forwardable!(nodes[2]);
2617         expect_payment_claimable!(nodes[2], payment_hash, payment_secret, amt_msat);
2618 }