_ Git - rust-lightning/blob - lightning/src/ln/payment_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Tests that test the payment retry logic in ChannelManager, including various edge-cases around
  11 //! serialization ordering between ChannelManager/ChannelMonitors and ensuring we can still retry
  12 //! payments thereafter.
  13
  14 use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Listen, Watch};
  15 use crate::chain::channelmonitor::{ANTI_REORG_DELAY, LATENCY_GRACE_PERIOD_BLOCKS};
  16 use crate::chain::keysinterface::EntropySource;
  17 use crate::chain::transaction::OutPoint;
  18 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
  19 use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, ChannelManager, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS};
  20 use crate::ln::features::InvoiceFeatures;
  21 use crate::ln::msgs;
  22 use crate::ln::msgs::ChannelMessageHandler;
  23 use crate::ln::outbound_payment::Retry;
  24 use crate::routing::gossip::RoutingFees;
  25 use crate::routing::router::{get_route, PaymentParameters, Route, RouteHint, RouteHintHop, RouteHop, RouteParameters};
  26 use crate::util::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
  27 use crate::util::test_utils;
  28 use crate::util::errors::APIError;
  29 use crate::util::ser::Writeable;
  30
  31 use bitcoin::{Block, BlockHeader, TxMerkleNode};
  32 use bitcoin::hashes::Hash;
  33 use bitcoin::network::constants::Network;
  34
  35 use crate::prelude::*;
  36
  37 use crate::ln::functional_test_utils::*;
  38 use crate::routing::gossip::NodeId;
  39 #[cfg(feature = "std")]
  40 use {
  41         crate::util::time::tests::SinceEpoch,
  42         std::time::{SystemTime, Duration}
  43 };
  44
  45 #[test]
  46 fn retry_single_path_payment() {
  47         let chanmon_cfgs = create_chanmon_cfgs(3);
  48         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
  49         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
  50         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
  51
  52         let _chan_0 = create_announced_chan_between_nodes(&nodes, 0, 1);
  53         let chan_1 = create_announced_chan_between_nodes(&nodes, 2, 1);
  54         // Rebalance to find a route
  55         send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000);
  56
  57         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 100_000);
  58
  59         // Rebalance so that the first hop fails.
  60         send_payment(&nodes[1], &vec!(&nodes[2])[..], 2_000_000);
  61
  62         // Make sure the payment fails on the first hop.
  63         let payment_id = PaymentId(payment_hash.0);
  64         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
  65         check_added_monitors!(nodes[0], 1);
  66         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
  67         assert_eq!(events.len(), 1);
  68         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
  69         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
  70         check_added_monitors!(nodes[1], 0);
  71         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  72         expect_pending_htlcs_forwardable!(nodes[1]);
  73         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_1.2 }]);
  74         let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
  75         assert!(htlc_updates.update_add_htlcs.is_empty());
  76         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
  77         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
  78         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
  79         check_added_monitors!(nodes[1], 1);
  80         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
  81         commitment_signed_dance!(nodes[0], nodes[1], htlc_updates.commitment_signed, false);
  82         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
  83
  84         // Rebalance the channel so the retry succeeds.
  85         send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000);
  86
  87         // Mine two blocks (we expire retries after 3, so this will check that we don't expire early)
  88         connect_blocks(&nodes[0], 2);
  89
  90         // Retry the payment and make sure it succeeds.
  91         nodes[0].node.retry_payment(&route, payment_id).unwrap();
  92         check_added_monitors!(nodes[0], 1);
  93         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
  94         assert_eq!(events.len(), 1);
  95         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 100_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
  96         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
  97 }
  98
  99 #[test]
 100 fn mpp_failure() {
 101         let chanmon_cfgs = create_chanmon_cfgs(4);
 102         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 103         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 104         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 105
 106         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
 107         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
 108         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
 109         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3).0.contents.short_channel_id;
 110
 111         let (mut route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
 112         let path = route.paths[0].clone();
 113         route.paths.push(path);
 114         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
 115         route.paths[0][0].short_channel_id = chan_1_id;
 116         route.paths[0][1].short_channel_id = chan_3_id;
 117         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
 118         route.paths[1][0].short_channel_id = chan_2_id;
 119         route.paths[1][1].short_channel_id = chan_4_id;
 120         send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
 121         fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
 122 }
 123
 124 #[test]
 125 fn mpp_retry() {
 126         let chanmon_cfgs = create_chanmon_cfgs(4);
 127         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 128         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 129         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 130
 131         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 132         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 133         let (chan_3_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 134         let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes(&nodes, 3, 2);
 135         // Rebalance
 136         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 137
 138         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 1_000_000);
 139         let path = route.paths[0].clone();
 140         route.paths.push(path);
 141         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
 142         route.paths[0][0].short_channel_id = chan_1_update.contents.short_channel_id;
 143         route.paths[0][1].short_channel_id = chan_3_update.contents.short_channel_id;
 144         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
 145         route.paths[1][0].short_channel_id = chan_2_update.contents.short_channel_id;
 146         route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id;
 147
 148         // Initiate the MPP payment.
 149         let payment_id = PaymentId(payment_hash.0);
 150         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
 151         check_added_monitors!(nodes[0], 2); // one monitor per path
 152         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 153         assert_eq!(events.len(), 2);
 154
 155         // Pass half of the payment along the success path.
 156         let (success_path_msgs, mut events) = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &events);
 157         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), success_path_msgs, false, None);
 158
 159         // Add the HTLC along the first hop.
 160         let (fail_path_msgs_1, _events) = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &events);
 161         let (update_add, commitment_signed) = match fail_path_msgs_1 {
 162                 MessageSendEvent::UpdateHTLCs { node_id: _, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 163                         assert_eq!(update_add_htlcs.len(), 1);
 164                         assert!(update_fail_htlcs.is_empty());
 165                         assert!(update_fulfill_htlcs.is_empty());
 166                         assert!(update_fail_malformed_htlcs.is_empty());
 167                         assert!(update_fee.is_none());
 168                         (update_add_htlcs[0].clone(), commitment_signed.clone())
 169                 },
 170                 _ => panic!("Unexpected event"),
 171         };
 172         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
 173         commitment_signed_dance!(nodes[2], nodes[0], commitment_signed, false);
 174
 175         // Attempt to forward the payment and complete the 2nd path's failure.
 176         expect_pending_htlcs_forwardable!(&nodes[2]);
 177         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id }]);
 178         let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 179         assert!(htlc_updates.update_add_htlcs.is_empty());
 180         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 181         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 182         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 183         check_added_monitors!(nodes[2], 1);
 184         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 185         commitment_signed_dance!(nodes[0], nodes[2], htlc_updates.commitment_signed, false);
 186         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 187
 188         // Rebalance the channel so the second half of the payment can succeed.
 189         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 190
 191         // Make sure it errors as expected given a too-large amount.
 192         if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, payment_id) {
 193                 assert!(err.contains("over total_payment_amt_msat"));
 194         } else { panic!("Unexpected error"); }
 195
 196         // Make sure it errors as expected given the wrong payment_id.
 197         if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, PaymentId([0; 32])) {
 198                 assert!(err.contains("not found"));
 199         } else { panic!("Unexpected error"); }
 200
 201         // Retry the second half of the payment and make sure it succeeds.
 202         let mut path = route.clone();
 203         path.paths.remove(0);
 204         nodes[0].node.retry_payment(&path, payment_id).unwrap();
 205         check_added_monitors!(nodes[0], 1);
 206         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 207         assert_eq!(events.len(), 1);
 208         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 209         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 210 }
 211
 212 fn do_mpp_receive_timeout(send_partial_mpp: bool) {
 213         let chanmon_cfgs = create_chanmon_cfgs(4);
 214         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 215         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 216         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 217
 218         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 219         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 220         let (chan_3_update, _, chan_3_id, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 221         let (chan_4_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 222
 223         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 100_000);
 224         let path = route.paths[0].clone();
 225         route.paths.push(path);
 226         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
 227         route.paths[0][0].short_channel_id = chan_1_update.contents.short_channel_id;
 228         route.paths[0][1].short_channel_id = chan_3_update.contents.short_channel_id;
 229         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
 230         route.paths[1][0].short_channel_id = chan_2_update.contents.short_channel_id;
 231         route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id;
 232
 233         // Initiate the MPP payment.
 234         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 235         check_added_monitors!(nodes[0], 2); // one monitor per path
 236         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 237         assert_eq!(events.len(), 2);
 238
 239         // Pass half of the payment along the first path.
 240         let (node_1_msgs, mut events) = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &events);
 241         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_1_msgs, false, None);
 242
 243         if send_partial_mpp {
 244                 // Time out the partial MPP
 245                 for _ in 0..MPP_TIMEOUT_TICKS {
 246                         nodes[3].node.timer_tick_occurred();
 247                 }
 248
 249                 // Failed HTLC from node 3 -> 1
 250                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 251                 let htlc_fail_updates_3_1 = get_htlc_update_msgs!(nodes[3], nodes[1].node.get_our_node_id());
 252                 assert_eq!(htlc_fail_updates_3_1.update_fail_htlcs.len(), 1);
 253                 nodes[1].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &htlc_fail_updates_3_1.update_fail_htlcs[0]);
 254                 check_added_monitors!(nodes[3], 1);
 255                 commitment_signed_dance!(nodes[1], nodes[3], htlc_fail_updates_3_1.commitment_signed, false);
 256
 257                 // Failed HTLC from node 1 -> 0
 258                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_3_id }]);
 259                 let htlc_fail_updates_1_0 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 260                 assert_eq!(htlc_fail_updates_1_0.update_fail_htlcs.len(), 1);
 261                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates_1_0.update_fail_htlcs[0]);
 262                 check_added_monitors!(nodes[1], 1);
 263                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates_1_0.commitment_signed, false);
 264
 265                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain().expected_htlc_error_data(23, &[][..]));
 266         } else {
 267                 // Pass half of the payment along the second path.
 268                 let (node_2_msgs, _events) = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &events);
 269                 pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_2_msgs, true, None);
 270
 271                 // Even after MPP_TIMEOUT_TICKS we should not timeout the MPP if we have all the parts
 272                 for _ in 0..MPP_TIMEOUT_TICKS {
 273                         nodes[3].node.timer_tick_occurred();
 274                 }
 275
 276                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 277         }
 278 }
 279
 280 #[test]
 281 fn mpp_receive_timeout() {
 282         do_mpp_receive_timeout(true);
 283         do_mpp_receive_timeout(false);
 284 }
 285
 286 #[test]
 287 fn retry_expired_payment() {
 288         let chanmon_cfgs = create_chanmon_cfgs(3);
 289         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 290         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 291         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 292
 293         let _chan_0 = create_announced_chan_between_nodes(&nodes, 0, 1);
 294         let chan_1 = create_announced_chan_between_nodes(&nodes, 2, 1);
 295         // Rebalance to find a route
 296         send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000);
 297
 298         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 100_000);
 299
 300         // Rebalance so that the first hop fails.
 301         send_payment(&nodes[1], &vec!(&nodes[2])[..], 2_000_000);
 302
 303         // Make sure the payment fails on the first hop.
 304         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 305         check_added_monitors!(nodes[0], 1);
 306         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 307         assert_eq!(events.len(), 1);
 308         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
 309         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 310         check_added_monitors!(nodes[1], 0);
 311         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 312         expect_pending_htlcs_forwardable!(nodes[1]);
 313         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_1.2 }]);
 314         let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 315         assert!(htlc_updates.update_add_htlcs.is_empty());
 316         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 317         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 318         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 319         check_added_monitors!(nodes[1], 1);
 320         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 321         commitment_signed_dance!(nodes[0], nodes[1], htlc_updates.commitment_signed, false);
 322         expect_payment_failed!(nodes[0], payment_hash, false);
 323
 324         // Mine blocks so the payment will have expired.
 325         connect_blocks(&nodes[0], 3);
 326
 327         // Retry the payment and make sure it errors as expected.
 328         if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, PaymentId(payment_hash.0)) {
 329                 assert!(err.contains("not found"));
 330         } else {
 331                 panic!("Unexpected error");
 332         }
 333 }
 334
 335 #[test]
 336 fn no_pending_leak_on_initial_send_failure() {
 337         // In an earlier version of our payment tracking, we'd have a retry entry even when the initial
 338         // HTLC for payment failed to send due to local channel errors (e.g. peer disconnected). In this
 339         // case, the user wouldn't have a PaymentId to retry the payment with, but we'd think we have a
 340         // pending payment forever and never time it out.
 341         // Here we test exactly that - retrying a payment when a peer was disconnected on the first
 342         // try, and then check that no pending payment is being tracked.
 343         let chanmon_cfgs = create_chanmon_cfgs(2);
 344         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 345         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 346         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 347
 348         create_announced_chan_between_nodes(&nodes, 0, 1);
 349
 350         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 351
 352         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 353         nodes[1].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 354
 355         unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)),
 356                 true, APIError::ChannelUnavailable { ref err },
 357                 assert_eq!(err, "Peer for first hop currently disconnected/pending monitor update!"));
 358
 359         assert!(!nodes[0].node.has_pending_payments());
 360 }
 361
 362 fn do_retry_with_no_persist(confirm_before_reload: bool) {
 363         // If we send a pending payment and `send_payment` returns success, we should always either
 364         // return a payment failure event or a payment success event, and on failure the payment should
 365         // be retryable.
 366         //
 367         // In order to do so when the ChannelManager isn't immediately persisted (which is normal - its
 368         // always persisted asynchronously), the ChannelManager has to reload some payment data from
 369         // ChannelMonitor(s) in some cases. This tests that reloading.
 370         //
 371         // `confirm_before_reload` confirms the channel-closing commitment transaction on-chain prior
 372         // to reloading the ChannelManager, increasing test coverage in ChannelMonitor HTLC tracking
 373         // which has separate codepaths for "commitment transaction already confirmed" and not.
 374         let chanmon_cfgs = create_chanmon_cfgs(3);
 375         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 376         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 377         let persister: test_utils::TestPersister;
 378         let new_chain_monitor: test_utils::TestChainMonitor;
 379         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 380         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 381
 382         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 383         let (_, _, chan_id_2, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
 384
 385         // Serialize the ChannelManager prior to sending payments
 386         let nodes_0_serialized = nodes[0].node.encode();
 387
 388         // Send two payments - one which will get to nodes[2] and will be claimed, one which we'll time
 389         // out and retry.
 390         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 391         let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000);
 392         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 393         check_added_monitors!(nodes[0], 1);
 394
 395         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 396         assert_eq!(events.len(), 1);
 397         let payment_event = SendEvent::from_event(events.pop().unwrap());
 398         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 399
 400         // We relay the payment to nodes[1] while its disconnected from nodes[2], causing the payment
 401         // to be returned immediately to nodes[0], without having nodes[2] fail the inbound payment
 402         // which would prevent retry.
 403         nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id(), false);
 404         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 405
 406         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 407         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true);
 408         // nodes[1] now immediately fails the HTLC as the next-hop channel is disconnected
 409         let _ = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 410
 411         reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 412
 413         let as_commitment_tx = get_local_commitment_txn!(nodes[0], chan_id)[0].clone();
 414         if confirm_before_reload {
 415                 mine_transaction(&nodes[0], &as_commitment_tx);
 416                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 417         }
 418
 419         // The ChannelMonitor should always be the latest version, as we're required to persist it
 420         // during the `commitment_signed_dance!()`.
 421         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 422         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 423
 424         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 425         // force-close the channel.
 426         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 427         assert!(nodes[0].node.list_channels().is_empty());
 428         assert!(nodes[0].node.has_pending_payments());
 429         let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 430         assert_eq!(as_broadcasted_txn.len(), 1);
 431         assert_eq!(as_broadcasted_txn[0], as_commitment_tx);
 432
 433         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 434         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
 435         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 436
 437         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 438         // error, as the channel has hit the chain.
 439         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
 440         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 441         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 442         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 443         assert_eq!(as_err.len(), 1);
 444         match as_err[0] {
 445                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 446                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 447                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 448                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()) });
 449                         check_added_monitors!(nodes[1], 1);
 450                         assert_eq!(nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 451                 },
 452                 _ => panic!("Unexpected event"),
 453         }
 454         check_closed_broadcast!(nodes[1], false);
 455
 456         // Now claim the first payment, which should allow nodes[1] to claim the payment on-chain when
 457         // we close in a moment.
 458         nodes[2].node.claim_funds(payment_preimage_1);
 459         check_added_monitors!(nodes[2], 1);
 460         expect_payment_claimed!(nodes[2], payment_hash_1, 1_000_000);
 461
 462         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 463         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 464         check_added_monitors!(nodes[1], 1);
 465         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 466         expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], None, false, false);
 467
 468         if confirm_before_reload {
 469                 let best_block = nodes[0].blocks.lock().unwrap().last().unwrap().clone();
 470                 nodes[0].node.best_block_updated(&best_block.0.header, best_block.1);
 471         }
 472
 473         // Create a new channel on which to retry the payment before we fail the payment via the
 474         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 475         // connecting several blocks while creating the channel (implying time has passed).
 476         create_announced_chan_between_nodes(&nodes, 0, 1);
 477         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 478
 479         mine_transaction(&nodes[1], &as_commitment_tx);
 480         let bs_htlc_claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 481         assert_eq!(bs_htlc_claim_txn.len(), 1);
 482         check_spends!(bs_htlc_claim_txn[0], as_commitment_tx);
 483
 484         if !confirm_before_reload {
 485                 mine_transaction(&nodes[0], &as_commitment_tx);
 486         }
 487         mine_transaction(&nodes[0], &bs_htlc_claim_txn[0]);
 488         expect_payment_sent!(nodes[0], payment_preimage_1);
 489         connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
 490         let as_htlc_timeout_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 491         assert_eq!(as_htlc_timeout_txn.len(), 2);
 492         let (first_htlc_timeout_tx, second_htlc_timeout_tx) = (&as_htlc_timeout_txn[0], &as_htlc_timeout_txn[1]);
 493         check_spends!(first_htlc_timeout_tx, as_commitment_tx);
 494         check_spends!(second_htlc_timeout_tx, as_commitment_tx);
 495         if first_htlc_timeout_tx.input[0].previous_output == bs_htlc_claim_txn[0].input[0].previous_output {
 496                 confirm_transaction(&nodes[0], &second_htlc_timeout_tx);
 497         } else {
 498                 confirm_transaction(&nodes[0], &first_htlc_timeout_tx);
 499         }
 500         nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 501         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 502
 503         // Finally, retry the payment (which was reloaded from the ChannelMonitor when nodes[0] was
 504         // reloaded) via a route over the new channel, which work without issue and eventually be
 505         // received and claimed at the recipient just like any other payment.
 506         let (mut new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 507
 508         // Update the fee on the middle hop to ensure PaymentSent events have the correct (retried) fee
 509         // and not the original fee. We also update node[1]'s relevant config as
 510         // do_claim_payment_along_route expects us to never overpay.
 511         {
 512                 let per_peer_state = nodes[1].node.per_peer_state.read().unwrap();
 513                 let mut peer_state = per_peer_state.get(&nodes[2].node.get_our_node_id())
 514                         .unwrap().lock().unwrap();
 515                 let mut channel = peer_state.channel_by_id.get_mut(&chan_id_2).unwrap();
 516                 let mut new_config = channel.config();
 517                 new_config.forwarding_fee_base_msat += 100_000;
 518                 channel.update_config(&new_config);
 519                 new_route.paths[0][0].fee_msat += 100_000;
 520         }
 521
 522         // Force expiration of the channel's previous config.
 523         for _ in 0..EXPIRE_PREV_CONFIG_TICKS {
 524                 nodes[1].node.timer_tick_occurred();
 525         }
 526
 527         assert!(nodes[0].node.retry_payment(&new_route, payment_id_1).is_err()); // Shouldn't be allowed to retry a fulfilled payment
 528         nodes[0].node.retry_payment(&new_route, PaymentId(payment_hash.0)).unwrap();
 529         check_added_monitors!(nodes[0], 1);
 530         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 531         assert_eq!(events.len(), 1);
 532         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 533         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
 534         expect_payment_sent!(nodes[0], payment_preimage, Some(new_route.paths[0][0].fee_msat));
 535 }
 536
 537 #[test]
 538 fn retry_with_no_persist() {
 539         do_retry_with_no_persist(true);
 540         do_retry_with_no_persist(false);
 541 }
 542
 543 fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 544         // Test that an off-chain completed payment is not retryable on restart. This was previously
 545         // broken for dust payments, but we test for both dust and non-dust payments.
 546         //
 547         // `use_dust` switches to using a dust HTLC, which results in the HTLC not having an on-chain
 548         // output at all.
 549         let chanmon_cfgs = create_chanmon_cfgs(3);
 550         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 551
 552         let mut manually_accept_config = test_default_channel_config();
 553         manually_accept_config.manually_accept_inbound_channels = true;
 554
 555         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(manually_accept_config), None]);
 556
 557         let first_persister: test_utils::TestPersister;
 558         let first_new_chain_monitor: test_utils::TestChainMonitor;
 559         let first_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 560         let second_persister: test_utils::TestPersister;
 561         let second_new_chain_monitor: test_utils::TestChainMonitor;
 562         let second_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 563         let third_persister: test_utils::TestPersister;
 564         let third_new_chain_monitor: test_utils::TestChainMonitor;
 565         let third_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 566
 567         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 568
 569         // Because we set nodes[1] to manually accept channels, just open a 0-conf channel.
 570         let (funding_tx, chan_id) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 571         confirm_transaction(&nodes[0], &funding_tx);
 572         confirm_transaction(&nodes[1], &funding_tx);
 573         // Ignore the announcement_signatures messages
 574         nodes[0].node.get_and_clear_pending_msg_events();
 575         nodes[1].node.get_and_clear_pending_msg_events();
 576         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 577
 578         // Serialize the ChannelManager prior to sending payments
 579         let mut nodes_0_serialized = nodes[0].node.encode();
 580
 581         let route = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 }).0;
 582         let (payment_preimage, payment_hash, payment_secret, payment_id) = send_along_route(&nodes[0], route, &[&nodes[1], &nodes[2]], if use_dust { 1_000 } else { 1_000_000 });
 583
 584         // The ChannelMonitor should always be the latest version, as we're required to persist it
 585         // during the `commitment_signed_dance!()`.
 586         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 587
 588         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized], first_persister, first_new_chain_monitor, first_nodes_0_deserialized);
 589         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 590
 591         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 592         // force-close the channel.
 593         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 594         assert!(nodes[0].node.list_channels().is_empty());
 595         assert!(nodes[0].node.has_pending_payments());
 596         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 597
 598         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
 599         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 600
 601         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 602         // error, as the channel has hit the chain.
 603         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
 604         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 605         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 606         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 607         assert_eq!(as_err.len(), 1);
 608         let bs_commitment_tx;
 609         match as_err[0] {
 610                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 611                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 612                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 613                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()) });
 614                         check_added_monitors!(nodes[1], 1);
 615                         bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 616                 },
 617                 _ => panic!("Unexpected event"),
 618         }
 619         check_closed_broadcast!(nodes[1], false);
 620
 621         // Now fail back the payment from nodes[2] to nodes[1]. This doesn't really matter as the
 622         // previous hop channel is already on-chain, but it makes nodes[2] willing to see additional
 623         // incoming HTLCs with the same payment hash later.
 624         nodes[2].node.fail_htlc_backwards(&payment_hash);
 625         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
 626         check_added_monitors!(nodes[2], 1);
 627
 628         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 629         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fail_htlcs[0]);
 630         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 631         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1],
 632                 [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 633
 634         // Connect the HTLC-Timeout transaction, timing out the HTLC on both nodes (but not confirming
 635         // the HTLC-Timeout transaction beyond 1 conf). For dust HTLCs, the HTLC is considered resolved
 636         // after the commitment transaction, so always connect the commitment transaction.
 637         mine_transaction(&nodes[0], &bs_commitment_tx[0]);
 638         mine_transaction(&nodes[1], &bs_commitment_tx[0]);
 639         if !use_dust {
 640                 connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1 + (MIN_CLTV_EXPIRY_DELTA as u32));
 641                 connect_blocks(&nodes[1], TEST_FINAL_CLTV - 1 + (MIN_CLTV_EXPIRY_DELTA as u32));
 642                 let as_htlc_timeout = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 643                 check_spends!(as_htlc_timeout[0], bs_commitment_tx[0]);
 644                 assert_eq!(as_htlc_timeout.len(), 1);
 645
 646                 mine_transaction(&nodes[0], &as_htlc_timeout[0]);
 647                 // nodes[0] may rebroadcast (or RBF-bump) its HTLC-Timeout, so wipe the announced set.
 648                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 649                 mine_transaction(&nodes[1], &as_htlc_timeout[0]);
 650         }
 651
 652         // Create a new channel on which to retry the payment before we fail the payment via the
 653         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 654         // connecting several blocks while creating the channel (implying time has passed).
 655         // We do this with a zero-conf channel to avoid connecting blocks as a side-effect.
 656         let (_, chan_id_3) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 657         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 658
 659         // If we attempt to retry prior to the HTLC-Timeout (or commitment transaction, for dust HTLCs)
 660         // confirming, we will fail as it's considered still-pending...
 661         let (new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 });
 662         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_err());
 663         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 664
 665         // After ANTI_REORG_DELAY confirmations, the HTLC should be failed and we can try the payment
 666         // again. We serialize the node first as we'll then test retrying the HTLC after a restart
 667         // (which should also still work).
 668         connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
 669         connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 670         // We set mpp_parts_remain to avoid having abandon_payment called
 671         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 672
 673         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 674         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 675         nodes_0_serialized = nodes[0].node.encode();
 676
 677         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_ok());
 678         assert!(!nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 679
 680         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], second_persister, second_new_chain_monitor, second_nodes_0_deserialized);
 681         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 682
 683         reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 684
 685         // Now resend the payment, delivering the HTLC and actually claiming it this time. This ensures
 686         // the payment is not (spuriously) listed as still pending.
 687         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_ok());
 688         check_added_monitors!(nodes[0], 1);
 689         pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], if use_dust { 1_000 } else { 1_000_000 }, payment_hash, payment_secret);
 690         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
 691
 692         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_err());
 693         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 694
 695         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 696         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 697         nodes_0_serialized = nodes[0].node.encode();
 698
 699         // Ensure that after reload we cannot retry the payment.
 700         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], third_persister, third_new_chain_monitor, third_nodes_0_deserialized);
 701         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 702
 703         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 704
 705         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_err());
 706         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 707 }
 708
 709 #[test]
 710 fn test_completed_payment_not_retryable_on_reload() {
 711         do_test_completed_payment_not_retryable_on_reload(true);
 712         do_test_completed_payment_not_retryable_on_reload(false);
 713 }
 714
 715
 716 fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, confirm_commitment_tx: bool, payment_timeout: bool) {
 717         // When a Channel is closed, any outbound HTLCs which were relayed through it are simply
 718         // dropped when the Channel is. From there, the ChannelManager relies on the ChannelMonitor
 719         // having a copy of the relevant fail-/claim-back data and processes the HTLC fail/claim when
 720         // the ChannelMonitor tells it to.
 721         //
 722         // If, due to an on-chain event, an HTLC is failed/claimed, we should avoid providing the
 723         // ChannelManager the HTLC event until after the monitor is re-persisted. This should prevent a
 724         // duplicate HTLC fail/claim (e.g. via a PaymentPathFailed event).
 725         let chanmon_cfgs = create_chanmon_cfgs(2);
 726         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 727         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 728         let persister: test_utils::TestPersister;
 729         let new_chain_monitor: test_utils::TestChainMonitor;
 730         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 731         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 732
 733         let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1);
 734
 735         // Route a payment, but force-close the channel before the HTLC fulfill message arrives at
 736         // nodes[0].
 737         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 10_000_000);
 738         nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap();
 739         check_closed_broadcast!(nodes[0], true);
 740         check_added_monitors!(nodes[0], 1);
 741         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 742
 743         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 744         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 745
 746         // Connect blocks until the CLTV timeout is up so that we get an HTLC-Timeout transaction
 747         connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
 748         let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 749         assert_eq!(node_txn.len(), 3);
 750         assert_eq!(node_txn[0], node_txn[1]);
 751         check_spends!(node_txn[1], funding_tx);
 752         check_spends!(node_txn[2], node_txn[1]);
 753         let timeout_txn = vec![node_txn[2].clone()];
 754
 755         nodes[1].node.claim_funds(payment_preimage);
 756         check_added_monitors!(nodes[1], 1);
 757         expect_payment_claimed!(nodes[1], payment_hash, 10_000_000);
 758
 759         let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 };
 760         connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[1].clone()]});
 761         check_closed_broadcast!(nodes[1], true);
 762         check_added_monitors!(nodes[1], 1);
 763         check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 764         let claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 765         assert_eq!(claim_txn.len(), 1);
 766         check_spends!(claim_txn[0], node_txn[1]);
 767
 768         header.prev_blockhash = nodes[0].best_block_hash();
 769         connect_block(&nodes[0], &Block { header, txdata: vec![node_txn[1].clone()]});
 770
 771         if confirm_commitment_tx {
 772                 connect_blocks(&nodes[0], BREAKDOWN_TIMEOUT as u32 - 1);
 773         }
 774
 775         header.prev_blockhash = nodes[0].best_block_hash();
 776         let claim_block = Block { header, txdata: if payment_timeout { timeout_txn } else { vec![claim_txn[0].clone()] } };
 777
 778         if payment_timeout {
 779                 assert!(confirm_commitment_tx); // Otherwise we're spending below our CSV!
 780                 connect_block(&nodes[0], &claim_block);
 781                 connect_blocks(&nodes[0], ANTI_REORG_DELAY - 2);
 782         }
 783
 784         // Now connect the HTLC claim transaction with the ChainMonitor-generated ChannelMonitor update
 785         // returning InProgress. This should cause the claim event to never make its way to the
 786         // ChannelManager.
 787         chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap().clear();
 788         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
 789
 790         if payment_timeout {
 791                 connect_blocks(&nodes[0], 1);
 792         } else {
 793                 connect_block(&nodes[0], &claim_block);
 794         }
 795
 796         let funding_txo = OutPoint { txid: funding_tx.txid(), index: 0 };
 797         let mon_updates: Vec<_> = chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap()
 798                 .get_mut(&funding_txo).unwrap().drain().collect();
 799         // If we are using chain::Confirm instead of chain::Listen, we will get the same update twice.
 800         // If we're testing connection idempotency we may get substantially more.
 801         assert!(mon_updates.len() >= 1);
 802         assert!(nodes[0].chain_monitor.release_pending_monitor_events().is_empty());
 803         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 804
 805         // If we persist the ChannelManager here, we should get the PaymentSent event after
 806         // deserialization.
 807         let mut chan_manager_serialized = Vec::new();
 808         if !persist_manager_post_event {
 809                 chan_manager_serialized = nodes[0].node.encode();
 810         }
 811
 812         // Now persist the ChannelMonitor and inform the ChainMonitor that we're done, generating the
 813         // payment sent event.
 814         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
 815         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 816         for update in mon_updates {
 817                 nodes[0].chain_monitor.chain_monitor.channel_monitor_updated(funding_txo, update).unwrap();
 818         }
 819         if payment_timeout {
 820                 expect_payment_failed!(nodes[0], payment_hash, false);
 821         } else {
 822                 expect_payment_sent!(nodes[0], payment_preimage);
 823         }
 824
 825         // If we persist the ChannelManager after we get the PaymentSent event, we shouldn't get it
 826         // twice.
 827         if persist_manager_post_event {
 828                 chan_manager_serialized = nodes[0].node.encode();
 829         }
 830
 831         // Now reload nodes[0]...
 832         reload_node!(nodes[0], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 833
 834         if persist_manager_post_event {
 835                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 836         } else if payment_timeout {
 837                 expect_payment_failed!(nodes[0], payment_hash, false);
 838         } else {
 839                 expect_payment_sent!(nodes[0], payment_preimage);
 840         }
 841
 842         // Note that if we re-connect the block which exposed nodes[0] to the payment preimage (but
 843         // which the current ChannelMonitor has not seen), the ChannelManager's de-duplication of
 844         // payment events should kick in, leaving us with no pending events here.
 845         let height = nodes[0].blocks.lock().unwrap().len() as u32 - 1;
 846         nodes[0].chain_monitor.chain_monitor.block_connected(&claim_block, height);
 847         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 848 }
 849
 850 #[test]
 851 fn test_dup_htlc_onchain_fails_on_reload() {
 852         do_test_dup_htlc_onchain_fails_on_reload(true, true, true);
 853         do_test_dup_htlc_onchain_fails_on_reload(true, true, false);
 854         do_test_dup_htlc_onchain_fails_on_reload(true, false, false);
 855         do_test_dup_htlc_onchain_fails_on_reload(false, true, true);
 856         do_test_dup_htlc_onchain_fails_on_reload(false, true, false);
 857         do_test_dup_htlc_onchain_fails_on_reload(false, false, false);
 858 }
 859
 860 #[test]
 861 fn test_fulfill_restart_failure() {
 862         // When we receive an update_fulfill_htlc message, we immediately consider the HTLC fully
 863         // fulfilled. At this point, the peer can reconnect and decide to either fulfill the HTLC
 864         // again, or fail it, giving us free money.
 865         //
 866         // Of course probably they won't fail it and give us free money, but because we have code to
 867         // handle it, we should test the logic for it anyway. We do that here.
 868         let chanmon_cfgs = create_chanmon_cfgs(2);
 869         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 870         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 871         let persister: test_utils::TestPersister;
 872         let new_chain_monitor: test_utils::TestChainMonitor;
 873         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 874         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 875
 876         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 877         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 100_000);
 878
 879         // The simplest way to get a failure after a fulfill is to reload nodes[1] from a state
 880         // pre-fulfill, which we do by serializing it here.
 881         let chan_manager_serialized = nodes[1].node.encode();
 882         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id).encode();
 883
 884         nodes[1].node.claim_funds(payment_preimage);
 885         check_added_monitors!(nodes[1], 1);
 886         expect_payment_claimed!(nodes[1], payment_hash, 100_000);
 887
 888         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 889         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 890         expect_payment_sent_without_paths!(nodes[0], payment_preimage);
 891
 892         // Now reload nodes[1]...
 893         reload_node!(nodes[1], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 894
 895         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 896         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 897
 898         nodes[1].node.fail_htlc_backwards(&payment_hash);
 899         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]);
 900         check_added_monitors!(nodes[1], 1);
 901         let htlc_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 902         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates.update_fail_htlcs[0]);
 903         commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates.commitment_signed, false);
 904         // nodes[0] shouldn't generate any events here, while it just got a payment failure completion
 905         // it had already considered the payment fulfilled, and now they just got free money.
 906         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 907 }
 908
 909 #[test]
 910 fn get_ldk_payment_preimage() {
 911         // Ensure that `ChannelManager::get_payment_preimage` can successfully be used to claim a payment.
 912         let chanmon_cfgs = create_chanmon_cfgs(2);
 913         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 914         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 915         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 916         create_announced_chan_between_nodes(&nodes, 0, 1);
 917
 918         let amt_msat = 60_000;
 919         let expiry_secs = 60 * 60;
 920         let (payment_hash, payment_secret) = nodes[1].node.create_inbound_payment(Some(amt_msat), expiry_secs, None).unwrap();
 921
 922         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
 923                 .with_features(nodes[1].node.invoice_features());
 924         let scorer = test_utils::TestScorer::with_penalty(0);
 925         let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet);
 926         let random_seed_bytes = keys_manager.get_secure_random_bytes();
 927         let route = get_route(
 928                 &nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(),
 929                 Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()),
 930                 amt_msat, TEST_FINAL_CLTV, nodes[0].logger, &scorer, &random_seed_bytes).unwrap();
 931         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 932         check_added_monitors!(nodes[0], 1);
 933
 934         // Make sure to use `get_payment_preimage`
 935         let payment_preimage = nodes[1].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
 936         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 937         assert_eq!(events.len(), 1);
 938         pass_along_path(&nodes[0], &[&nodes[1]], amt_msat, payment_hash, Some(payment_secret), events.pop().unwrap(), true, Some(payment_preimage));
 939         claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, payment_preimage);
 940 }
 941
 942 #[test]
 943 fn sent_probe_is_probe_of_sending_node() {
 944         let chanmon_cfgs = create_chanmon_cfgs(3);
 945         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 946         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
 947         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 948
 949         create_announced_chan_between_nodes(&nodes, 0, 1);
 950         create_announced_chan_between_nodes(&nodes, 1, 2);
 951
 952         // First check we refuse to build a single-hop probe
 953         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[1], 100_000);
 954         assert!(nodes[0].node.send_probe(route.paths[0].clone()).is_err());
 955
 956         // Then build an actual two-hop probing path
 957         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
 958
 959         match nodes[0].node.send_probe(route.paths[0].clone()) {
 960                 Ok((payment_hash, payment_id)) => {
 961                         assert!(nodes[0].node.payment_is_probe(&payment_hash, &payment_id));
 962                         assert!(!nodes[1].node.payment_is_probe(&payment_hash, &payment_id));
 963                         assert!(!nodes[2].node.payment_is_probe(&payment_hash, &payment_id));
 964                 },
 965                 _ => panic!(),
 966         }
 967
 968         get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 969         check_added_monitors!(nodes[0], 1);
 970 }
 971
 972 #[test]
 973 fn successful_probe_yields_event() {
 974         let chanmon_cfgs = create_chanmon_cfgs(3);
 975         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 976         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
 977         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 978
 979         create_announced_chan_between_nodes(&nodes, 0, 1);
 980         create_announced_chan_between_nodes(&nodes, 1, 2);
 981
 982         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
 983
 984         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
 985
 986         // node[0] -- update_add_htlcs -> node[1]
 987         check_added_monitors!(nodes[0], 1);
 988         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 989         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
 990         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
 991         check_added_monitors!(nodes[1], 0);
 992         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
 993         expect_pending_htlcs_forwardable!(nodes[1]);
 994
 995         // node[1] -- update_add_htlcs -> node[2]
 996         check_added_monitors!(nodes[1], 1);
 997         let updates = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 998         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
 999         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &probe_event.msgs[0]);
1000         check_added_monitors!(nodes[2], 0);
1001         commitment_signed_dance!(nodes[2], nodes[1], probe_event.commitment_msg, true, true);
1002
1003         // node[1] <- update_fail_htlcs -- node[2]
1004         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1005         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1006         check_added_monitors!(nodes[1], 0);
1007         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, true);
1008
1009         // node[0] <- update_fail_htlcs -- node[1]
1010         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1011         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1012         check_added_monitors!(nodes[0], 0);
1013         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1014
1015         let mut events = nodes[0].node.get_and_clear_pending_events();
1016         assert_eq!(events.len(), 1);
1017         match events.drain(..).next().unwrap() {
1018                 crate::util::events::Event::ProbeSuccessful { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1019                         assert_eq!(payment_id, ev_pid);
1020                         assert_eq!(payment_hash, ev_ph);
1021                 },
1022                 _ => panic!(),
1023         };
1024 }
1025
1026 #[test]
1027 fn failed_probe_yields_event() {
1028         let chanmon_cfgs = create_chanmon_cfgs(3);
1029         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1030         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1031         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1032
1033         create_announced_chan_between_nodes(&nodes, 0, 1);
1034         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 100000, 90000000);
1035
1036         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1037
1038         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 9_998_000, 42);
1039
1040         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1041
1042         // node[0] -- update_add_htlcs -> node[1]
1043         check_added_monitors!(nodes[0], 1);
1044         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1045         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1046         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1047         check_added_monitors!(nodes[1], 0);
1048         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1049         expect_pending_htlcs_forwardable!(nodes[1]);
1050
1051         // node[0] <- update_fail_htlcs -- node[1]
1052         check_added_monitors!(nodes[1], 1);
1053         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1054         // Skip the PendingHTLCsForwardable event
1055         let _events = nodes[1].node.get_and_clear_pending_events();
1056         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1057         check_added_monitors!(nodes[0], 0);
1058         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1059
1060         let mut events = nodes[0].node.get_and_clear_pending_events();
1061         assert_eq!(events.len(), 1);
1062         match events.drain(..).next().unwrap() {
1063                 crate::util::events::Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1064                         assert_eq!(payment_id, ev_pid);
1065                         assert_eq!(payment_hash, ev_ph);
1066                 },
1067                 _ => panic!(),
1068         };
1069 }
1070
1071 #[test]
1072 fn onchain_failed_probe_yields_event() {
1073         // Tests that an attempt to probe over a channel that is eventaully closed results in a failure
1074         // event.
1075         let chanmon_cfgs = create_chanmon_cfgs(3);
1076         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1077         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1078         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1079
1080         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1081         create_announced_chan_between_nodes(&nodes, 1, 2);
1082
1083         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1084
1085         // Send a dust HTLC, which will be treated as if it timed out once the channel hits the chain.
1086         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 1_000, 42);
1087         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1088
1089         // node[0] -- update_add_htlcs -> node[1]
1090         check_added_monitors!(nodes[0], 1);
1091         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1092         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1093         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1094         check_added_monitors!(nodes[1], 0);
1095         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1096         expect_pending_htlcs_forwardable!(nodes[1]);
1097
1098         check_added_monitors!(nodes[1], 1);
1099         let _ = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1100
1101         // Don't bother forwarding the HTLC onwards and just confirm the force-close transaction on
1102         // Node A, which after 6 confirmations should result in a probe failure event.
1103         let bs_txn = get_local_commitment_txn!(nodes[1], chan_id);
1104         confirm_transaction(&nodes[0], &bs_txn[0]);
1105         check_closed_broadcast!(&nodes[0], true);
1106         check_added_monitors!(nodes[0], 1);
1107
1108         let mut events = nodes[0].node.get_and_clear_pending_events();
1109         assert_eq!(events.len(), 2);
1110         let mut found_probe_failed = false;
1111         for event in events.drain(..) {
1112                 match event {
1113                         Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1114                                 assert_eq!(payment_id, ev_pid);
1115                                 assert_eq!(payment_hash, ev_ph);
1116                                 found_probe_failed = true;
1117                         },
1118                         Event::ChannelClosed { .. } => {},
1119                         _ => panic!(),
1120                 }
1121         }
1122         assert!(found_probe_failed);
1123 }
1124
1125 #[test]
1126 fn claimed_send_payment_idempotent() {
1127         // Tests that `send_payment` (and friends) are (reasonably) idempotent.
1128         let chanmon_cfgs = create_chanmon_cfgs(2);
1129         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1130         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1131         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1132
1133         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1134
1135         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1136         let (first_payment_preimage, _, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1137
1138         macro_rules! check_send_rejected {
1139                 () => {
1140                         // If we try to resend a new payment with a different payment_hash but with the same
1141                         // payment_id, it should be rejected.
1142                         let send_result = nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id);
1143                         match send_result {
1144                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1145                                 _ => panic!("Unexpected send result: {:?}", send_result),
1146                         }
1147
1148                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1149                         // also be rejected.
1150                         let send_result = nodes[0].node.send_spontaneous_payment(&route, None, payment_id);
1151                         match send_result {
1152                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1153                                 _ => panic!("Unexpected send result: {:?}", send_result),
1154                         }
1155                 }
1156         }
1157
1158         check_send_rejected!();
1159
1160         // Claim the payment backwards, but note that the PaymentSent event is still pending and has
1161         // not been seen by the user. At this point, from the user perspective nothing has changed, so
1162         // we must remain just as idempotent as we were before.
1163         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, first_payment_preimage);
1164
1165         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1166                 nodes[0].node.timer_tick_occurred();
1167         }
1168
1169         check_send_rejected!();
1170
1171         // Once the user sees and handles the `PaymentSent` event, we expect them to no longer call
1172         // `send_payment`, and our idempotency guarantees are off - they should have atomically marked
1173         // the payment complete. However, they could have called `send_payment` while the event was
1174         // being processed, leading to a race in our idempotency guarantees. Thus, even immediately
1175         // after the event is handled a duplicate payment should sitll be rejected.
1176         expect_payment_sent!(&nodes[0], first_payment_preimage, Some(0));
1177         check_send_rejected!();
1178
1179         // If relatively little time has passed, a duplicate payment should still fail.
1180         nodes[0].node.timer_tick_occurred();
1181         check_send_rejected!();
1182
1183         // However, after some time has passed (at least more than the one timer tick above), a
1184         // duplicate payment should go through, as ChannelManager should no longer have any remaining
1185         // references to the old payment data.
1186         for _ in 0..IDEMPOTENCY_TIMEOUT_TICKS {
1187                 nodes[0].node.timer_tick_occurred();
1188         }
1189
1190         nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id).unwrap();
1191         check_added_monitors!(nodes[0], 1);
1192         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1193         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1194 }
1195
1196 #[test]
1197 fn abandoned_send_payment_idempotent() {
1198         // Tests that `send_payment` (and friends) allow duplicate PaymentIds immediately after
1199         // abandon_payment.
1200         let chanmon_cfgs = create_chanmon_cfgs(2);
1201         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1202         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1203         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1204
1205         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1206
1207         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1208         let (_, first_payment_hash, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1209
1210         macro_rules! check_send_rejected {
1211                 () => {
1212                         // If we try to resend a new payment with a different payment_hash but with the same
1213                         // payment_id, it should be rejected.
1214                         let send_result = nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id);
1215                         match send_result {
1216                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1217                                 _ => panic!("Unexpected send result: {:?}", send_result),
1218                         }
1219
1220                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1221                         // also be rejected.
1222                         let send_result = nodes[0].node.send_spontaneous_payment(&route, None, payment_id);
1223                         match send_result {
1224                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1225                                 _ => panic!("Unexpected send result: {:?}", send_result),
1226                         }
1227                 }
1228         }
1229
1230         check_send_rejected!();
1231
1232         nodes[1].node.fail_htlc_backwards(&first_payment_hash);
1233         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash: first_payment_hash }]);
1234
1235         pass_failed_payment_back_no_abandon(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash);
1236         check_send_rejected!();
1237
1238         // Until we abandon the payment, no matter how many timer ticks pass, we still cannot reuse the
1239         // PaymentId.
1240         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1241                 nodes[0].node.timer_tick_occurred();
1242         }
1243         check_send_rejected!();
1244
1245         nodes[0].node.abandon_payment(payment_id);
1246         get_event!(nodes[0], Event::PaymentFailed);
1247
1248         // However, we can reuse the PaymentId immediately after we `abandon_payment`.
1249         nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id).unwrap();
1250         check_added_monitors!(nodes[0], 1);
1251         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1252         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1253 }
1254
1255 #[derive(PartialEq)]
1256 enum InterceptTest {
1257         Forward,
1258         Fail,
1259         Timeout,
1260 }
1261
1262 #[test]
1263 fn test_trivial_inflight_htlc_tracking(){
1264         // In this test, we test three scenarios:
1265         // (1) Sending + claiming a payment successfully should return `None` when querying InFlightHtlcs
1266         // (2) Sending a payment without claiming it should return the payment's value (500000) when querying InFlightHtlcs
1267         // (3) After we claim the payment sent in (2), InFlightHtlcs should return `None` for the query.
1268         let chanmon_cfgs = create_chanmon_cfgs(3);
1269         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1270         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1271         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1272
1273         let (_, _, chan_1_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
1274         let (_, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
1275
1276         // Send and claim the payment. Inflight HTLCs should be empty.
1277         send_payment(&nodes[0], &vec!(&nodes[1], &nodes[2])[..], 500000);
1278         {
1279                 let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1280
1281                 let mut node_0_per_peer_lock;
1282                 let mut node_0_peer_state_lock;
1283                 let mut node_1_per_peer_lock;
1284                 let mut node_1_peer_state_lock;
1285                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1286                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1287
1288                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1289                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1290                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1291                         channel_1.get_short_channel_id().unwrap()
1292                 );
1293                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1294                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1295                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1296                         channel_2.get_short_channel_id().unwrap()
1297                 );
1298
1299                 assert_eq!(chan_1_used_liquidity, None);
1300                 assert_eq!(chan_2_used_liquidity, None);
1301         }
1302
1303         // Send the payment, but do not claim it. Our inflight HTLCs should contain the pending payment.
1304         let (payment_preimage, _,  _) = route_payment(&nodes[0], &vec!(&nodes[1], &nodes[2])[..], 500000);
1305         {
1306                 let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1307
1308                 let mut node_0_per_peer_lock;
1309                 let mut node_0_peer_state_lock;
1310                 let mut node_1_per_peer_lock;
1311                 let mut node_1_peer_state_lock;
1312                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1313                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1314
1315                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1316                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1317                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1318                         channel_1.get_short_channel_id().unwrap()
1319                 );
1320                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1321                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1322                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1323                         channel_2.get_short_channel_id().unwrap()
1324                 );
1325
1326                 // First hop accounts for expected 1000 msat fee
1327                 assert_eq!(chan_1_used_liquidity, Some(501000));
1328                 assert_eq!(chan_2_used_liquidity, Some(500000));
1329         }
1330
1331         // Now, let's claim the payment. This should result in the used liquidity to return `None`.
1332         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
1333         {
1334                 let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1335
1336                 let mut node_0_per_peer_lock;
1337                 let mut node_0_peer_state_lock;
1338                 let mut node_1_per_peer_lock;
1339                 let mut node_1_peer_state_lock;
1340                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1341                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1342
1343                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1344                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1345                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1346                         channel_1.get_short_channel_id().unwrap()
1347                 );
1348                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1349                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1350                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1351                         channel_2.get_short_channel_id().unwrap()
1352                 );
1353
1354                 assert_eq!(chan_1_used_liquidity, None);
1355                 assert_eq!(chan_2_used_liquidity, None);
1356         }
1357 }
1358
1359 #[test]
1360 fn test_holding_cell_inflight_htlcs() {
1361         let chanmon_cfgs = create_chanmon_cfgs(2);
1362         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1363         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1364         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1365         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1366
1367         let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
1368         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1369
1370         // Queue up two payments - one will be delivered right away, one immediately goes into the
1371         // holding cell as nodes[0] is AwaitingRAA.
1372         {
1373                 nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
1374                 check_added_monitors!(nodes[0], 1);
1375                 nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
1376                 check_added_monitors!(nodes[0], 0);
1377         }
1378
1379         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1380
1381         {
1382                 let mut node_0_per_peer_lock;
1383                 let mut node_0_peer_state_lock;
1384                 let channel =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, channel_id);
1385
1386                 let used_liquidity = inflight_htlcs.used_liquidity_msat(
1387                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1388                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1389                         channel.get_short_channel_id().unwrap()
1390                 );
1391
1392                 assert_eq!(used_liquidity, Some(2000000));
1393         }
1394
1395         // Clear pending events so test doesn't throw a "Had excess message on node..." error
1396         nodes[0].node.get_and_clear_pending_msg_events();
1397 }
1398
1399 #[test]
1400 fn intercepted_payment() {
1401         // Test that detecting an intercept scid on payment forward will signal LDK to generate an
1402         // intercept event, which the LSP can then use to either (a) open a JIT channel to forward the
1403         // payment or (b) fail the payment.
1404         do_test_intercepted_payment(InterceptTest::Forward);
1405         do_test_intercepted_payment(InterceptTest::Fail);
1406         // Make sure that intercepted payments will be automatically failed back if too many blocks pass.
1407         do_test_intercepted_payment(InterceptTest::Timeout);
1408 }
1409
1410 fn do_test_intercepted_payment(test: InterceptTest) {
1411         let chanmon_cfgs = create_chanmon_cfgs(3);
1412         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1413
1414         let mut zero_conf_chan_config = test_default_channel_config();
1415         zero_conf_chan_config.manually_accept_inbound_channels = true;
1416         let mut intercept_forwards_config = test_default_channel_config();
1417         intercept_forwards_config.accept_intercept_htlcs = true;
1418         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(zero_conf_chan_config)]);
1419
1420         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1421         let scorer = test_utils::TestScorer::with_penalty(0);
1422         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
1423
1424         let _ = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1425
1426         let amt_msat = 100_000;
1427         let intercept_scid = nodes[1].node.get_intercept_scid();
1428         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1429                 .with_route_hints(vec![
1430                         RouteHint(vec![RouteHintHop {
1431                                 src_node_id: nodes[1].node.get_our_node_id(),
1432                                 short_channel_id: intercept_scid,
1433                                 fees: RoutingFees {
1434                                         base_msat: 1000,
1435                                         proportional_millionths: 0,
1436                                 },
1437                                 cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
1438                                 htlc_minimum_msat: None,
1439                                 htlc_maximum_msat: None,
1440                         }])
1441                 ])
1442                 .with_features(nodes[2].node.invoice_features());
1443         let route_params = RouteParameters {
1444                 payment_params,
1445                 final_value_msat: amt_msat,
1446                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
1447         };
1448         let route = get_route(
1449                 &nodes[0].node.get_our_node_id(), &route_params.payment_params,
1450                 &nodes[0].network_graph.read_only(), None, route_params.final_value_msat,
1451                 route_params.final_cltv_expiry_delta, nodes[0].logger, &scorer, &random_seed_bytes
1452         ).unwrap();
1453
1454         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
1455         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
1456         let payment_event = {
1457                 {
1458                         let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
1459                         assert_eq!(added_monitors.len(), 1);
1460                         added_monitors.clear();
1461                 }
1462                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1463                 assert_eq!(events.len(), 1);
1464                 SendEvent::from_event(events.remove(0))
1465         };
1466         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1467         commitment_signed_dance!(nodes[1], nodes[0], &payment_event.commitment_msg, false, true);
1468
1469         // Check that we generate the PaymentIntercepted event when an intercept forward is detected.
1470         let events = nodes[1].node.get_and_clear_pending_events();
1471         assert_eq!(events.len(), 1);
1472         let (intercept_id, expected_outbound_amount_msat) = match events[0] {
1473                 crate::util::events::Event::HTLCIntercepted {
1474                         intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, inbound_amount_msat, requested_next_hop_scid: short_channel_id
1475                 } => {
1476                         assert_eq!(pmt_hash, payment_hash);
1477                         assert_eq!(inbound_amount_msat, route.get_total_amount() + route.get_total_fees());
1478                         assert_eq!(short_channel_id, intercept_scid);
1479                         (intercept_id, expected_outbound_amount_msat)
1480                 },
1481                 _ => panic!()
1482         };
1483
1484         // Check for unknown channel id error.
1485         let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &[42; 32], nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1486         assert_eq!(unknown_chan_id_err , APIError::ChannelUnavailable  { err: format!("Channel with id {} not found for the passed counterparty node_id {}", log_bytes!([42; 32]), nodes[2].node.get_our_node_id()) });
1487
1488         if test == InterceptTest::Fail {
1489                 // Ensure we can fail the intercepted payment back.
1490                 nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap();
1491                 expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::UnknownNextHop { requested_forward_scid: intercept_scid }]);
1492                 nodes[1].node.process_pending_htlc_forwards();
1493                 let update_fail = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1494                 check_added_monitors!(&nodes[1], 1);
1495                 assert!(update_fail.update_fail_htlcs.len() == 1);
1496                 let fail_msg = update_fail.update_fail_htlcs[0].clone();
1497                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1498                 commitment_signed_dance!(nodes[0], nodes[1], update_fail.commitment_signed, false);
1499
1500                 // Ensure the payment fails with the expected error.
1501                 let fail_conditions = PaymentFailedConditions::new()
1502                         .blamed_scid(intercept_scid)
1503                         .blamed_chan_closed(true)
1504                         .expected_htlc_error_data(0x4000 | 10, &[]);
1505                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
1506         } else if test == InterceptTest::Forward {
1507                 // Check that we'll fail as expected when sending to a channel that isn't in `ChannelReady` yet.
1508                 let temp_chan_id = nodes[1].node.create_channel(nodes[2].node.get_our_node_id(), 100_000, 0, 42, None).unwrap();
1509                 let unusable_chan_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &temp_chan_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1510                 assert_eq!(unusable_chan_err , APIError::ChannelUnavailable { err: format!("Channel with id {} not fully established", log_bytes!(temp_chan_id)) });
1511                 assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
1512
1513                 // Open the just-in-time channel so the payment can then be forwarded.
1514                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1515
1516                 // Finally, forward the intercepted payment through and claim it.
1517                 nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap();
1518                 expect_pending_htlcs_forwardable!(nodes[1]);
1519
1520                 let payment_event = {
1521                         {
1522                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
1523                                 assert_eq!(added_monitors.len(), 1);
1524                                 added_monitors.clear();
1525                         }
1526                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
1527                         assert_eq!(events.len(), 1);
1528                         SendEvent::from_event(events.remove(0))
1529                 };
1530                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1531                 commitment_signed_dance!(nodes[2], nodes[1], &payment_event.commitment_msg, false, true);
1532                 expect_pending_htlcs_forwardable!(nodes[2]);
1533
1534                 let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1535                 expect_payment_claimable!(&nodes[2], payment_hash, payment_secret, amt_msat, Some(payment_preimage), nodes[2].node.get_our_node_id());
1536                 do_claim_payment_along_route(&nodes[0], &vec!(&vec!(&nodes[1], &nodes[2])[..]), false, payment_preimage);
1537                 let events = nodes[0].node.get_and_clear_pending_events();
1538                 assert_eq!(events.len(), 2);
1539                 match events[0] {
1540                         Event::PaymentSent { payment_preimage: ref ev_preimage, payment_hash: ref ev_hash, ref fee_paid_msat, .. } => {
1541                                 assert_eq!(payment_preimage, *ev_preimage);
1542                                 assert_eq!(payment_hash, *ev_hash);
1543                                 assert_eq!(fee_paid_msat, &Some(1000));
1544                         },
1545                         _ => panic!("Unexpected event")
1546                 }
1547                 match events[1] {
1548                         Event::PaymentPathSuccessful { payment_hash: hash, .. } => {
1549                                 assert_eq!(hash, Some(payment_hash));
1550                         },
1551                         _ => panic!("Unexpected event")
1552                 }
1553         } else if test == InterceptTest::Timeout {
1554                 let mut block = Block {
1555                         header: BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 },
1556                         txdata: vec![],
1557                 };
1558                 connect_block(&nodes[0], &block);
1559                 connect_block(&nodes[1], &block);
1560                 for _ in 0..TEST_FINAL_CLTV {
1561                         block.header.prev_blockhash = block.block_hash();
1562                         connect_block(&nodes[0], &block);
1563                         connect_block(&nodes[1], &block);
1564                 }
1565                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::InvalidForward { requested_forward_scid: intercept_scid }]);
1566                 check_added_monitors!(nodes[1], 1);
1567                 let htlc_timeout_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1568                 assert!(htlc_timeout_updates.update_add_htlcs.is_empty());
1569                 assert_eq!(htlc_timeout_updates.update_fail_htlcs.len(), 1);
1570                 assert!(htlc_timeout_updates.update_fail_malformed_htlcs.is_empty());
1571                 assert!(htlc_timeout_updates.update_fee.is_none());
1572
1573                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_timeout_updates.update_fail_htlcs[0]);
1574                 commitment_signed_dance!(nodes[0], nodes[1], htlc_timeout_updates.commitment_signed, false);
1575                 expect_payment_failed!(nodes[0], payment_hash, false, 0x2000 | 2, []);
1576
1577                 // Check for unknown intercept id error.
1578                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1579                 let unknown_intercept_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1580                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1581                 let unknown_intercept_id_err = nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap_err();
1582                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1583         }
1584 }
1585
1586 #[derive(PartialEq)]
1587 enum AutoRetry {
1588         Success,
1589         Spontaneous,
1590         FailAttempts,
1591         FailTimeout,
1592         FailOnRestart,
1593 }
1594
1595 #[test]
1596 fn automatic_retries() {
1597         do_automatic_retries(AutoRetry::Success);
1598         do_automatic_retries(AutoRetry::Spontaneous);
1599         do_automatic_retries(AutoRetry::FailAttempts);
1600         do_automatic_retries(AutoRetry::FailTimeout);
1601         do_automatic_retries(AutoRetry::FailOnRestart);
1602 }
1603 fn do_automatic_retries(test: AutoRetry) {
1604         // Test basic automatic payment retries in ChannelManager. See individual `test` variant comments
1605         // below.
1606         let chanmon_cfgs = create_chanmon_cfgs(3);
1607         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1608         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1609
1610         let persister;
1611         let new_chain_monitor;
1612         let node_0_deserialized;
1613
1614         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1615         let channel_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1616         let channel_id_2 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1617
1618         // Marshall data to send the payment
1619         #[cfg(feature = "std")]
1620         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1621         #[cfg(not(feature = "std"))]
1622         let payment_expiry_secs = 60 * 60;
1623         let amt_msat = 1000;
1624         let mut invoice_features = InvoiceFeatures::empty();
1625         invoice_features.set_variable_length_onion_required();
1626         invoice_features.set_payment_secret_required();
1627         invoice_features.set_basic_mpp_optional();
1628         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1629                 .with_expiry_time(payment_expiry_secs as u64)
1630                 .with_features(invoice_features);
1631         let route_params = RouteParameters {
1632                 payment_params,
1633                 final_value_msat: amt_msat,
1634                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
1635         };
1636         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
1637
1638         macro_rules! pass_failed_attempt_with_retry_along_path {
1639                 ($failing_channel_id: expr, $expect_pending_htlcs_forwardable: expr) => {
1640                         // Send a payment attempt that fails due to lack of liquidity on the second hop
1641                         check_added_monitors!(nodes[0], 1);
1642                         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1643                         let mut update_add = update_0.update_add_htlcs[0].clone();
1644                         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
1645                         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
1646                         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
1647                         nodes[1].node.process_pending_htlc_forwards();
1648                         expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1],
1649                                 vec![HTLCDestination::NextHopChannel {
1650                                         node_id: Some(nodes[2].node.get_our_node_id()),
1651                                         channel_id: $failing_channel_id,
1652                                 }]);
1653                         nodes[1].node.process_pending_htlc_forwards();
1654                         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1655                         check_added_monitors!(&nodes[1], 1);
1656                         assert!(update_1.update_fail_htlcs.len() == 1);
1657                         let fail_msg = update_1.update_fail_htlcs[0].clone();
1658
1659                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1660                         commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
1661
1662                         // Ensure the attempt fails and a new PendingHTLCsForwardable event is generated for the retry
1663                         let mut events = nodes[0].node.get_and_clear_pending_events();
1664                         match events[0] {
1665                                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
1666                                         assert_eq!(payment_hash, ev_payment_hash);
1667                                         assert_eq!(payment_failed_permanently, false);
1668                                 },
1669                                 _ => panic!("Unexpected event"),
1670                         }
1671                         if $expect_pending_htlcs_forwardable {
1672                                 assert_eq!(events.len(), 2);
1673                                 match events[1] {
1674                                         Event::PendingHTLCsForwardable { .. } => {},
1675                                         _ => panic!("Unexpected event"),
1676                                 }
1677                         } else { assert_eq!(events.len(), 1) }
1678                 }
1679         }
1680
1681         if test == AutoRetry::Success {
1682                 // Test that we can succeed on the first retry.
1683                 nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1684                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1685
1686                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
1687                 // attempt, since the initial second hop channel will be excluded from pathfinding
1688                 create_announced_chan_between_nodes(&nodes, 1, 2);
1689
1690                 // We retry payments in `process_pending_htlc_forwards`
1691                 nodes[0].node.process_pending_htlc_forwards();
1692                 check_added_monitors!(nodes[0], 1);
1693                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1694                 assert_eq!(msg_events.len(), 1);
1695                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, Some(payment_secret), msg_events.pop().unwrap(), true, None);
1696                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
1697         } else if test == AutoRetry::Spontaneous {
1698                 nodes[0].node.send_spontaneous_payment_with_retry(Some(payment_preimage), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1699                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1700
1701                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
1702                 // attempt, since the initial second hop channel will be excluded from pathfinding
1703                 create_announced_chan_between_nodes(&nodes, 1, 2);
1704
1705                 // We retry payments in `process_pending_htlc_forwards`
1706                 nodes[0].node.process_pending_htlc_forwards();
1707                 check_added_monitors!(nodes[0], 1);
1708                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1709                 assert_eq!(msg_events.len(), 1);
1710                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, None, msg_events.pop().unwrap(), true, Some(payment_preimage));
1711                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
1712         } else if test == AutoRetry::FailAttempts {
1713                 // Ensure ChannelManager will not retry a payment if it has run out of payment attempts.
1714                 nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1715                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1716
1717                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
1718                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
1719                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1720
1721                 // We retry payments in `process_pending_htlc_forwards`
1722                 nodes[0].node.process_pending_htlc_forwards();
1723                 pass_failed_attempt_with_retry_along_path!(channel_id_3, false);
1724
1725                 // Ensure we won't retry a second time.
1726                 nodes[0].node.process_pending_htlc_forwards();
1727                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1728                 assert_eq!(msg_events.len(), 0);
1729
1730                 nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
1731                 let events = nodes[0].node.get_and_clear_pending_events();
1732                 assert_eq!(events.len(), 1);
1733                 match events[0] {
1734                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
1735                                 assert_eq!(payment_hash, *ev_payment_hash);
1736                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1737                         },
1738                         _ => panic!("Unexpected event"),
1739                 }
1740         } else if test == AutoRetry::FailTimeout {
1741                 #[cfg(not(feature = "no-std"))] {
1742                         // Ensure ChannelManager will not retry a payment if it times out due to Retry::Timeout.
1743                         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
1744                         pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1745
1746                         // Advance the time so the second attempt fails due to timeout.
1747                         SinceEpoch::advance(Duration::from_secs(61));
1748
1749                         // Make sure we don't retry again.
1750                         nodes[0].node.process_pending_htlc_forwards();
1751                         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1752                         assert_eq!(msg_events.len(), 0);
1753
1754                         nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
1755                         let mut events = nodes[0].node.get_and_clear_pending_events();
1756                         assert_eq!(events.len(), 1);
1757                         match events[0] {
1758                                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
1759                                         assert_eq!(payment_hash, *ev_payment_hash);
1760                                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1761                                 },
1762                                 _ => panic!("Unexpected event"),
1763                         }
1764                 }
1765         } else if test == AutoRetry::FailOnRestart {
1766                 // Ensure ChannelManager will not retry a payment after restart, even if there were retry
1767                 // attempts remaining prior to restart.
1768                 nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
1769                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1770
1771                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
1772                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
1773                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1774
1775                 // Ensure the first retry attempt fails, with 1 retry attempt remaining
1776                 nodes[0].node.process_pending_htlc_forwards();
1777                 pass_failed_attempt_with_retry_along_path!(channel_id_3, true);
1778
1779                 // Restart the node and ensure that ChannelManager does not use its remaining retry attempt
1780                 let node_encoded = nodes[0].node.encode();
1781                 let chan_1_monitor_serialized = get_monitor!(nodes[0], channel_id_1).encode();
1782                 reload_node!(nodes[0], node_encoded, &[&chan_1_monitor_serialized], persister, new_chain_monitor, node_0_deserialized);
1783
1784                 // Make sure we don't retry again.
1785                 nodes[0].node.process_pending_htlc_forwards();
1786                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1787                 assert_eq!(msg_events.len(), 0);
1788
1789                 nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
1790                 let mut events = nodes[0].node.get_and_clear_pending_events();
1791                 assert_eq!(events.len(), 1);
1792                 match events[0] {
1793                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
1794                                 assert_eq!(payment_hash, *ev_payment_hash);
1795                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1796                         },
1797                         _ => panic!("Unexpected event"),
1798                 }
1799         }
1800 }
1801
1802 #[test]
1803 fn auto_retry_partial_failure() {
1804         // Test that we'll retry appropriately on send partial failure and retry partial failure.
1805         let chanmon_cfgs = create_chanmon_cfgs(2);
1806         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1807         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1808         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1809
1810         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1811         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1812         let chan_3_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1813
1814         // Marshall data to send the payment
1815         let amt_msat = 20_000;
1816         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
1817         #[cfg(feature = "std")]
1818         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1819         #[cfg(not(feature = "std"))]
1820         let payment_expiry_secs = 60 * 60;
1821         let mut invoice_features = InvoiceFeatures::empty();
1822         invoice_features.set_variable_length_onion_required();
1823         invoice_features.set_payment_secret_required();
1824         invoice_features.set_basic_mpp_optional();
1825         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
1826                 .with_expiry_time(payment_expiry_secs as u64)
1827                 .with_features(invoice_features);
1828         let route_params = RouteParameters {
1829                 payment_params,
1830                 final_value_msat: amt_msat,
1831                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
1832         };
1833
1834         // Ensure the first monitor update (for the initial send path1 over chan_1) succeeds, but the
1835         // second (for the initial send path2 over chan_2) fails.
1836         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1837         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
1838         // Ensure third monitor update (for the retry1's path1 over chan_1) succeeds, but the fourth (for
1839         // the retry1's path2 over chan_3) fails, and monitor updates succeed after that.
1840         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1841         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
1842         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1843
1844         // Configure the initial send, retry1 and retry2's paths.
1845         let send_route = Route {
1846                 paths: vec![
1847                         vec![RouteHop {
1848                                 pubkey: nodes[1].node.get_our_node_id(),
1849                                 node_features: nodes[1].node.node_features(),
1850                                 short_channel_id: chan_1_id,
1851                                 channel_features: nodes[1].node.channel_features(),
1852                                 fee_msat: amt_msat / 2,
1853                                 cltv_expiry_delta: 100,
1854                         }],
1855                         vec![RouteHop {
1856                                 pubkey: nodes[1].node.get_our_node_id(),
1857                                 node_features: nodes[1].node.node_features(),
1858                                 short_channel_id: chan_2_id,
1859                                 channel_features: nodes[1].node.channel_features(),
1860                                 fee_msat: amt_msat / 2,
1861                                 cltv_expiry_delta: 100,
1862                         }],
1863                 ],
1864                 payment_params: Some(route_params.payment_params.clone()),
1865         };
1866         let retry_1_route = Route {
1867                 paths: vec![
1868                         vec![RouteHop {
1869                                 pubkey: nodes[1].node.get_our_node_id(),
1870                                 node_features: nodes[1].node.node_features(),
1871                                 short_channel_id: chan_1_id,
1872                                 channel_features: nodes[1].node.channel_features(),
1873                                 fee_msat: amt_msat / 4,
1874                                 cltv_expiry_delta: 100,
1875                         }],
1876                         vec![RouteHop {
1877                                 pubkey: nodes[1].node.get_our_node_id(),
1878                                 node_features: nodes[1].node.node_features(),
1879                                 short_channel_id: chan_3_id,
1880                                 channel_features: nodes[1].node.channel_features(),
1881                                 fee_msat: amt_msat / 4,
1882                                 cltv_expiry_delta: 100,
1883                         }],
1884                 ],
1885                 payment_params: Some(route_params.payment_params.clone()),
1886         };
1887         let retry_2_route = Route {
1888                 paths: vec![
1889                         vec![RouteHop {
1890                                 pubkey: nodes[1].node.get_our_node_id(),
1891                                 node_features: nodes[1].node.node_features(),
1892                                 short_channel_id: chan_1_id,
1893                                 channel_features: nodes[1].node.channel_features(),
1894                                 fee_msat: amt_msat / 4,
1895                                 cltv_expiry_delta: 100,
1896                         }],
1897                 ],
1898                 payment_params: Some(route_params.payment_params.clone()),
1899         };
1900         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
1901         nodes[0].router.expect_find_route(RouteParameters {
1902                         payment_params: route_params.payment_params.clone(),
1903                         final_value_msat: amt_msat / 2, final_cltv_expiry_delta: TEST_FINAL_CLTV
1904                 }, Ok(retry_1_route));
1905         nodes[0].router.expect_find_route(RouteParameters {
1906                         payment_params: route_params.payment_params.clone(),
1907                         final_value_msat: amt_msat / 4, final_cltv_expiry_delta: TEST_FINAL_CLTV
1908                 }, Ok(retry_2_route));
1909
1910         // Send a payment that will partially fail on send, then partially fail on retry, then succeed.
1911         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap();
1912         let closed_chan_events = nodes[0].node.get_and_clear_pending_events();
1913         assert_eq!(closed_chan_events.len(), 2);
1914         match closed_chan_events[0] {
1915                 Event::ChannelClosed { .. } => {},
1916                 _ => panic!("Unexpected event"),
1917         }
1918         match closed_chan_events[1] {
1919                 Event::ChannelClosed { .. } => {},
1920                 _ => panic!("Unexpected event"),
1921         }
1922
1923         // Pass the first part of the payment along the path.
1924         check_added_monitors!(nodes[0], 5); // three outbound channel updates succeeded, two permanently failed
1925         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1926
1927         // First message is the first update_add, remaining messages are broadcasting channel updates and
1928         // errors for the permfailed channels
1929         assert_eq!(msg_events.len(), 5);
1930         let mut payment_event = SendEvent::from_event(msg_events.remove(0));
1931
1932         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1933         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1934         check_added_monitors!(nodes[1], 1);
1935         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1936
1937         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
1938         check_added_monitors!(nodes[0], 1);
1939         let as_second_htlc_updates = SendEvent::from_node(&nodes[0]);
1940
1941         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
1942         check_added_monitors!(nodes[0], 1);
1943         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1944
1945         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
1946         check_added_monitors!(nodes[1], 1);
1947
1948         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[0]);
1949         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[1]);
1950         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.commitment_msg);
1951         check_added_monitors!(nodes[1], 1);
1952         let (bs_second_raa, bs_second_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1953
1954         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
1955         check_added_monitors!(nodes[0], 1);
1956
1957         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_cs);
1958         check_added_monitors!(nodes[0], 1);
1959         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1960
1961         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
1962         check_added_monitors!(nodes[1], 1);
1963
1964         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
1965         nodes[1].node.process_pending_htlc_forwards();
1966         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, amt_msat);
1967         nodes[1].node.claim_funds(payment_preimage);
1968         expect_payment_claimed!(nodes[1], payment_hash, amt_msat);
1969         let bs_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1970         assert_eq!(bs_claim_update.update_fulfill_htlcs.len(), 1);
1971
1972         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_claim_update.update_fulfill_htlcs[0]);
1973         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_claim_update.commitment_signed);
1974         check_added_monitors!(nodes[0], 1);
1975         let (as_third_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1976
1977         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
1978         check_added_monitors!(nodes[1], 4);
1979         let bs_second_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1980
1981         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
1982         check_added_monitors!(nodes[1], 1);
1983         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1984
1985         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
1986         check_added_monitors!(nodes[0], 1);
1987
1988         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[0]);
1989         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[1]);
1990         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.commitment_signed);
1991         check_added_monitors!(nodes[0], 1);
1992         let (as_fourth_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1993
1994         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_fourth_raa);
1995         check_added_monitors!(nodes[1], 1);
1996
1997         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
1998         check_added_monitors!(nodes[1], 1);
1999         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2000
2001         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2002         check_added_monitors!(nodes[0], 1);
2003         expect_payment_sent!(nodes[0], payment_preimage);
2004 }
2005
2006 #[test]
2007 fn auto_retry_zero_attempts_send_error() {
2008         let chanmon_cfgs = create_chanmon_cfgs(2);
2009         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2010         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2011         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2012
2013         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2014         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2015
2016         // Marshall data to send the payment
2017         let amt_msat = 20_000;
2018         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2019         #[cfg(feature = "std")]
2020         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2021         #[cfg(not(feature = "std"))]
2022         let payment_expiry_secs = 60 * 60;
2023         let mut invoice_features = InvoiceFeatures::empty();
2024         invoice_features.set_variable_length_onion_required();
2025         invoice_features.set_payment_secret_required();
2026         invoice_features.set_basic_mpp_optional();
2027         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2028                 .with_expiry_time(payment_expiry_secs as u64)
2029                 .with_features(invoice_features);
2030         let route_params = RouteParameters {
2031                 payment_params,
2032                 final_value_msat: amt_msat,
2033                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2034         };
2035
2036         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2037         let err = nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap_err();
2038         if let PaymentSendFailure::AllFailedResendSafe(_) = err {
2039         } else { panic!("Unexpected error"); }
2040         assert_eq!(nodes[0].node.get_and_clear_pending_msg_events().len(), 2); // channel close messages
2041         assert_eq!(nodes[0].node.get_and_clear_pending_events().len(), 1); // channel close event
2042         check_added_monitors!(nodes[0], 2);
2043 }
2044
2045 #[test]
2046 fn fails_paying_after_rejected_by_payee() {
2047         let chanmon_cfgs = create_chanmon_cfgs(2);
2048         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2049         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2050         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2051
2052         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2053
2054         // Marshall data to send the payment
2055         let amt_msat = 20_000;
2056         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2057         #[cfg(feature = "std")]
2058         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2059         #[cfg(not(feature = "std"))]
2060         let payment_expiry_secs = 60 * 60;
2061         let mut invoice_features = InvoiceFeatures::empty();
2062         invoice_features.set_variable_length_onion_required();
2063         invoice_features.set_payment_secret_required();
2064         invoice_features.set_basic_mpp_optional();
2065         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2066                 .with_expiry_time(payment_expiry_secs as u64)
2067                 .with_features(invoice_features);
2068         let route_params = RouteParameters {
2069                 payment_params,
2070                 final_value_msat: amt_msat,
2071                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2072         };
2073
2074         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2075         check_added_monitors!(nodes[0], 1);
2076         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
2077         assert_eq!(events.len(), 1);
2078         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
2079         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2080         check_added_monitors!(nodes[1], 0);
2081         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
2082         expect_pending_htlcs_forwardable!(nodes[1]);
2083         expect_payment_claimable!(&nodes[1], payment_hash, payment_secret, amt_msat);
2084
2085         nodes[1].node.fail_htlc_backwards(&payment_hash);
2086         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash }]);
2087         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, payment_hash);
2088 }
2089
2090 #[test]
2091 fn retry_multi_path_single_failed_payment() {
2092         // Tests that we can/will retry after a single path of an MPP payment failed immediately
2093         let chanmon_cfgs = create_chanmon_cfgs(2);
2094         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2095         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2096         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2097
2098         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2099         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2100
2101         let amt_msat = 100_010_000;
2102
2103         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2104         #[cfg(feature = "std")]
2105         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2106         #[cfg(not(feature = "std"))]
2107         let payment_expiry_secs = 60 * 60;
2108         let mut invoice_features = InvoiceFeatures::empty();
2109         invoice_features.set_variable_length_onion_required();
2110         invoice_features.set_payment_secret_required();
2111         invoice_features.set_basic_mpp_optional();
2112         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2113                 .with_expiry_time(payment_expiry_secs as u64)
2114                 .with_features(invoice_features);
2115         let route_params = RouteParameters {
2116                 payment_params: payment_params.clone(),
2117                 final_value_msat: amt_msat,
2118                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2119         };
2120
2121         let chans = nodes[0].node.list_usable_channels();
2122         let mut route = Route {
2123                 paths: vec![
2124                         vec![RouteHop {
2125                                 pubkey: nodes[1].node.get_our_node_id(),
2126                                 node_features: nodes[1].node.node_features(),
2127                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2128                                 channel_features: nodes[1].node.channel_features(),
2129                                 fee_msat: 10_000,
2130                                 cltv_expiry_delta: 100,
2131                         }],
2132                         vec![RouteHop {
2133                                 pubkey: nodes[1].node.get_our_node_id(),
2134                                 node_features: nodes[1].node.node_features(),
2135                                 short_channel_id: chans[1].short_channel_id.unwrap(),
2136                                 channel_features: nodes[1].node.channel_features(),
2137                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2138                                 cltv_expiry_delta: 100,
2139                         }],
2140                 ],
2141                 payment_params: Some(payment_params),
2142         };
2143         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2144         // On retry, split the payment across both channels.
2145         route.paths[0][0].fee_msat = 50_000_001;
2146         route.paths[1][0].fee_msat = 50_000_000;
2147         nodes[0].router.expect_find_route(RouteParameters {
2148                         payment_params: route.payment_params.clone().unwrap(),
2149                         // Note that the second request here requests the amount we originally failed to send,
2150                         // not the amount remaining on the full payment, which should be changed.
2151                         final_value_msat: 100_000_001, final_cltv_expiry_delta: TEST_FINAL_CLTV
2152                 }, Ok(route.clone()));
2153
2154         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2155         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2156         assert_eq!(htlc_msgs.len(), 2);
2157         check_added_monitors!(nodes[0], 2);
2158 }
2159
2160 #[test]
2161 fn immediate_retry_on_failure() {
2162         // Tests that we can/will retry immediately after a failure
2163         let chanmon_cfgs = create_chanmon_cfgs(2);
2164         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2165         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2166         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2167
2168         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2169         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2170
2171         let amt_msat = 100_000_001;
2172         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2173         #[cfg(feature = "std")]
2174         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2175         #[cfg(not(feature = "std"))]
2176         let payment_expiry_secs = 60 * 60;
2177         let mut invoice_features = InvoiceFeatures::empty();
2178         invoice_features.set_variable_length_onion_required();
2179         invoice_features.set_payment_secret_required();
2180         invoice_features.set_basic_mpp_optional();
2181         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2182                 .with_expiry_time(payment_expiry_secs as u64)
2183                 .with_features(invoice_features);
2184         let route_params = RouteParameters {
2185                 payment_params,
2186                 final_value_msat: amt_msat,
2187                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2188         };
2189
2190         let chans = nodes[0].node.list_usable_channels();
2191         let mut route = Route {
2192                 paths: vec![
2193                         vec![RouteHop {
2194                                 pubkey: nodes[1].node.get_our_node_id(),
2195                                 node_features: nodes[1].node.node_features(),
2196                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2197                                 channel_features: nodes[1].node.channel_features(),
2198                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2199                                 cltv_expiry_delta: 100,
2200                         }],
2201                 ],
2202                 payment_params: Some(PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)),
2203         };
2204         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2205         // On retry, split the payment across both channels.
2206         route.paths.push(route.paths[0].clone());
2207         route.paths[0][0].short_channel_id = chans[1].short_channel_id.unwrap();
2208         route.paths[0][0].fee_msat = 50_000_000;
2209         route.paths[1][0].fee_msat = 50_000_001;
2210         nodes[0].router.expect_find_route(RouteParameters {
2211                         payment_params: route_params.payment_params.clone(),
2212                         final_value_msat: amt_msat, final_cltv_expiry_delta: TEST_FINAL_CLTV
2213                 }, Ok(route.clone()));
2214
2215         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2216         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2217         assert_eq!(htlc_msgs.len(), 2);
2218         check_added_monitors!(nodes[0], 2);
2219 }
2220
2221 #[test]
2222 fn no_extra_retries_on_back_to_back_fail() {
2223         // In a previous release, we had a race where we may exceed the payment retry count if we
2224         // get two failures in a row with the second having `all_paths_failed` set.
2225         // Generally, when we give up trying to retry a payment, we don't know for sure what the
2226         // current state of the ChannelManager event queue is. Specifically, we cannot be sure that
2227         // there are not multiple additional `PaymentPathFailed` or even `PaymentSent` events
2228         // pending which we will see later. Thus, when we previously removed the retry tracking map
2229         // entry after a `all_paths_failed` `PaymentPathFailed` event, we may have dropped the
2230         // retry entry even though more events for the same payment were still pending. This led to
2231         // us retrying a payment again even though we'd already given up on it.
2232         //
2233         // We now have a separate event - `PaymentFailed` which indicates no HTLCs remain and which
2234         // is used to remove the payment retry counter entries instead. This tests for the specific
2235         // excess-retry case while also testing `PaymentFailed` generation.
2236
2237         let chanmon_cfgs = create_chanmon_cfgs(3);
2238         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2239         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2240         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2241
2242         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2243         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2244
2245         let amt_msat = 200_000_000;
2246         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2247         #[cfg(feature = "std")]
2248         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2249         #[cfg(not(feature = "std"))]
2250         let payment_expiry_secs = 60 * 60;
2251         let mut invoice_features = InvoiceFeatures::empty();
2252         invoice_features.set_variable_length_onion_required();
2253         invoice_features.set_payment_secret_required();
2254         invoice_features.set_basic_mpp_optional();
2255         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2256                 .with_expiry_time(payment_expiry_secs as u64)
2257                 .with_features(invoice_features);
2258         let route_params = RouteParameters {
2259                 payment_params,
2260                 final_value_msat: amt_msat,
2261                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2262         };
2263
2264         let mut route = Route {
2265                 paths: vec![
2266                         vec![RouteHop {
2267                                 pubkey: nodes[1].node.get_our_node_id(),
2268                                 node_features: nodes[1].node.node_features(),
2269                                 short_channel_id: chan_1_scid,
2270                                 channel_features: nodes[1].node.channel_features(),
2271                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2272                                 cltv_expiry_delta: 100,
2273                         }, RouteHop {
2274                                 pubkey: nodes[2].node.get_our_node_id(),
2275                                 node_features: nodes[2].node.node_features(),
2276                                 short_channel_id: chan_2_scid,
2277                                 channel_features: nodes[2].node.channel_features(),
2278                                 fee_msat: 100_000_000,
2279                                 cltv_expiry_delta: 100,
2280                         }],
2281                         vec![RouteHop {
2282                                 pubkey: nodes[1].node.get_our_node_id(),
2283                                 node_features: nodes[1].node.node_features(),
2284                                 short_channel_id: chan_1_scid,
2285                                 channel_features: nodes[1].node.channel_features(),
2286                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2287                                 cltv_expiry_delta: 100,
2288                         }, RouteHop {
2289                                 pubkey: nodes[2].node.get_our_node_id(),
2290                                 node_features: nodes[2].node.node_features(),
2291                                 short_channel_id: chan_2_scid,
2292                                 channel_features: nodes[2].node.channel_features(),
2293                                 fee_msat: 100_000_000,
2294                                 cltv_expiry_delta: 100,
2295                         }]
2296                 ],
2297                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2298         };
2299         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2300         let mut second_payment_params = route_params.payment_params.clone();
2301         second_payment_params.previously_failed_channels = vec![chan_2_scid, chan_2_scid];
2302         // On retry, we'll only return one path
2303         route.paths.remove(1);
2304         route.paths[0][1].fee_msat = amt_msat;
2305         nodes[0].router.expect_find_route(RouteParameters {
2306                         payment_params: second_payment_params,
2307                         final_value_msat: amt_msat, final_cltv_expiry_delta: TEST_FINAL_CLTV,
2308                 }, Ok(route.clone()));
2309
2310         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2311         let htlc_updates = SendEvent::from_node(&nodes[0]);
2312         check_added_monitors!(nodes[0], 1);
2313         assert_eq!(htlc_updates.msgs.len(), 1);
2314
2315         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2316         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2317         check_added_monitors!(nodes[1], 1);
2318         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2319
2320         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2321         check_added_monitors!(nodes[0], 1);
2322         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2323
2324         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2325         check_added_monitors!(nodes[0], 1);
2326         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2327
2328         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2329         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2330         check_added_monitors!(nodes[1], 1);
2331         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2332
2333         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2334         check_added_monitors!(nodes[1], 1);
2335         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2336
2337         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2338         check_added_monitors!(nodes[0], 1);
2339
2340         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2341         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2342         check_added_monitors!(nodes[0], 1);
2343         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2344
2345         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2346         check_added_monitors!(nodes[1], 1);
2347         let bs_second_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2348
2349         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2350         check_added_monitors!(nodes[1], 1);
2351         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2352
2353         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.update_fail_htlcs[0]);
2354         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.commitment_signed);
2355         check_added_monitors!(nodes[0], 1);
2356
2357         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2358         check_added_monitors!(nodes[0], 1);
2359         let (as_third_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2360
2361         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2362         check_added_monitors!(nodes[1], 1);
2363         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2364         check_added_monitors!(nodes[1], 1);
2365         let bs_fourth_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2366
2367         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_fourth_raa);
2368         check_added_monitors!(nodes[0], 1);
2369
2370         // At this point A has sent two HTLCs which both failed due to lack of fee. It now has two
2371         // pending `PaymentPathFailed` events, one with `all_paths_failed` unset, and the second
2372         // with it set.
2373         //
2374         // Previously, we retried payments in an event consumer, which would retry each
2375         // `PaymentPathFailed` individually. In that setup, we had retried the payment in response to
2376         // the first `PaymentPathFailed`, then seen the second `PaymentPathFailed` with
2377         // `all_paths_failed` set and assumed the payment was completely failed. We ultimately fixed it
2378         // by adding the `PaymentFailed` event.
2379         //
2380         // Because we now retry payments as a batch, we simply return a single-path route in the
2381         // second, batched, request, have that fail, then complete the payment via `abandon_payment`.
2382         let mut events = nodes[0].node.get_and_clear_pending_events();
2383         assert_eq!(events.len(), 4);
2384         match events[0] {
2385                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2386                         assert_eq!(payment_hash, ev_payment_hash);
2387                         assert_eq!(payment_failed_permanently, false);
2388                 },
2389                 _ => panic!("Unexpected event"),
2390         }
2391         match events[1] {
2392                 Event::PendingHTLCsForwardable { .. } => {},
2393                 _ => panic!("Unexpected event"),
2394         }
2395         match events[2] {
2396                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2397                         assert_eq!(payment_hash, ev_payment_hash);
2398                         assert_eq!(payment_failed_permanently, false);
2399                 },
2400                 _ => panic!("Unexpected event"),
2401         }
2402         match events[3] {
2403                 Event::PendingHTLCsForwardable { .. } => {},
2404                 _ => panic!("Unexpected event"),
2405         }
2406
2407         nodes[0].node.process_pending_htlc_forwards();
2408         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2409         check_added_monitors!(nodes[0], 1);
2410
2411         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2412         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2413         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2414         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2415         commitment_signed_dance!(nodes[0], nodes[1], &bs_fail_update.commitment_signed, false, true);
2416
2417         let mut events = nodes[0].node.get_and_clear_pending_events();
2418         assert_eq!(events.len(), 1);
2419         match events[0] {
2420                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2421                         assert_eq!(payment_hash, ev_payment_hash);
2422                         assert_eq!(payment_failed_permanently, false);
2423                 },
2424                 _ => panic!("Unexpected event"),
2425         }
2426         nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
2427         events = nodes[0].node.get_and_clear_pending_events();
2428         assert_eq!(events.len(), 1);
2429         match events[0] {
2430                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
2431                         assert_eq!(payment_hash, *ev_payment_hash);
2432                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2433                 },
2434                 _ => panic!("Unexpected event"),
2435         }
2436 }
2437
2438 #[test]
2439 fn test_simple_partial_retry() {
2440         // In the first version of the in-`ChannelManager` payment retries, retries were sent for the
2441         // full amount of the payment, rather than only the missing amount. Here we simply test for
2442         // this by sending a payment with two parts, failing one, and retrying the second. Note that
2443         // `TestRouter` will check that the `RouteParameters` (which contain the amount) matches the
2444         // request.
2445         let chanmon_cfgs = create_chanmon_cfgs(3);
2446         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2447         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2448         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2449
2450         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2451         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2452
2453         let amt_msat = 200_000_000;
2454         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
2455         #[cfg(feature = "std")]
2456         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2457         #[cfg(not(feature = "std"))]
2458         let payment_expiry_secs = 60 * 60;
2459         let mut invoice_features = InvoiceFeatures::empty();
2460         invoice_features.set_variable_length_onion_required();
2461         invoice_features.set_payment_secret_required();
2462         invoice_features.set_basic_mpp_optional();
2463         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2464                 .with_expiry_time(payment_expiry_secs as u64)
2465                 .with_features(invoice_features);
2466         let route_params = RouteParameters {
2467                 payment_params,
2468                 final_value_msat: amt_msat,
2469                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2470         };
2471
2472         let mut route = Route {
2473                 paths: vec![
2474                         vec![RouteHop {
2475                                 pubkey: nodes[1].node.get_our_node_id(),
2476                                 node_features: nodes[1].node.node_features(),
2477                                 short_channel_id: chan_1_scid,
2478                                 channel_features: nodes[1].node.channel_features(),
2479                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2480                                 cltv_expiry_delta: 100,
2481                         }, RouteHop {
2482                                 pubkey: nodes[2].node.get_our_node_id(),
2483                                 node_features: nodes[2].node.node_features(),
2484                                 short_channel_id: chan_2_scid,
2485                                 channel_features: nodes[2].node.channel_features(),
2486                                 fee_msat: 100_000_000,
2487                                 cltv_expiry_delta: 100,
2488                         }],
2489                         vec![RouteHop {
2490                                 pubkey: nodes[1].node.get_our_node_id(),
2491                                 node_features: nodes[1].node.node_features(),
2492                                 short_channel_id: chan_1_scid,
2493                                 channel_features: nodes[1].node.channel_features(),
2494                                 fee_msat: 100_000,
2495                                 cltv_expiry_delta: 100,
2496                         }, RouteHop {
2497                                 pubkey: nodes[2].node.get_our_node_id(),
2498                                 node_features: nodes[2].node.node_features(),
2499                                 short_channel_id: chan_2_scid,
2500                                 channel_features: nodes[2].node.channel_features(),
2501                                 fee_msat: 100_000_000,
2502                                 cltv_expiry_delta: 100,
2503                         }]
2504                 ],
2505                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2506         };
2507         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2508         let mut second_payment_params = route_params.payment_params.clone();
2509         second_payment_params.previously_failed_channels = vec![chan_2_scid];
2510         // On retry, we'll only be asked for one path (or 100k sats)
2511         route.paths.remove(0);
2512         nodes[0].router.expect_find_route(RouteParameters {
2513                         payment_params: second_payment_params,
2514                         final_value_msat: amt_msat / 2, final_cltv_expiry_delta: TEST_FINAL_CLTV,
2515                 }, Ok(route.clone()));
2516
2517         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2518         let htlc_updates = SendEvent::from_node(&nodes[0]);
2519         check_added_monitors!(nodes[0], 1);
2520         assert_eq!(htlc_updates.msgs.len(), 1);
2521
2522         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2523         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2524         check_added_monitors!(nodes[1], 1);
2525         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2526
2527         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2528         check_added_monitors!(nodes[0], 1);
2529         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2530
2531         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2532         check_added_monitors!(nodes[0], 1);
2533         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2534
2535         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2536         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2537         check_added_monitors!(nodes[1], 1);
2538         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2539
2540         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2541         check_added_monitors!(nodes[1], 1);
2542         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2543
2544         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2545         check_added_monitors!(nodes[0], 1);
2546
2547         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2548         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2549         check_added_monitors!(nodes[0], 1);
2550         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2551
2552         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2553         check_added_monitors!(nodes[1], 1);
2554
2555         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2556         check_added_monitors!(nodes[1], 1);
2557
2558         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2559
2560         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2561         check_added_monitors!(nodes[0], 1);
2562
2563         let mut events = nodes[0].node.get_and_clear_pending_events();
2564         assert_eq!(events.len(), 2);
2565         match events[0] {
2566                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2567                         assert_eq!(payment_hash, ev_payment_hash);
2568                         assert_eq!(payment_failed_permanently, false);
2569                 },
2570                 _ => panic!("Unexpected event"),
2571         }
2572         match events[1] {
2573                 Event::PendingHTLCsForwardable { .. } => {},
2574                 _ => panic!("Unexpected event"),
2575         }
2576
2577         nodes[0].node.process_pending_htlc_forwards();
2578         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2579         check_added_monitors!(nodes[0], 1);
2580
2581         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2582         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2583
2584         expect_pending_htlcs_forwardable!(nodes[1]);
2585         check_added_monitors!(nodes[1], 1);
2586
2587         let bs_forward_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
2588         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[0]);
2589         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[1]);
2590         commitment_signed_dance!(nodes[2], nodes[1], &bs_forward_update.commitment_signed, false);
2591
2592         expect_pending_htlcs_forwardable!(nodes[2]);
2593         expect_payment_claimable!(nodes[2], payment_hash, payment_secret, amt_msat);
2594 }