_ Git - rust-lightning/blob - lightning/src/ln/payment_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Tests that test the payment retry logic in ChannelManager, including various edge-cases around
  11 //! serialization ordering between ChannelManager/ChannelMonitors and ensuring we can still retry
  12 //! payments thereafter.
  13
  14 use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Listen, Watch};
  15 use crate::chain::channelmonitor::{ANTI_REORG_DELAY, LATENCY_GRACE_PERIOD_BLOCKS};
  16 use crate::chain::keysinterface::EntropySource;
  17 use crate::chain::transaction::OutPoint;
  18 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
  19 use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, ChannelManager, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS};
  20 use crate::ln::features::InvoiceFeatures;
  21 use crate::ln::msgs;
  22 use crate::ln::msgs::ChannelMessageHandler;
  23 use crate::ln::outbound_payment::Retry;
  24 use crate::routing::gossip::RoutingFees;
  25 use crate::routing::router::{get_route, PaymentParameters, Route, RouteHint, RouteHintHop, RouteHop, RouteParameters};
  26 use crate::util::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
  27 use crate::util::test_utils;
  28 use crate::util::errors::APIError;
  29 use crate::util::ser::Writeable;
  30
  31 use bitcoin::{Block, BlockHeader, TxMerkleNode};
  32 use bitcoin::hashes::Hash;
  33 use bitcoin::network::constants::Network;
  34
  35 use crate::prelude::*;
  36
  37 use crate::ln::functional_test_utils::*;
  38 use crate::routing::gossip::NodeId;
  39 #[cfg(feature = "std")]
  40 use {
  41         crate::util::time::tests::SinceEpoch,
  42         std::time::{SystemTime, Duration}
  43 };
  44
  45 #[test]
  46 fn retry_single_path_payment() {
  47         let chanmon_cfgs = create_chanmon_cfgs(3);
  48         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
  49         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
  50         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
  51
  52         let _chan_0 = create_announced_chan_between_nodes(&nodes, 0, 1);
  53         let chan_1 = create_announced_chan_between_nodes(&nodes, 2, 1);
  54         // Rebalance to find a route
  55         send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000);
  56
  57         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 100_000);
  58
  59         // Rebalance so that the first hop fails.
  60         send_payment(&nodes[1], &vec!(&nodes[2])[..], 2_000_000);
  61
  62         // Make sure the payment fails on the first hop.
  63         let payment_id = PaymentId(payment_hash.0);
  64         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
  65         check_added_monitors!(nodes[0], 1);
  66         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
  67         assert_eq!(events.len(), 1);
  68         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
  69         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
  70         check_added_monitors!(nodes[1], 0);
  71         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  72         expect_pending_htlcs_forwardable!(nodes[1]);
  73         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_1.2 }]);
  74         let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
  75         assert!(htlc_updates.update_add_htlcs.is_empty());
  76         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
  77         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
  78         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
  79         check_added_monitors!(nodes[1], 1);
  80         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
  81         commitment_signed_dance!(nodes[0], nodes[1], htlc_updates.commitment_signed, false);
  82         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
  83
  84         // Rebalance the channel so the retry succeeds.
  85         send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000);
  86
  87         // Mine two blocks (we expire retries after 3, so this will check that we don't expire early)
  88         connect_blocks(&nodes[0], 2);
  89
  90         // Retry the payment and make sure it succeeds.
  91         nodes[0].node.retry_payment(&route, payment_id).unwrap();
  92         check_added_monitors!(nodes[0], 1);
  93         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
  94         assert_eq!(events.len(), 1);
  95         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 100_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
  96         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
  97 }
  98
  99 #[test]
 100 fn mpp_failure() {
 101         let chanmon_cfgs = create_chanmon_cfgs(4);
 102         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 103         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 104         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 105
 106         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
 107         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
 108         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
 109         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3).0.contents.short_channel_id;
 110
 111         let (mut route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
 112         let path = route.paths[0].clone();
 113         route.paths.push(path);
 114         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
 115         route.paths[0][0].short_channel_id = chan_1_id;
 116         route.paths[0][1].short_channel_id = chan_3_id;
 117         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
 118         route.paths[1][0].short_channel_id = chan_2_id;
 119         route.paths[1][1].short_channel_id = chan_4_id;
 120         send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
 121         fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
 122 }
 123
 124 #[test]
 125 fn mpp_retry() {
 126         let chanmon_cfgs = create_chanmon_cfgs(4);
 127         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 128         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 129         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 130
 131         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 132         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 133         let (chan_3_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 134         let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes(&nodes, 3, 2);
 135         // Rebalance
 136         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 137
 138         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 1_000_000);
 139         let path = route.paths[0].clone();
 140         route.paths.push(path);
 141         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
 142         route.paths[0][0].short_channel_id = chan_1_update.contents.short_channel_id;
 143         route.paths[0][1].short_channel_id = chan_3_update.contents.short_channel_id;
 144         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
 145         route.paths[1][0].short_channel_id = chan_2_update.contents.short_channel_id;
 146         route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id;
 147
 148         // Initiate the MPP payment.
 149         let payment_id = PaymentId(payment_hash.0);
 150         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
 151         check_added_monitors!(nodes[0], 2); // one monitor per path
 152         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 153         assert_eq!(events.len(), 2);
 154
 155         // Pass half of the payment along the success path.
 156         let (success_path_msgs, mut events) = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &events);
 157         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), success_path_msgs, false, None);
 158
 159         // Add the HTLC along the first hop.
 160         let (fail_path_msgs_1, _events) = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &events);
 161         let (update_add, commitment_signed) = match fail_path_msgs_1 {
 162                 MessageSendEvent::UpdateHTLCs { node_id: _, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 163                         assert_eq!(update_add_htlcs.len(), 1);
 164                         assert!(update_fail_htlcs.is_empty());
 165                         assert!(update_fulfill_htlcs.is_empty());
 166                         assert!(update_fail_malformed_htlcs.is_empty());
 167                         assert!(update_fee.is_none());
 168                         (update_add_htlcs[0].clone(), commitment_signed.clone())
 169                 },
 170                 _ => panic!("Unexpected event"),
 171         };
 172         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
 173         commitment_signed_dance!(nodes[2], nodes[0], commitment_signed, false);
 174
 175         // Attempt to forward the payment and complete the 2nd path's failure.
 176         expect_pending_htlcs_forwardable!(&nodes[2]);
 177         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id }]);
 178         let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 179         assert!(htlc_updates.update_add_htlcs.is_empty());
 180         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 181         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 182         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 183         check_added_monitors!(nodes[2], 1);
 184         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 185         commitment_signed_dance!(nodes[0], nodes[2], htlc_updates.commitment_signed, false);
 186         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 187
 188         // Rebalance the channel so the second half of the payment can succeed.
 189         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 190
 191         // Make sure it errors as expected given a too-large amount.
 192         if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, payment_id) {
 193                 assert!(err.contains("over total_payment_amt_msat"));
 194         } else { panic!("Unexpected error"); }
 195
 196         // Make sure it errors as expected given the wrong payment_id.
 197         if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, PaymentId([0; 32])) {
 198                 assert!(err.contains("not found"));
 199         } else { panic!("Unexpected error"); }
 200
 201         // Retry the second half of the payment and make sure it succeeds.
 202         let mut path = route.clone();
 203         path.paths.remove(0);
 204         nodes[0].node.retry_payment(&path, payment_id).unwrap();
 205         check_added_monitors!(nodes[0], 1);
 206         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 207         assert_eq!(events.len(), 1);
 208         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 209         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 210 }
 211
 212 fn do_mpp_receive_timeout(send_partial_mpp: bool) {
 213         let chanmon_cfgs = create_chanmon_cfgs(4);
 214         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 215         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 216         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 217
 218         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 219         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 220         let (chan_3_update, _, chan_3_id, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 221         let (chan_4_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 222
 223         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 100_000);
 224         let path = route.paths[0].clone();
 225         route.paths.push(path);
 226         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
 227         route.paths[0][0].short_channel_id = chan_1_update.contents.short_channel_id;
 228         route.paths[0][1].short_channel_id = chan_3_update.contents.short_channel_id;
 229         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
 230         route.paths[1][0].short_channel_id = chan_2_update.contents.short_channel_id;
 231         route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id;
 232
 233         // Initiate the MPP payment.
 234         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 235         check_added_monitors!(nodes[0], 2); // one monitor per path
 236         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 237         assert_eq!(events.len(), 2);
 238
 239         // Pass half of the payment along the first path.
 240         let (node_1_msgs, mut events) = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &events);
 241         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_1_msgs, false, None);
 242
 243         if send_partial_mpp {
 244                 // Time out the partial MPP
 245                 for _ in 0..MPP_TIMEOUT_TICKS {
 246                         nodes[3].node.timer_tick_occurred();
 247                 }
 248
 249                 // Failed HTLC from node 3 -> 1
 250                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 251                 let htlc_fail_updates_3_1 = get_htlc_update_msgs!(nodes[3], nodes[1].node.get_our_node_id());
 252                 assert_eq!(htlc_fail_updates_3_1.update_fail_htlcs.len(), 1);
 253                 nodes[1].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &htlc_fail_updates_3_1.update_fail_htlcs[0]);
 254                 check_added_monitors!(nodes[3], 1);
 255                 commitment_signed_dance!(nodes[1], nodes[3], htlc_fail_updates_3_1.commitment_signed, false);
 256
 257                 // Failed HTLC from node 1 -> 0
 258                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_3_id }]);
 259                 let htlc_fail_updates_1_0 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 260                 assert_eq!(htlc_fail_updates_1_0.update_fail_htlcs.len(), 1);
 261                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates_1_0.update_fail_htlcs[0]);
 262                 check_added_monitors!(nodes[1], 1);
 263                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates_1_0.commitment_signed, false);
 264
 265                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain().expected_htlc_error_data(23, &[][..]));
 266         } else {
 267                 // Pass half of the payment along the second path.
 268                 let (node_2_msgs, _events) = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &events);
 269                 pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_2_msgs, true, None);
 270
 271                 // Even after MPP_TIMEOUT_TICKS we should not timeout the MPP if we have all the parts
 272                 for _ in 0..MPP_TIMEOUT_TICKS {
 273                         nodes[3].node.timer_tick_occurred();
 274                 }
 275
 276                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 277         }
 278 }
 279
 280 #[test]
 281 fn mpp_receive_timeout() {
 282         do_mpp_receive_timeout(true);
 283         do_mpp_receive_timeout(false);
 284 }
 285
 286 #[test]
 287 fn retry_expired_payment() {
 288         let chanmon_cfgs = create_chanmon_cfgs(3);
 289         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 290         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 291         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 292
 293         let _chan_0 = create_announced_chan_between_nodes(&nodes, 0, 1);
 294         let chan_1 = create_announced_chan_between_nodes(&nodes, 2, 1);
 295         // Rebalance to find a route
 296         send_payment(&nodes[2], &vec!(&nodes[1])[..], 3_000_000);
 297
 298         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 100_000);
 299
 300         // Rebalance so that the first hop fails.
 301         send_payment(&nodes[1], &vec!(&nodes[2])[..], 2_000_000);
 302
 303         // Make sure the payment fails on the first hop.
 304         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 305         check_added_monitors!(nodes[0], 1);
 306         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 307         assert_eq!(events.len(), 1);
 308         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
 309         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 310         check_added_monitors!(nodes[1], 0);
 311         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 312         expect_pending_htlcs_forwardable!(nodes[1]);
 313         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_1.2 }]);
 314         let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 315         assert!(htlc_updates.update_add_htlcs.is_empty());
 316         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 317         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 318         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 319         check_added_monitors!(nodes[1], 1);
 320         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 321         commitment_signed_dance!(nodes[0], nodes[1], htlc_updates.commitment_signed, false);
 322         expect_payment_failed!(nodes[0], payment_hash, false);
 323
 324         // Mine blocks so the payment will have expired.
 325         connect_blocks(&nodes[0], 3);
 326
 327         // Retry the payment and make sure it errors as expected.
 328         if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, PaymentId(payment_hash.0)) {
 329                 assert!(err.contains("not found"));
 330         } else {
 331                 panic!("Unexpected error");
 332         }
 333 }
 334
 335 #[test]
 336 fn no_pending_leak_on_initial_send_failure() {
 337         // In an earlier version of our payment tracking, we'd have a retry entry even when the initial
 338         // HTLC for payment failed to send due to local channel errors (e.g. peer disconnected). In this
 339         // case, the user wouldn't have a PaymentId to retry the payment with, but we'd think we have a
 340         // pending payment forever and never time it out.
 341         // Here we test exactly that - retrying a payment when a peer was disconnected on the first
 342         // try, and then check that no pending payment is being tracked.
 343         let chanmon_cfgs = create_chanmon_cfgs(2);
 344         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 345         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 346         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 347
 348         create_announced_chan_between_nodes(&nodes, 0, 1);
 349
 350         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 351
 352         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 353         nodes[1].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 354
 355         unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)),
 356                 true, APIError::ChannelUnavailable { ref err },
 357                 assert_eq!(err, "Peer for first hop currently disconnected/pending monitor update!"));
 358
 359         assert!(!nodes[0].node.has_pending_payments());
 360 }
 361
 362 fn do_retry_with_no_persist(confirm_before_reload: bool) {
 363         // If we send a pending payment and `send_payment` returns success, we should always either
 364         // return a payment failure event or a payment success event, and on failure the payment should
 365         // be retryable.
 366         //
 367         // In order to do so when the ChannelManager isn't immediately persisted (which is normal - its
 368         // always persisted asynchronously), the ChannelManager has to reload some payment data from
 369         // ChannelMonitor(s) in some cases. This tests that reloading.
 370         //
 371         // `confirm_before_reload` confirms the channel-closing commitment transaction on-chain prior
 372         // to reloading the ChannelManager, increasing test coverage in ChannelMonitor HTLC tracking
 373         // which has separate codepaths for "commitment transaction already confirmed" and not.
 374         let chanmon_cfgs = create_chanmon_cfgs(3);
 375         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 376         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 377         let persister: test_utils::TestPersister;
 378         let new_chain_monitor: test_utils::TestChainMonitor;
 379         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 380         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 381
 382         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 383         let (_, _, chan_id_2, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
 384
 385         // Serialize the ChannelManager prior to sending payments
 386         let nodes_0_serialized = nodes[0].node.encode();
 387
 388         // Send two payments - one which will get to nodes[2] and will be claimed, one which we'll time
 389         // out and retry.
 390         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 391         let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000);
 392         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 393         check_added_monitors!(nodes[0], 1);
 394
 395         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 396         assert_eq!(events.len(), 1);
 397         let payment_event = SendEvent::from_event(events.pop().unwrap());
 398         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 399
 400         // We relay the payment to nodes[1] while its disconnected from nodes[2], causing the payment
 401         // to be returned immediately to nodes[0], without having nodes[2] fail the inbound payment
 402         // which would prevent retry.
 403         nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id(), false);
 404         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 405
 406         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 407         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true);
 408         // nodes[1] now immediately fails the HTLC as the next-hop channel is disconnected
 409         let _ = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 410
 411         reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 412
 413         let as_commitment_tx = get_local_commitment_txn!(nodes[0], chan_id)[0].clone();
 414         if confirm_before_reload {
 415                 mine_transaction(&nodes[0], &as_commitment_tx);
 416                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 417         }
 418
 419         // The ChannelMonitor should always be the latest version, as we're required to persist it
 420         // during the `commitment_signed_dance!()`.
 421         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 422         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 423
 424         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 425         // force-close the channel.
 426         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 427         assert!(nodes[0].node.list_channels().is_empty());
 428         assert!(nodes[0].node.has_pending_payments());
 429         let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 430         assert_eq!(as_broadcasted_txn.len(), 1);
 431         assert_eq!(as_broadcasted_txn[0], as_commitment_tx);
 432
 433         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 434         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
 435         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 436
 437         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 438         // error, as the channel has hit the chain.
 439         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
 440         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 441         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 442         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 443         assert_eq!(as_err.len(), 1);
 444         match as_err[0] {
 445                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 446                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 447                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 448                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()) });
 449                         check_added_monitors!(nodes[1], 1);
 450                         assert_eq!(nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 451                 },
 452                 _ => panic!("Unexpected event"),
 453         }
 454         check_closed_broadcast!(nodes[1], false);
 455
 456         // Now claim the first payment, which should allow nodes[1] to claim the payment on-chain when
 457         // we close in a moment.
 458         nodes[2].node.claim_funds(payment_preimage_1);
 459         check_added_monitors!(nodes[2], 1);
 460         expect_payment_claimed!(nodes[2], payment_hash_1, 1_000_000);
 461
 462         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 463         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 464         check_added_monitors!(nodes[1], 1);
 465         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 466         expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], None, false, false);
 467
 468         if confirm_before_reload {
 469                 let best_block = nodes[0].blocks.lock().unwrap().last().unwrap().clone();
 470                 nodes[0].node.best_block_updated(&best_block.0.header, best_block.1);
 471         }
 472
 473         // Create a new channel on which to retry the payment before we fail the payment via the
 474         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 475         // connecting several blocks while creating the channel (implying time has passed).
 476         create_announced_chan_between_nodes(&nodes, 0, 1);
 477         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 478
 479         mine_transaction(&nodes[1], &as_commitment_tx);
 480         let bs_htlc_claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 481         assert_eq!(bs_htlc_claim_txn.len(), 1);
 482         check_spends!(bs_htlc_claim_txn[0], as_commitment_tx);
 483
 484         if !confirm_before_reload {
 485                 mine_transaction(&nodes[0], &as_commitment_tx);
 486         }
 487         mine_transaction(&nodes[0], &bs_htlc_claim_txn[0]);
 488         expect_payment_sent!(nodes[0], payment_preimage_1);
 489         connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
 490         let as_htlc_timeout_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 491         assert_eq!(as_htlc_timeout_txn.len(), 2);
 492         let (first_htlc_timeout_tx, second_htlc_timeout_tx) = (&as_htlc_timeout_txn[0], &as_htlc_timeout_txn[1]);
 493         check_spends!(first_htlc_timeout_tx, as_commitment_tx);
 494         check_spends!(second_htlc_timeout_tx, as_commitment_tx);
 495         if first_htlc_timeout_tx.input[0].previous_output == bs_htlc_claim_txn[0].input[0].previous_output {
 496                 confirm_transaction(&nodes[0], &second_htlc_timeout_tx);
 497         } else {
 498                 confirm_transaction(&nodes[0], &first_htlc_timeout_tx);
 499         }
 500         nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 501         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 502
 503         // Finally, retry the payment (which was reloaded from the ChannelMonitor when nodes[0] was
 504         // reloaded) via a route over the new channel, which work without issue and eventually be
 505         // received and claimed at the recipient just like any other payment.
 506         let (mut new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 507
 508         // Update the fee on the middle hop to ensure PaymentSent events have the correct (retried) fee
 509         // and not the original fee. We also update node[1]'s relevant config as
 510         // do_claim_payment_along_route expects us to never overpay.
 511         {
 512                 let per_peer_state = nodes[1].node.per_peer_state.read().unwrap();
 513                 let mut peer_state = per_peer_state.get(&nodes[2].node.get_our_node_id())
 514                         .unwrap().lock().unwrap();
 515                 let mut channel = peer_state.channel_by_id.get_mut(&chan_id_2).unwrap();
 516                 let mut new_config = channel.config();
 517                 new_config.forwarding_fee_base_msat += 100_000;
 518                 channel.update_config(&new_config);
 519                 new_route.paths[0][0].fee_msat += 100_000;
 520         }
 521
 522         // Force expiration of the channel's previous config.
 523         for _ in 0..EXPIRE_PREV_CONFIG_TICKS {
 524                 nodes[1].node.timer_tick_occurred();
 525         }
 526
 527         assert!(nodes[0].node.retry_payment(&new_route, payment_id_1).is_err()); // Shouldn't be allowed to retry a fulfilled payment
 528         nodes[0].node.retry_payment(&new_route, PaymentId(payment_hash.0)).unwrap();
 529         check_added_monitors!(nodes[0], 1);
 530         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 531         assert_eq!(events.len(), 1);
 532         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 533         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
 534         expect_payment_sent!(nodes[0], payment_preimage, Some(new_route.paths[0][0].fee_msat));
 535 }
 536
 537 #[test]
 538 fn retry_with_no_persist() {
 539         do_retry_with_no_persist(true);
 540         do_retry_with_no_persist(false);
 541 }
 542
 543 fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 544         // Test that an off-chain completed payment is not retryable on restart. This was previously
 545         // broken for dust payments, but we test for both dust and non-dust payments.
 546         //
 547         // `use_dust` switches to using a dust HTLC, which results in the HTLC not having an on-chain
 548         // output at all.
 549         let chanmon_cfgs = create_chanmon_cfgs(3);
 550         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 551
 552         let mut manually_accept_config = test_default_channel_config();
 553         manually_accept_config.manually_accept_inbound_channels = true;
 554
 555         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(manually_accept_config), None]);
 556
 557         let first_persister: test_utils::TestPersister;
 558         let first_new_chain_monitor: test_utils::TestChainMonitor;
 559         let first_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 560         let second_persister: test_utils::TestPersister;
 561         let second_new_chain_monitor: test_utils::TestChainMonitor;
 562         let second_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 563         let third_persister: test_utils::TestPersister;
 564         let third_new_chain_monitor: test_utils::TestChainMonitor;
 565         let third_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 566
 567         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 568
 569         // Because we set nodes[1] to manually accept channels, just open a 0-conf channel.
 570         let (funding_tx, chan_id) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 571         confirm_transaction(&nodes[0], &funding_tx);
 572         confirm_transaction(&nodes[1], &funding_tx);
 573         // Ignore the announcement_signatures messages
 574         nodes[0].node.get_and_clear_pending_msg_events();
 575         nodes[1].node.get_and_clear_pending_msg_events();
 576         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 577
 578         // Serialize the ChannelManager prior to sending payments
 579         let mut nodes_0_serialized = nodes[0].node.encode();
 580
 581         let route = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 }).0;
 582         let (payment_preimage, payment_hash, payment_secret, payment_id) = send_along_route(&nodes[0], route, &[&nodes[1], &nodes[2]], if use_dust { 1_000 } else { 1_000_000 });
 583
 584         // The ChannelMonitor should always be the latest version, as we're required to persist it
 585         // during the `commitment_signed_dance!()`.
 586         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 587
 588         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized], first_persister, first_new_chain_monitor, first_nodes_0_deserialized);
 589         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 590
 591         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 592         // force-close the channel.
 593         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 594         assert!(nodes[0].node.list_channels().is_empty());
 595         assert!(nodes[0].node.has_pending_payments());
 596         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 597
 598         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
 599         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 600
 601         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 602         // error, as the channel has hit the chain.
 603         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
 604         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 605         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 606         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 607         assert_eq!(as_err.len(), 1);
 608         let bs_commitment_tx;
 609         match as_err[0] {
 610                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 611                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 612                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 613                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()) });
 614                         check_added_monitors!(nodes[1], 1);
 615                         bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 616                 },
 617                 _ => panic!("Unexpected event"),
 618         }
 619         check_closed_broadcast!(nodes[1], false);
 620
 621         // Now fail back the payment from nodes[2] to nodes[1]. This doesn't really matter as the
 622         // previous hop channel is already on-chain, but it makes nodes[2] willing to see additional
 623         // incoming HTLCs with the same payment hash later.
 624         nodes[2].node.fail_htlc_backwards(&payment_hash);
 625         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
 626         check_added_monitors!(nodes[2], 1);
 627
 628         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 629         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fail_htlcs[0]);
 630         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 631         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1],
 632                 [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 633
 634         // Connect the HTLC-Timeout transaction, timing out the HTLC on both nodes (but not confirming
 635         // the HTLC-Timeout transaction beyond 1 conf). For dust HTLCs, the HTLC is considered resolved
 636         // after the commitment transaction, so always connect the commitment transaction.
 637         mine_transaction(&nodes[0], &bs_commitment_tx[0]);
 638         mine_transaction(&nodes[1], &bs_commitment_tx[0]);
 639         if !use_dust {
 640                 connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1 + (MIN_CLTV_EXPIRY_DELTA as u32));
 641                 connect_blocks(&nodes[1], TEST_FINAL_CLTV - 1 + (MIN_CLTV_EXPIRY_DELTA as u32));
 642                 let as_htlc_timeout = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 643                 check_spends!(as_htlc_timeout[0], bs_commitment_tx[0]);
 644                 assert_eq!(as_htlc_timeout.len(), 1);
 645
 646                 mine_transaction(&nodes[0], &as_htlc_timeout[0]);
 647                 // nodes[0] may rebroadcast (or RBF-bump) its HTLC-Timeout, so wipe the announced set.
 648                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 649                 mine_transaction(&nodes[1], &as_htlc_timeout[0]);
 650         }
 651
 652         // Create a new channel on which to retry the payment before we fail the payment via the
 653         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 654         // connecting several blocks while creating the channel (implying time has passed).
 655         // We do this with a zero-conf channel to avoid connecting blocks as a side-effect.
 656         let (_, chan_id_3) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 657         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 658
 659         // If we attempt to retry prior to the HTLC-Timeout (or commitment transaction, for dust HTLCs)
 660         // confirming, we will fail as it's considered still-pending...
 661         let (new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 });
 662         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_err());
 663         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 664
 665         // After ANTI_REORG_DELAY confirmations, the HTLC should be failed and we can try the payment
 666         // again. We serialize the node first as we'll then test retrying the HTLC after a restart
 667         // (which should also still work).
 668         connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
 669         connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 670         // We set mpp_parts_remain to avoid having abandon_payment called
 671         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 672
 673         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 674         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 675         nodes_0_serialized = nodes[0].node.encode();
 676
 677         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_ok());
 678         assert!(!nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 679
 680         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], second_persister, second_new_chain_monitor, second_nodes_0_deserialized);
 681         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 682
 683         reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 684
 685         // Now resend the payment, delivering the HTLC and actually claiming it this time. This ensures
 686         // the payment is not (spuriously) listed as still pending.
 687         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_ok());
 688         check_added_monitors!(nodes[0], 1);
 689         pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], if use_dust { 1_000 } else { 1_000_000 }, payment_hash, payment_secret);
 690         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
 691
 692         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_err());
 693         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 694
 695         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 696         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 697         nodes_0_serialized = nodes[0].node.encode();
 698
 699         // Ensure that after reload we cannot retry the payment.
 700         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], third_persister, third_new_chain_monitor, third_nodes_0_deserialized);
 701         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 702
 703         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 704
 705         assert!(nodes[0].node.retry_payment(&new_route, payment_id).is_err());
 706         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 707 }
 708
 709 #[test]
 710 fn test_completed_payment_not_retryable_on_reload() {
 711         do_test_completed_payment_not_retryable_on_reload(true);
 712         do_test_completed_payment_not_retryable_on_reload(false);
 713 }
 714
 715
 716 fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, confirm_commitment_tx: bool, payment_timeout: bool) {
 717         // When a Channel is closed, any outbound HTLCs which were relayed through it are simply
 718         // dropped when the Channel is. From there, the ChannelManager relies on the ChannelMonitor
 719         // having a copy of the relevant fail-/claim-back data and processes the HTLC fail/claim when
 720         // the ChannelMonitor tells it to.
 721         //
 722         // If, due to an on-chain event, an HTLC is failed/claimed, we should avoid providing the
 723         // ChannelManager the HTLC event until after the monitor is re-persisted. This should prevent a
 724         // duplicate HTLC fail/claim (e.g. via a PaymentPathFailed event).
 725         let chanmon_cfgs = create_chanmon_cfgs(2);
 726         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 727         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 728         let persister: test_utils::TestPersister;
 729         let new_chain_monitor: test_utils::TestChainMonitor;
 730         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 731         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 732
 733         let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1);
 734
 735         // Route a payment, but force-close the channel before the HTLC fulfill message arrives at
 736         // nodes[0].
 737         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 10_000_000);
 738         nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap();
 739         check_closed_broadcast!(nodes[0], true);
 740         check_added_monitors!(nodes[0], 1);
 741         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 742
 743         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 744         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 745
 746         // Connect blocks until the CLTV timeout is up so that we get an HTLC-Timeout transaction
 747         connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
 748         let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 749         assert_eq!(node_txn.len(), 3);
 750         assert_eq!(node_txn[0], node_txn[1]);
 751         check_spends!(node_txn[1], funding_tx);
 752         check_spends!(node_txn[2], node_txn[1]);
 753         let timeout_txn = vec![node_txn[2].clone()];
 754
 755         nodes[1].node.claim_funds(payment_preimage);
 756         check_added_monitors!(nodes[1], 1);
 757         expect_payment_claimed!(nodes[1], payment_hash, 10_000_000);
 758
 759         let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 };
 760         connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[1].clone()]});
 761         check_closed_broadcast!(nodes[1], true);
 762         check_added_monitors!(nodes[1], 1);
 763         check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 764         let claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 765         assert_eq!(claim_txn.len(), 1);
 766         check_spends!(claim_txn[0], node_txn[1]);
 767
 768         header.prev_blockhash = nodes[0].best_block_hash();
 769         connect_block(&nodes[0], &Block { header, txdata: vec![node_txn[1].clone()]});
 770
 771         if confirm_commitment_tx {
 772                 connect_blocks(&nodes[0], BREAKDOWN_TIMEOUT as u32 - 1);
 773         }
 774
 775         header.prev_blockhash = nodes[0].best_block_hash();
 776         let claim_block = Block { header, txdata: if payment_timeout { timeout_txn } else { vec![claim_txn[0].clone()] } };
 777
 778         if payment_timeout {
 779                 assert!(confirm_commitment_tx); // Otherwise we're spending below our CSV!
 780                 connect_block(&nodes[0], &claim_block);
 781                 connect_blocks(&nodes[0], ANTI_REORG_DELAY - 2);
 782         }
 783
 784         // Now connect the HTLC claim transaction with the ChainMonitor-generated ChannelMonitor update
 785         // returning InProgress. This should cause the claim event to never make its way to the
 786         // ChannelManager.
 787         chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap().clear();
 788         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
 789
 790         if payment_timeout {
 791                 connect_blocks(&nodes[0], 1);
 792         } else {
 793                 connect_block(&nodes[0], &claim_block);
 794         }
 795
 796         let funding_txo = OutPoint { txid: funding_tx.txid(), index: 0 };
 797         let mon_updates: Vec<_> = chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap()
 798                 .get_mut(&funding_txo).unwrap().drain().collect();
 799         // If we are using chain::Confirm instead of chain::Listen, we will get the same update twice.
 800         // If we're testing connection idempotency we may get substantially more.
 801         assert!(mon_updates.len() >= 1);
 802         assert!(nodes[0].chain_monitor.release_pending_monitor_events().is_empty());
 803         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 804
 805         // If we persist the ChannelManager here, we should get the PaymentSent event after
 806         // deserialization.
 807         let mut chan_manager_serialized = Vec::new();
 808         if !persist_manager_post_event {
 809                 chan_manager_serialized = nodes[0].node.encode();
 810         }
 811
 812         // Now persist the ChannelMonitor and inform the ChainMonitor that we're done, generating the
 813         // payment sent event.
 814         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
 815         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 816         for update in mon_updates {
 817                 nodes[0].chain_monitor.chain_monitor.channel_monitor_updated(funding_txo, update).unwrap();
 818         }
 819         if payment_timeout {
 820                 expect_payment_failed!(nodes[0], payment_hash, false);
 821         } else {
 822                 expect_payment_sent!(nodes[0], payment_preimage);
 823         }
 824
 825         // If we persist the ChannelManager after we get the PaymentSent event, we shouldn't get it
 826         // twice.
 827         if persist_manager_post_event {
 828                 chan_manager_serialized = nodes[0].node.encode();
 829         }
 830
 831         // Now reload nodes[0]...
 832         reload_node!(nodes[0], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 833
 834         if persist_manager_post_event {
 835                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 836         } else if payment_timeout {
 837                 expect_payment_failed!(nodes[0], payment_hash, false);
 838         } else {
 839                 expect_payment_sent!(nodes[0], payment_preimage);
 840         }
 841
 842         // Note that if we re-connect the block which exposed nodes[0] to the payment preimage (but
 843         // which the current ChannelMonitor has not seen), the ChannelManager's de-duplication of
 844         // payment events should kick in, leaving us with no pending events here.
 845         let height = nodes[0].blocks.lock().unwrap().len() as u32 - 1;
 846         nodes[0].chain_monitor.chain_monitor.block_connected(&claim_block, height);
 847         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 848 }
 849
 850 #[test]
 851 fn test_dup_htlc_onchain_fails_on_reload() {
 852         do_test_dup_htlc_onchain_fails_on_reload(true, true, true);
 853         do_test_dup_htlc_onchain_fails_on_reload(true, true, false);
 854         do_test_dup_htlc_onchain_fails_on_reload(true, false, false);
 855         do_test_dup_htlc_onchain_fails_on_reload(false, true, true);
 856         do_test_dup_htlc_onchain_fails_on_reload(false, true, false);
 857         do_test_dup_htlc_onchain_fails_on_reload(false, false, false);
 858 }
 859
 860 #[test]
 861 fn test_fulfill_restart_failure() {
 862         // When we receive an update_fulfill_htlc message, we immediately consider the HTLC fully
 863         // fulfilled. At this point, the peer can reconnect and decide to either fulfill the HTLC
 864         // again, or fail it, giving us free money.
 865         //
 866         // Of course probably they won't fail it and give us free money, but because we have code to
 867         // handle it, we should test the logic for it anyway. We do that here.
 868         let chanmon_cfgs = create_chanmon_cfgs(2);
 869         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 870         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 871         let persister: test_utils::TestPersister;
 872         let new_chain_monitor: test_utils::TestChainMonitor;
 873         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 874         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 875
 876         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 877         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 100_000);
 878
 879         // The simplest way to get a failure after a fulfill is to reload nodes[1] from a state
 880         // pre-fulfill, which we do by serializing it here.
 881         let chan_manager_serialized = nodes[1].node.encode();
 882         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id).encode();
 883
 884         nodes[1].node.claim_funds(payment_preimage);
 885         check_added_monitors!(nodes[1], 1);
 886         expect_payment_claimed!(nodes[1], payment_hash, 100_000);
 887
 888         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 889         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 890         expect_payment_sent_without_paths!(nodes[0], payment_preimage);
 891
 892         // Now reload nodes[1]...
 893         reload_node!(nodes[1], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 894
 895         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 896         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 897
 898         nodes[1].node.fail_htlc_backwards(&payment_hash);
 899         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]);
 900         check_added_monitors!(nodes[1], 1);
 901         let htlc_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 902         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates.update_fail_htlcs[0]);
 903         commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates.commitment_signed, false);
 904         // nodes[0] shouldn't generate any events here, while it just got a payment failure completion
 905         // it had already considered the payment fulfilled, and now they just got free money.
 906         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 907 }
 908
 909 #[test]
 910 fn get_ldk_payment_preimage() {
 911         // Ensure that `ChannelManager::get_payment_preimage` can successfully be used to claim a payment.
 912         let chanmon_cfgs = create_chanmon_cfgs(2);
 913         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 914         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 915         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 916         create_announced_chan_between_nodes(&nodes, 0, 1);
 917
 918         let amt_msat = 60_000;
 919         let expiry_secs = 60 * 60;
 920         let (payment_hash, payment_secret) = nodes[1].node.create_inbound_payment(Some(amt_msat), expiry_secs, None).unwrap();
 921
 922         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
 923                 .with_features(nodes[1].node.invoice_features());
 924         let scorer = test_utils::TestScorer::with_penalty(0);
 925         let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet);
 926         let random_seed_bytes = keys_manager.get_secure_random_bytes();
 927         let route = get_route(
 928                 &nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(),
 929                 Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()),
 930                 amt_msat, TEST_FINAL_CLTV, nodes[0].logger, &scorer, &random_seed_bytes).unwrap();
 931         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 932         check_added_monitors!(nodes[0], 1);
 933
 934         // Make sure to use `get_payment_preimage`
 935         let payment_preimage = nodes[1].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
 936         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 937         assert_eq!(events.len(), 1);
 938         pass_along_path(&nodes[0], &[&nodes[1]], amt_msat, payment_hash, Some(payment_secret), events.pop().unwrap(), true, Some(payment_preimage));
 939         claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, payment_preimage);
 940 }
 941
 942 #[test]
 943 fn sent_probe_is_probe_of_sending_node() {
 944         let chanmon_cfgs = create_chanmon_cfgs(3);
 945         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 946         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
 947         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 948
 949         create_announced_chan_between_nodes(&nodes, 0, 1);
 950         create_announced_chan_between_nodes(&nodes, 1, 2);
 951
 952         // First check we refuse to build a single-hop probe
 953         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[1], 100_000);
 954         assert!(nodes[0].node.send_probe(route.paths[0].clone()).is_err());
 955
 956         // Then build an actual two-hop probing path
 957         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
 958
 959         match nodes[0].node.send_probe(route.paths[0].clone()) {
 960                 Ok((payment_hash, payment_id)) => {
 961                         assert!(nodes[0].node.payment_is_probe(&payment_hash, &payment_id));
 962                         assert!(!nodes[1].node.payment_is_probe(&payment_hash, &payment_id));
 963                         assert!(!nodes[2].node.payment_is_probe(&payment_hash, &payment_id));
 964                 },
 965                 _ => panic!(),
 966         }
 967
 968         get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 969         check_added_monitors!(nodes[0], 1);
 970 }
 971
 972 #[test]
 973 fn successful_probe_yields_event() {
 974         let chanmon_cfgs = create_chanmon_cfgs(3);
 975         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 976         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
 977         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 978
 979         create_announced_chan_between_nodes(&nodes, 0, 1);
 980         create_announced_chan_between_nodes(&nodes, 1, 2);
 981
 982         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
 983
 984         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
 985
 986         // node[0] -- update_add_htlcs -> node[1]
 987         check_added_monitors!(nodes[0], 1);
 988         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 989         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
 990         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
 991         check_added_monitors!(nodes[1], 0);
 992         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
 993         expect_pending_htlcs_forwardable!(nodes[1]);
 994
 995         // node[1] -- update_add_htlcs -> node[2]
 996         check_added_monitors!(nodes[1], 1);
 997         let updates = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 998         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
 999         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &probe_event.msgs[0]);
1000         check_added_monitors!(nodes[2], 0);
1001         commitment_signed_dance!(nodes[2], nodes[1], probe_event.commitment_msg, true, true);
1002
1003         // node[1] <- update_fail_htlcs -- node[2]
1004         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1005         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1006         check_added_monitors!(nodes[1], 0);
1007         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, true);
1008
1009         // node[0] <- update_fail_htlcs -- node[1]
1010         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1011         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1012         check_added_monitors!(nodes[0], 0);
1013         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1014
1015         let mut events = nodes[0].node.get_and_clear_pending_events();
1016         assert_eq!(events.len(), 1);
1017         match events.drain(..).next().unwrap() {
1018                 crate::util::events::Event::ProbeSuccessful { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1019                         assert_eq!(payment_id, ev_pid);
1020                         assert_eq!(payment_hash, ev_ph);
1021                 },
1022                 _ => panic!(),
1023         };
1024 }
1025
1026 #[test]
1027 fn failed_probe_yields_event() {
1028         let chanmon_cfgs = create_chanmon_cfgs(3);
1029         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1030         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1031         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1032
1033         create_announced_chan_between_nodes(&nodes, 0, 1);
1034         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 100000, 90000000);
1035
1036         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1037
1038         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 9_998_000, 42);
1039
1040         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1041
1042         // node[0] -- update_add_htlcs -> node[1]
1043         check_added_monitors!(nodes[0], 1);
1044         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1045         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1046         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1047         check_added_monitors!(nodes[1], 0);
1048         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1049         expect_pending_htlcs_forwardable!(nodes[1]);
1050
1051         // node[0] <- update_fail_htlcs -- node[1]
1052         check_added_monitors!(nodes[1], 1);
1053         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1054         // Skip the PendingHTLCsForwardable event
1055         let _events = nodes[1].node.get_and_clear_pending_events();
1056         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1057         check_added_monitors!(nodes[0], 0);
1058         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1059
1060         let mut events = nodes[0].node.get_and_clear_pending_events();
1061         assert_eq!(events.len(), 1);
1062         match events.drain(..).next().unwrap() {
1063                 crate::util::events::Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1064                         assert_eq!(payment_id, ev_pid);
1065                         assert_eq!(payment_hash, ev_ph);
1066                 },
1067                 _ => panic!(),
1068         };
1069 }
1070
1071 #[test]
1072 fn onchain_failed_probe_yields_event() {
1073         // Tests that an attempt to probe over a channel that is eventaully closed results in a failure
1074         // event.
1075         let chanmon_cfgs = create_chanmon_cfgs(3);
1076         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1077         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1078         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1079
1080         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1081         create_announced_chan_between_nodes(&nodes, 1, 2);
1082
1083         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1084
1085         // Send a dust HTLC, which will be treated as if it timed out once the channel hits the chain.
1086         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 1_000, 42);
1087         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1088
1089         // node[0] -- update_add_htlcs -> node[1]
1090         check_added_monitors!(nodes[0], 1);
1091         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1092         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1093         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1094         check_added_monitors!(nodes[1], 0);
1095         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1096         expect_pending_htlcs_forwardable!(nodes[1]);
1097
1098         check_added_monitors!(nodes[1], 1);
1099         let _ = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1100
1101         // Don't bother forwarding the HTLC onwards and just confirm the force-close transaction on
1102         // Node A, which after 6 confirmations should result in a probe failure event.
1103         let bs_txn = get_local_commitment_txn!(nodes[1], chan_id);
1104         confirm_transaction(&nodes[0], &bs_txn[0]);
1105         check_closed_broadcast!(&nodes[0], true);
1106         check_added_monitors!(nodes[0], 1);
1107
1108         let mut events = nodes[0].node.get_and_clear_pending_events();
1109         assert_eq!(events.len(), 2);
1110         let mut found_probe_failed = false;
1111         for event in events.drain(..) {
1112                 match event {
1113                         Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1114                                 assert_eq!(payment_id, ev_pid);
1115                                 assert_eq!(payment_hash, ev_ph);
1116                                 found_probe_failed = true;
1117                         },
1118                         Event::ChannelClosed { .. } => {},
1119                         _ => panic!(),
1120                 }
1121         }
1122         assert!(found_probe_failed);
1123 }
1124
1125 #[test]
1126 fn claimed_send_payment_idempotent() {
1127         // Tests that `send_payment` (and friends) are (reasonably) idempotent.
1128         let chanmon_cfgs = create_chanmon_cfgs(2);
1129         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1130         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1131         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1132
1133         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1134
1135         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1136         let (first_payment_preimage, _, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1137
1138         macro_rules! check_send_rejected {
1139                 () => {
1140                         // If we try to resend a new payment with a different payment_hash but with the same
1141                         // payment_id, it should be rejected.
1142                         let send_result = nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id);
1143                         match send_result {
1144                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1145                                 _ => panic!("Unexpected send result: {:?}", send_result),
1146                         }
1147
1148                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1149                         // also be rejected.
1150                         let send_result = nodes[0].node.send_spontaneous_payment(&route, None, payment_id);
1151                         match send_result {
1152                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1153                                 _ => panic!("Unexpected send result: {:?}", send_result),
1154                         }
1155                 }
1156         }
1157
1158         check_send_rejected!();
1159
1160         // Claim the payment backwards, but note that the PaymentSent event is still pending and has
1161         // not been seen by the user. At this point, from the user perspective nothing has changed, so
1162         // we must remain just as idempotent as we were before.
1163         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, first_payment_preimage);
1164
1165         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1166                 nodes[0].node.timer_tick_occurred();
1167         }
1168
1169         check_send_rejected!();
1170
1171         // Once the user sees and handles the `PaymentSent` event, we expect them to no longer call
1172         // `send_payment`, and our idempotency guarantees are off - they should have atomically marked
1173         // the payment complete. However, they could have called `send_payment` while the event was
1174         // being processed, leading to a race in our idempotency guarantees. Thus, even immediately
1175         // after the event is handled a duplicate payment should sitll be rejected.
1176         expect_payment_sent!(&nodes[0], first_payment_preimage, Some(0));
1177         check_send_rejected!();
1178
1179         // If relatively little time has passed, a duplicate payment should still fail.
1180         nodes[0].node.timer_tick_occurred();
1181         check_send_rejected!();
1182
1183         // However, after some time has passed (at least more than the one timer tick above), a
1184         // duplicate payment should go through, as ChannelManager should no longer have any remaining
1185         // references to the old payment data.
1186         for _ in 0..IDEMPOTENCY_TIMEOUT_TICKS {
1187                 nodes[0].node.timer_tick_occurred();
1188         }
1189
1190         nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id).unwrap();
1191         check_added_monitors!(nodes[0], 1);
1192         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1193         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1194 }
1195
1196 #[test]
1197 fn abandoned_send_payment_idempotent() {
1198         // Tests that `send_payment` (and friends) allow duplicate PaymentIds immediately after
1199         // abandon_payment.
1200         let chanmon_cfgs = create_chanmon_cfgs(2);
1201         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1202         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1203         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1204
1205         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1206
1207         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1208         let (_, first_payment_hash, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1209
1210         macro_rules! check_send_rejected {
1211                 () => {
1212                         // If we try to resend a new payment with a different payment_hash but with the same
1213                         // payment_id, it should be rejected.
1214                         let send_result = nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id);
1215                         match send_result {
1216                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1217                                 _ => panic!("Unexpected send result: {:?}", send_result),
1218                         }
1219
1220                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1221                         // also be rejected.
1222                         let send_result = nodes[0].node.send_spontaneous_payment(&route, None, payment_id);
1223                         match send_result {
1224                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1225                                 _ => panic!("Unexpected send result: {:?}", send_result),
1226                         }
1227                 }
1228         }
1229
1230         check_send_rejected!();
1231
1232         nodes[1].node.fail_htlc_backwards(&first_payment_hash);
1233         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash: first_payment_hash }]);
1234
1235         pass_failed_payment_back_no_abandon(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash);
1236         check_send_rejected!();
1237
1238         // Until we abandon the payment, no matter how many timer ticks pass, we still cannot reuse the
1239         // PaymentId.
1240         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1241                 nodes[0].node.timer_tick_occurred();
1242         }
1243         check_send_rejected!();
1244
1245         nodes[0].node.abandon_payment(payment_id);
1246         get_event!(nodes[0], Event::PaymentFailed);
1247
1248         // However, we can reuse the PaymentId immediately after we `abandon_payment`.
1249         nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id).unwrap();
1250         check_added_monitors!(nodes[0], 1);
1251         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1252         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1253 }
1254
1255 #[derive(PartialEq)]
1256 enum InterceptTest {
1257         Forward,
1258         Fail,
1259         Timeout,
1260 }
1261
1262 #[test]
1263 fn test_trivial_inflight_htlc_tracking(){
1264         // In this test, we test three scenarios:
1265         // (1) Sending + claiming a payment successfully should return `None` when querying InFlightHtlcs
1266         // (2) Sending a payment without claiming it should return the payment's value (500000) when querying InFlightHtlcs
1267         // (3) After we claim the payment sent in (2), InFlightHtlcs should return `None` for the query.
1268         let chanmon_cfgs = create_chanmon_cfgs(3);
1269         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1270         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1271         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1272
1273         let (_, _, chan_1_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
1274         let (_, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
1275
1276         // Send and claim the payment. Inflight HTLCs should be empty.
1277         send_payment(&nodes[0], &vec!(&nodes[1], &nodes[2])[..], 500000);
1278         {
1279                 let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1280
1281                 let mut node_0_per_peer_lock;
1282                 let mut node_0_peer_state_lock;
1283                 let mut node_1_per_peer_lock;
1284                 let mut node_1_peer_state_lock;
1285                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1286                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1287
1288                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1289                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1290                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1291                         channel_1.get_short_channel_id().unwrap()
1292                 );
1293                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1294                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1295                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1296                         channel_2.get_short_channel_id().unwrap()
1297                 );
1298
1299                 assert_eq!(chan_1_used_liquidity, None);
1300                 assert_eq!(chan_2_used_liquidity, None);
1301         }
1302
1303         // Send the payment, but do not claim it. Our inflight HTLCs should contain the pending payment.
1304         let (payment_preimage, _,  _) = route_payment(&nodes[0], &vec!(&nodes[1], &nodes[2])[..], 500000);
1305         {
1306                 let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1307
1308                 let mut node_0_per_peer_lock;
1309                 let mut node_0_peer_state_lock;
1310                 let mut node_1_per_peer_lock;
1311                 let mut node_1_peer_state_lock;
1312                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1313                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1314
1315                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1316                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1317                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1318                         channel_1.get_short_channel_id().unwrap()
1319                 );
1320                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1321                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1322                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1323                         channel_2.get_short_channel_id().unwrap()
1324                 );
1325
1326                 // First hop accounts for expected 1000 msat fee
1327                 assert_eq!(chan_1_used_liquidity, Some(501000));
1328                 assert_eq!(chan_2_used_liquidity, Some(500000));
1329         }
1330
1331         // Now, let's claim the payment. This should result in the used liquidity to return `None`.
1332         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
1333         {
1334                 let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1335
1336                 let mut node_0_per_peer_lock;
1337                 let mut node_0_peer_state_lock;
1338                 let mut node_1_per_peer_lock;
1339                 let mut node_1_peer_state_lock;
1340                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1341                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1342
1343                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1344                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1345                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1346                         channel_1.get_short_channel_id().unwrap()
1347                 );
1348                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1349                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1350                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1351                         channel_2.get_short_channel_id().unwrap()
1352                 );
1353
1354                 assert_eq!(chan_1_used_liquidity, None);
1355                 assert_eq!(chan_2_used_liquidity, None);
1356         }
1357 }
1358
1359 #[test]
1360 fn test_holding_cell_inflight_htlcs() {
1361         let chanmon_cfgs = create_chanmon_cfgs(2);
1362         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1363         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1364         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1365         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1366
1367         let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
1368         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1369
1370         // Queue up two payments - one will be delivered right away, one immediately goes into the
1371         // holding cell as nodes[0] is AwaitingRAA.
1372         {
1373                 nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
1374                 check_added_monitors!(nodes[0], 1);
1375                 nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
1376                 check_added_monitors!(nodes[0], 0);
1377         }
1378
1379         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1380
1381         {
1382                 let mut node_0_per_peer_lock;
1383                 let mut node_0_peer_state_lock;
1384                 let channel =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, channel_id);
1385
1386                 let used_liquidity = inflight_htlcs.used_liquidity_msat(
1387                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1388                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1389                         channel.get_short_channel_id().unwrap()
1390                 );
1391
1392                 assert_eq!(used_liquidity, Some(2000000));
1393         }
1394
1395         // Clear pending events so test doesn't throw a "Had excess message on node..." error
1396         nodes[0].node.get_and_clear_pending_msg_events();
1397 }
1398
1399 #[test]
1400 fn intercepted_payment() {
1401         // Test that detecting an intercept scid on payment forward will signal LDK to generate an
1402         // intercept event, which the LSP can then use to either (a) open a JIT channel to forward the
1403         // payment or (b) fail the payment.
1404         do_test_intercepted_payment(InterceptTest::Forward);
1405         do_test_intercepted_payment(InterceptTest::Fail);
1406         // Make sure that intercepted payments will be automatically failed back if too many blocks pass.
1407         do_test_intercepted_payment(InterceptTest::Timeout);
1408 }
1409
1410 fn do_test_intercepted_payment(test: InterceptTest) {
1411         let chanmon_cfgs = create_chanmon_cfgs(3);
1412         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1413
1414         let mut zero_conf_chan_config = test_default_channel_config();
1415         zero_conf_chan_config.manually_accept_inbound_channels = true;
1416         let mut intercept_forwards_config = test_default_channel_config();
1417         intercept_forwards_config.accept_intercept_htlcs = true;
1418         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(zero_conf_chan_config)]);
1419
1420         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1421         let scorer = test_utils::TestScorer::with_penalty(0);
1422         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
1423
1424         let _ = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1425
1426         let amt_msat = 100_000;
1427         let intercept_scid = nodes[1].node.get_intercept_scid();
1428         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1429                 .with_route_hints(vec![
1430                         RouteHint(vec![RouteHintHop {
1431                                 src_node_id: nodes[1].node.get_our_node_id(),
1432                                 short_channel_id: intercept_scid,
1433                                 fees: RoutingFees {
1434                                         base_msat: 1000,
1435                                         proportional_millionths: 0,
1436                                 },
1437                                 cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
1438                                 htlc_minimum_msat: None,
1439                                 htlc_maximum_msat: None,
1440                         }])
1441                 ])
1442                 .with_features(nodes[2].node.invoice_features());
1443         let route_params = RouteParameters {
1444                 payment_params,
1445                 final_value_msat: amt_msat,
1446                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
1447         };
1448         let route = get_route(
1449                 &nodes[0].node.get_our_node_id(), &route_params.payment_params,
1450                 &nodes[0].network_graph.read_only(), None, route_params.final_value_msat,
1451                 route_params.final_cltv_expiry_delta, nodes[0].logger, &scorer, &random_seed_bytes
1452         ).unwrap();
1453
1454         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
1455         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
1456         let payment_event = {
1457                 {
1458                         let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
1459                         assert_eq!(added_monitors.len(), 1);
1460                         added_monitors.clear();
1461                 }
1462                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1463                 assert_eq!(events.len(), 1);
1464                 SendEvent::from_event(events.remove(0))
1465         };
1466         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1467         commitment_signed_dance!(nodes[1], nodes[0], &payment_event.commitment_msg, false, true);
1468
1469         // Check that we generate the PaymentIntercepted event when an intercept forward is detected.
1470         let events = nodes[1].node.get_and_clear_pending_events();
1471         assert_eq!(events.len(), 1);
1472         let (intercept_id, expected_outbound_amount_msat) = match events[0] {
1473                 crate::util::events::Event::HTLCIntercepted {
1474                         intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, inbound_amount_msat, requested_next_hop_scid: short_channel_id
1475                 } => {
1476                         assert_eq!(pmt_hash, payment_hash);
1477                         assert_eq!(inbound_amount_msat, route.get_total_amount() + route.get_total_fees());
1478                         assert_eq!(short_channel_id, intercept_scid);
1479                         (intercept_id, expected_outbound_amount_msat)
1480                 },
1481                 _ => panic!()
1482         };
1483
1484         // Check for unknown channel id error.
1485         let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &[42; 32], nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1486         assert_eq!(unknown_chan_id_err , APIError::ChannelUnavailable  { err: format!("Channel with id {} not found for the passed counterparty node_id {}", log_bytes!([42; 32]), nodes[2].node.get_our_node_id()) });
1487
1488         if test == InterceptTest::Fail {
1489                 // Ensure we can fail the intercepted payment back.
1490                 nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap();
1491                 expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::UnknownNextHop { requested_forward_scid: intercept_scid }]);
1492                 nodes[1].node.process_pending_htlc_forwards();
1493                 let update_fail = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1494                 check_added_monitors!(&nodes[1], 1);
1495                 assert!(update_fail.update_fail_htlcs.len() == 1);
1496                 let fail_msg = update_fail.update_fail_htlcs[0].clone();
1497                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1498                 commitment_signed_dance!(nodes[0], nodes[1], update_fail.commitment_signed, false);
1499
1500                 // Ensure the payment fails with the expected error.
1501                 let fail_conditions = PaymentFailedConditions::new()
1502                         .blamed_scid(intercept_scid)
1503                         .blamed_chan_closed(true)
1504                         .expected_htlc_error_data(0x4000 | 10, &[]);
1505                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
1506         } else if test == InterceptTest::Forward {
1507                 // Check that we'll fail as expected when sending to a channel that isn't in `ChannelReady` yet.
1508                 let temp_chan_id = nodes[1].node.create_channel(nodes[2].node.get_our_node_id(), 100_000, 0, 42, None).unwrap();
1509                 let unusable_chan_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &temp_chan_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1510                 assert_eq!(unusable_chan_err , APIError::ChannelUnavailable { err: format!("Channel with id {} not fully established", log_bytes!(temp_chan_id)) });
1511                 assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
1512
1513                 // Open the just-in-time channel so the payment can then be forwarded.
1514                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1515
1516                 // Finally, forward the intercepted payment through and claim it.
1517                 nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap();
1518                 expect_pending_htlcs_forwardable!(nodes[1]);
1519
1520                 let payment_event = {
1521                         {
1522                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
1523                                 assert_eq!(added_monitors.len(), 1);
1524                                 added_monitors.clear();
1525                         }
1526                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
1527                         assert_eq!(events.len(), 1);
1528                         SendEvent::from_event(events.remove(0))
1529                 };
1530                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1531                 commitment_signed_dance!(nodes[2], nodes[1], &payment_event.commitment_msg, false, true);
1532                 expect_pending_htlcs_forwardable!(nodes[2]);
1533
1534                 let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1535                 expect_payment_claimable!(&nodes[2], payment_hash, payment_secret, amt_msat, Some(payment_preimage), nodes[2].node.get_our_node_id());
1536                 do_claim_payment_along_route(&nodes[0], &vec!(&vec!(&nodes[1], &nodes[2])[..]), false, payment_preimage);
1537                 let events = nodes[0].node.get_and_clear_pending_events();
1538                 assert_eq!(events.len(), 2);
1539                 match events[0] {
1540                         Event::PaymentSent { payment_preimage: ref ev_preimage, payment_hash: ref ev_hash, ref fee_paid_msat, .. } => {
1541                                 assert_eq!(payment_preimage, *ev_preimage);
1542                                 assert_eq!(payment_hash, *ev_hash);
1543                                 assert_eq!(fee_paid_msat, &Some(1000));
1544                         },
1545                         _ => panic!("Unexpected event")
1546                 }
1547                 match events[1] {
1548                         Event::PaymentPathSuccessful { payment_hash: hash, .. } => {
1549                                 assert_eq!(hash, Some(payment_hash));
1550                         },
1551                         _ => panic!("Unexpected event")
1552                 }
1553         } else if test == InterceptTest::Timeout {
1554                 let mut block = Block {
1555                         header: BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 },
1556                         txdata: vec![],
1557                 };
1558                 connect_block(&nodes[0], &block);
1559                 connect_block(&nodes[1], &block);
1560                 for _ in 0..TEST_FINAL_CLTV {
1561                         block.header.prev_blockhash = block.block_hash();
1562                         connect_block(&nodes[0], &block);
1563                         connect_block(&nodes[1], &block);
1564                 }
1565                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::InvalidForward { requested_forward_scid: intercept_scid }]);
1566                 check_added_monitors!(nodes[1], 1);
1567                 let htlc_timeout_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1568                 assert!(htlc_timeout_updates.update_add_htlcs.is_empty());
1569                 assert_eq!(htlc_timeout_updates.update_fail_htlcs.len(), 1);
1570                 assert!(htlc_timeout_updates.update_fail_malformed_htlcs.is_empty());
1571                 assert!(htlc_timeout_updates.update_fee.is_none());
1572
1573                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_timeout_updates.update_fail_htlcs[0]);
1574                 commitment_signed_dance!(nodes[0], nodes[1], htlc_timeout_updates.commitment_signed, false);
1575                 expect_payment_failed!(nodes[0], payment_hash, false, 0x2000 | 2, []);
1576
1577                 // Check for unknown intercept id error.
1578                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1579                 let unknown_intercept_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1580                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1581                 let unknown_intercept_id_err = nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap_err();
1582                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1583         }
1584 }
1585
1586 #[derive(PartialEq)]
1587 enum AutoRetry {
1588         Success,
1589         FailAttempts,
1590         FailTimeout,
1591         FailOnRestart,
1592 }
1593
1594 #[test]
1595 fn automatic_retries() {
1596         do_automatic_retries(AutoRetry::Success);
1597         do_automatic_retries(AutoRetry::FailAttempts);
1598         do_automatic_retries(AutoRetry::FailTimeout);
1599         do_automatic_retries(AutoRetry::FailOnRestart);
1600 }
1601 fn do_automatic_retries(test: AutoRetry) {
1602         // Test basic automatic payment retries in ChannelManager. See individual `test` variant comments
1603         // below.
1604         let chanmon_cfgs = create_chanmon_cfgs(3);
1605         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1606         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1607
1608         let persister;
1609         let new_chain_monitor;
1610         let node_0_deserialized;
1611
1612         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1613         let channel_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1614         let channel_id_2 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1615
1616         // Marshall data to send the payment
1617         #[cfg(feature = "std")]
1618         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1619         #[cfg(not(feature = "std"))]
1620         let payment_expiry_secs = 60 * 60;
1621         let amt_msat = 1000;
1622         let mut invoice_features = InvoiceFeatures::empty();
1623         invoice_features.set_variable_length_onion_required();
1624         invoice_features.set_payment_secret_required();
1625         invoice_features.set_basic_mpp_optional();
1626         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1627                 .with_expiry_time(payment_expiry_secs as u64)
1628                 .with_features(invoice_features);
1629         let route_params = RouteParameters {
1630                 payment_params,
1631                 final_value_msat: amt_msat,
1632                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
1633         };
1634         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
1635
1636         macro_rules! pass_failed_attempt_with_retry_along_path {
1637                 ($failing_channel_id: expr, $expect_pending_htlcs_forwardable: expr) => {
1638                         // Send a payment attempt that fails due to lack of liquidity on the second hop
1639                         check_added_monitors!(nodes[0], 1);
1640                         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1641                         let mut update_add = update_0.update_add_htlcs[0].clone();
1642                         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
1643                         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
1644                         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
1645                         nodes[1].node.process_pending_htlc_forwards();
1646                         expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1],
1647                                 vec![HTLCDestination::NextHopChannel {
1648                                         node_id: Some(nodes[2].node.get_our_node_id()),
1649                                         channel_id: $failing_channel_id,
1650                                 }]);
1651                         nodes[1].node.process_pending_htlc_forwards();
1652                         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1653                         check_added_monitors!(&nodes[1], 1);
1654                         assert!(update_1.update_fail_htlcs.len() == 1);
1655                         let fail_msg = update_1.update_fail_htlcs[0].clone();
1656
1657                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1658                         commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
1659
1660                         // Ensure the attempt fails and a new PendingHTLCsForwardable event is generated for the retry
1661                         let mut events = nodes[0].node.get_and_clear_pending_events();
1662                         match events[0] {
1663                                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
1664                                         assert_eq!(payment_hash, ev_payment_hash);
1665                                         assert_eq!(payment_failed_permanently, false);
1666                                 },
1667                                 _ => panic!("Unexpected event"),
1668                         }
1669                         if $expect_pending_htlcs_forwardable {
1670                                 assert_eq!(events.len(), 2);
1671                                 match events[1] {
1672                                         Event::PendingHTLCsForwardable { .. } => {},
1673                                         _ => panic!("Unexpected event"),
1674                                 }
1675                         } else { assert_eq!(events.len(), 1) }
1676                 }
1677         }
1678
1679         if test == AutoRetry::Success {
1680                 // Test that we can succeed on the first retry.
1681                 nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1682                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1683
1684                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
1685                 // attempt, since the initial second hop channel will be excluded from pathfinding
1686                 create_announced_chan_between_nodes(&nodes, 1, 2);
1687
1688                 // We retry payments in `process_pending_htlc_forwards`
1689                 nodes[0].node.process_pending_htlc_forwards();
1690                 check_added_monitors!(nodes[0], 1);
1691                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1692                 assert_eq!(msg_events.len(), 1);
1693                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, Some(payment_secret), msg_events.pop().unwrap(), true, None);
1694                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
1695         } else if test == AutoRetry::FailAttempts {
1696                 // Ensure ChannelManager will not retry a payment if it has run out of payment attempts.
1697                 nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1698                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1699
1700                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
1701                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
1702                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1703
1704                 // We retry payments in `process_pending_htlc_forwards`
1705                 nodes[0].node.process_pending_htlc_forwards();
1706                 pass_failed_attempt_with_retry_along_path!(channel_id_3, false);
1707
1708                 // Ensure we won't retry a second time.
1709                 nodes[0].node.process_pending_htlc_forwards();
1710                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1711                 assert_eq!(msg_events.len(), 0);
1712
1713                 nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
1714                 let events = nodes[0].node.get_and_clear_pending_events();
1715                 assert_eq!(events.len(), 1);
1716                 match events[0] {
1717                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
1718                                 assert_eq!(payment_hash, *ev_payment_hash);
1719                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1720                         },
1721                         _ => panic!("Unexpected event"),
1722                 }
1723         } else if test == AutoRetry::FailTimeout {
1724                 #[cfg(not(feature = "no-std"))] {
1725                         // Ensure ChannelManager will not retry a payment if it times out due to Retry::Timeout.
1726                         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
1727                         pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1728
1729                         // Advance the time so the second attempt fails due to timeout.
1730                         SinceEpoch::advance(Duration::from_secs(61));
1731
1732                         // Make sure we don't retry again.
1733                         nodes[0].node.process_pending_htlc_forwards();
1734                         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1735                         assert_eq!(msg_events.len(), 0);
1736
1737                         nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
1738                         let mut events = nodes[0].node.get_and_clear_pending_events();
1739                         assert_eq!(events.len(), 1);
1740                         match events[0] {
1741                                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
1742                                         assert_eq!(payment_hash, *ev_payment_hash);
1743                                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1744                                 },
1745                                 _ => panic!("Unexpected event"),
1746                         }
1747                 }
1748         } else if test == AutoRetry::FailOnRestart {
1749                 // Ensure ChannelManager will not retry a payment after restart, even if there were retry
1750                 // attempts remaining prior to restart.
1751                 nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
1752                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1753
1754                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
1755                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
1756                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1757
1758                 // Ensure the first retry attempt fails, with 1 retry attempt remaining
1759                 nodes[0].node.process_pending_htlc_forwards();
1760                 pass_failed_attempt_with_retry_along_path!(channel_id_3, true);
1761
1762                 // Restart the node and ensure that ChannelManager does not use its remaining retry attempt
1763                 let node_encoded = nodes[0].node.encode();
1764                 let chan_1_monitor_serialized = get_monitor!(nodes[0], channel_id_1).encode();
1765                 reload_node!(nodes[0], node_encoded, &[&chan_1_monitor_serialized], persister, new_chain_monitor, node_0_deserialized);
1766
1767                 // Make sure we don't retry again.
1768                 nodes[0].node.process_pending_htlc_forwards();
1769                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1770                 assert_eq!(msg_events.len(), 0);
1771
1772                 nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
1773                 let mut events = nodes[0].node.get_and_clear_pending_events();
1774                 assert_eq!(events.len(), 1);
1775                 match events[0] {
1776                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
1777                                 assert_eq!(payment_hash, *ev_payment_hash);
1778                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1779                         },
1780                         _ => panic!("Unexpected event"),
1781                 }
1782         }
1783 }
1784
1785 #[test]
1786 fn auto_retry_partial_failure() {
1787         // Test that we'll retry appropriately on send partial failure and retry partial failure.
1788         let chanmon_cfgs = create_chanmon_cfgs(2);
1789         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1790         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1791         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1792
1793         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1794         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1795         let chan_3_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1796
1797         // Marshall data to send the payment
1798         let amt_msat = 20_000;
1799         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
1800         #[cfg(feature = "std")]
1801         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1802         #[cfg(not(feature = "std"))]
1803         let payment_expiry_secs = 60 * 60;
1804         let mut invoice_features = InvoiceFeatures::empty();
1805         invoice_features.set_variable_length_onion_required();
1806         invoice_features.set_payment_secret_required();
1807         invoice_features.set_basic_mpp_optional();
1808         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
1809                 .with_expiry_time(payment_expiry_secs as u64)
1810                 .with_features(invoice_features);
1811         let route_params = RouteParameters {
1812                 payment_params,
1813                 final_value_msat: amt_msat,
1814                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
1815         };
1816
1817         // Ensure the first monitor update (for the initial send path1 over chan_1) succeeds, but the
1818         // second (for the initial send path2 over chan_2) fails.
1819         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1820         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
1821         // Ensure third monitor update (for the retry1's path1 over chan_1) succeeds, but the fourth (for
1822         // the retry1's path2 over chan_3) fails, and monitor updates succeed after that.
1823         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1824         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
1825         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1826
1827         // Configure the initial send, retry1 and retry2's paths.
1828         let send_route = Route {
1829                 paths: vec![
1830                         vec![RouteHop {
1831                                 pubkey: nodes[1].node.get_our_node_id(),
1832                                 node_features: nodes[1].node.node_features(),
1833                                 short_channel_id: chan_1_id,
1834                                 channel_features: nodes[1].node.channel_features(),
1835                                 fee_msat: amt_msat / 2,
1836                                 cltv_expiry_delta: 100,
1837                         }],
1838                         vec![RouteHop {
1839                                 pubkey: nodes[1].node.get_our_node_id(),
1840                                 node_features: nodes[1].node.node_features(),
1841                                 short_channel_id: chan_2_id,
1842                                 channel_features: nodes[1].node.channel_features(),
1843                                 fee_msat: amt_msat / 2,
1844                                 cltv_expiry_delta: 100,
1845                         }],
1846                 ],
1847                 payment_params: Some(route_params.payment_params.clone()),
1848         };
1849         let retry_1_route = Route {
1850                 paths: vec![
1851                         vec![RouteHop {
1852                                 pubkey: nodes[1].node.get_our_node_id(),
1853                                 node_features: nodes[1].node.node_features(),
1854                                 short_channel_id: chan_1_id,
1855                                 channel_features: nodes[1].node.channel_features(),
1856                                 fee_msat: amt_msat / 4,
1857                                 cltv_expiry_delta: 100,
1858                         }],
1859                         vec![RouteHop {
1860                                 pubkey: nodes[1].node.get_our_node_id(),
1861                                 node_features: nodes[1].node.node_features(),
1862                                 short_channel_id: chan_3_id,
1863                                 channel_features: nodes[1].node.channel_features(),
1864                                 fee_msat: amt_msat / 4,
1865                                 cltv_expiry_delta: 100,
1866                         }],
1867                 ],
1868                 payment_params: Some(route_params.payment_params.clone()),
1869         };
1870         let retry_2_route = Route {
1871                 paths: vec![
1872                         vec![RouteHop {
1873                                 pubkey: nodes[1].node.get_our_node_id(),
1874                                 node_features: nodes[1].node.node_features(),
1875                                 short_channel_id: chan_1_id,
1876                                 channel_features: nodes[1].node.channel_features(),
1877                                 fee_msat: amt_msat / 4,
1878                                 cltv_expiry_delta: 100,
1879                         }],
1880                 ],
1881                 payment_params: Some(route_params.payment_params.clone()),
1882         };
1883         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
1884         nodes[0].router.expect_find_route(RouteParameters {
1885                         payment_params: route_params.payment_params.clone(),
1886                         final_value_msat: amt_msat / 2, final_cltv_expiry_delta: TEST_FINAL_CLTV
1887                 }, Ok(retry_1_route));
1888         nodes[0].router.expect_find_route(RouteParameters {
1889                         payment_params: route_params.payment_params.clone(),
1890                         final_value_msat: amt_msat / 4, final_cltv_expiry_delta: TEST_FINAL_CLTV
1891                 }, Ok(retry_2_route));
1892
1893         // Send a payment that will partially fail on send, then partially fail on retry, then succeed.
1894         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap();
1895         let closed_chan_events = nodes[0].node.get_and_clear_pending_events();
1896         assert_eq!(closed_chan_events.len(), 2);
1897         match closed_chan_events[0] {
1898                 Event::ChannelClosed { .. } => {},
1899                 _ => panic!("Unexpected event"),
1900         }
1901         match closed_chan_events[1] {
1902                 Event::ChannelClosed { .. } => {},
1903                 _ => panic!("Unexpected event"),
1904         }
1905
1906         // Pass the first part of the payment along the path.
1907         check_added_monitors!(nodes[0], 5); // three outbound channel updates succeeded, two permanently failed
1908         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1909
1910         // First message is the first update_add, remaining messages are broadcasting channel updates and
1911         // errors for the permfailed channels
1912         assert_eq!(msg_events.len(), 5);
1913         let mut payment_event = SendEvent::from_event(msg_events.remove(0));
1914
1915         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1916         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1917         check_added_monitors!(nodes[1], 1);
1918         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1919
1920         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
1921         check_added_monitors!(nodes[0], 1);
1922         let as_second_htlc_updates = SendEvent::from_node(&nodes[0]);
1923
1924         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
1925         check_added_monitors!(nodes[0], 1);
1926         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1927
1928         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
1929         check_added_monitors!(nodes[1], 1);
1930
1931         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[0]);
1932         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[1]);
1933         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.commitment_msg);
1934         check_added_monitors!(nodes[1], 1);
1935         let (bs_second_raa, bs_second_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1936
1937         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
1938         check_added_monitors!(nodes[0], 1);
1939
1940         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_cs);
1941         check_added_monitors!(nodes[0], 1);
1942         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1943
1944         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
1945         check_added_monitors!(nodes[1], 1);
1946
1947         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
1948         nodes[1].node.process_pending_htlc_forwards();
1949         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, amt_msat);
1950         nodes[1].node.claim_funds(payment_preimage);
1951         expect_payment_claimed!(nodes[1], payment_hash, amt_msat);
1952         let bs_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1953         assert_eq!(bs_claim_update.update_fulfill_htlcs.len(), 1);
1954
1955         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_claim_update.update_fulfill_htlcs[0]);
1956         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_claim_update.commitment_signed);
1957         check_added_monitors!(nodes[0], 1);
1958         let (as_third_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1959
1960         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
1961         check_added_monitors!(nodes[1], 4);
1962         let bs_second_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1963
1964         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
1965         check_added_monitors!(nodes[1], 1);
1966         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1967
1968         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
1969         check_added_monitors!(nodes[0], 1);
1970
1971         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[0]);
1972         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[1]);
1973         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.commitment_signed);
1974         check_added_monitors!(nodes[0], 1);
1975         let (as_fourth_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1976
1977         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_fourth_raa);
1978         check_added_monitors!(nodes[1], 1);
1979
1980         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
1981         check_added_monitors!(nodes[1], 1);
1982         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1983
1984         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
1985         check_added_monitors!(nodes[0], 1);
1986         expect_payment_sent!(nodes[0], payment_preimage);
1987 }
1988
1989 #[test]
1990 fn auto_retry_zero_attempts_send_error() {
1991         let chanmon_cfgs = create_chanmon_cfgs(2);
1992         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1993         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1994         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1995
1996         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1997         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1998
1999         // Marshall data to send the payment
2000         let amt_msat = 20_000;
2001         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2002         #[cfg(feature = "std")]
2003         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2004         #[cfg(not(feature = "std"))]
2005         let payment_expiry_secs = 60 * 60;
2006         let mut invoice_features = InvoiceFeatures::empty();
2007         invoice_features.set_variable_length_onion_required();
2008         invoice_features.set_payment_secret_required();
2009         invoice_features.set_basic_mpp_optional();
2010         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2011                 .with_expiry_time(payment_expiry_secs as u64)
2012                 .with_features(invoice_features);
2013         let route_params = RouteParameters {
2014                 payment_params,
2015                 final_value_msat: amt_msat,
2016                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2017         };
2018
2019         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2020         let err = nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap_err();
2021         if let PaymentSendFailure::AllFailedResendSafe(_) = err {
2022         } else { panic!("Unexpected error"); }
2023         assert_eq!(nodes[0].node.get_and_clear_pending_msg_events().len(), 2); // channel close messages
2024         assert_eq!(nodes[0].node.get_and_clear_pending_events().len(), 1); // channel close event
2025         check_added_monitors!(nodes[0], 2);
2026 }
2027
2028 #[test]
2029 fn fails_paying_after_rejected_by_payee() {
2030         let chanmon_cfgs = create_chanmon_cfgs(2);
2031         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2032         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2033         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2034
2035         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2036
2037         // Marshall data to send the payment
2038         let amt_msat = 20_000;
2039         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2040         #[cfg(feature = "std")]
2041         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2042         #[cfg(not(feature = "std"))]
2043         let payment_expiry_secs = 60 * 60;
2044         let mut invoice_features = InvoiceFeatures::empty();
2045         invoice_features.set_variable_length_onion_required();
2046         invoice_features.set_payment_secret_required();
2047         invoice_features.set_basic_mpp_optional();
2048         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2049                 .with_expiry_time(payment_expiry_secs as u64)
2050                 .with_features(invoice_features);
2051         let route_params = RouteParameters {
2052                 payment_params,
2053                 final_value_msat: amt_msat,
2054                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2055         };
2056
2057         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2058         check_added_monitors!(nodes[0], 1);
2059         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
2060         assert_eq!(events.len(), 1);
2061         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
2062         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2063         check_added_monitors!(nodes[1], 0);
2064         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
2065         expect_pending_htlcs_forwardable!(nodes[1]);
2066         expect_payment_claimable!(&nodes[1], payment_hash, payment_secret, amt_msat);
2067
2068         nodes[1].node.fail_htlc_backwards(&payment_hash);
2069         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash }]);
2070         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, payment_hash);
2071 }
2072
2073 #[test]
2074 fn retry_multi_path_single_failed_payment() {
2075         // Tests that we can/will retry after a single path of an MPP payment failed immediately
2076         let chanmon_cfgs = create_chanmon_cfgs(2);
2077         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2078         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2079         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2080
2081         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2082         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2083
2084         let amt_msat = 100_010_000;
2085
2086         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2087         #[cfg(feature = "std")]
2088         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2089         #[cfg(not(feature = "std"))]
2090         let payment_expiry_secs = 60 * 60;
2091         let mut invoice_features = InvoiceFeatures::empty();
2092         invoice_features.set_variable_length_onion_required();
2093         invoice_features.set_payment_secret_required();
2094         invoice_features.set_basic_mpp_optional();
2095         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2096                 .with_expiry_time(payment_expiry_secs as u64)
2097                 .with_features(invoice_features);
2098         let route_params = RouteParameters {
2099                 payment_params: payment_params.clone(),
2100                 final_value_msat: amt_msat,
2101                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2102         };
2103
2104         let chans = nodes[0].node.list_usable_channels();
2105         let mut route = Route {
2106                 paths: vec![
2107                         vec![RouteHop {
2108                                 pubkey: nodes[1].node.get_our_node_id(),
2109                                 node_features: nodes[1].node.node_features(),
2110                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2111                                 channel_features: nodes[1].node.channel_features(),
2112                                 fee_msat: 10_000,
2113                                 cltv_expiry_delta: 100,
2114                         }],
2115                         vec![RouteHop {
2116                                 pubkey: nodes[1].node.get_our_node_id(),
2117                                 node_features: nodes[1].node.node_features(),
2118                                 short_channel_id: chans[1].short_channel_id.unwrap(),
2119                                 channel_features: nodes[1].node.channel_features(),
2120                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2121                                 cltv_expiry_delta: 100,
2122                         }],
2123                 ],
2124                 payment_params: Some(payment_params),
2125         };
2126         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2127         // On retry, split the payment across both channels.
2128         route.paths[0][0].fee_msat = 50_000_001;
2129         route.paths[1][0].fee_msat = 50_000_000;
2130         nodes[0].router.expect_find_route(RouteParameters {
2131                         payment_params: route.payment_params.clone().unwrap(),
2132                         // Note that the second request here requests the amount we originally failed to send,
2133                         // not the amount remaining on the full payment, which should be changed.
2134                         final_value_msat: 100_000_001, final_cltv_expiry_delta: TEST_FINAL_CLTV
2135                 }, Ok(route.clone()));
2136
2137         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2138         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2139         assert_eq!(htlc_msgs.len(), 2);
2140         check_added_monitors!(nodes[0], 2);
2141 }
2142
2143 #[test]
2144 fn immediate_retry_on_failure() {
2145         // Tests that we can/will retry immediately after a failure
2146         let chanmon_cfgs = create_chanmon_cfgs(2);
2147         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2148         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2149         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2150
2151         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2152         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2153
2154         let amt_msat = 100_000_001;
2155         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2156         #[cfg(feature = "std")]
2157         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2158         #[cfg(not(feature = "std"))]
2159         let payment_expiry_secs = 60 * 60;
2160         let mut invoice_features = InvoiceFeatures::empty();
2161         invoice_features.set_variable_length_onion_required();
2162         invoice_features.set_payment_secret_required();
2163         invoice_features.set_basic_mpp_optional();
2164         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2165                 .with_expiry_time(payment_expiry_secs as u64)
2166                 .with_features(invoice_features);
2167         let route_params = RouteParameters {
2168                 payment_params,
2169                 final_value_msat: amt_msat,
2170                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2171         };
2172
2173         let chans = nodes[0].node.list_usable_channels();
2174         let mut route = Route {
2175                 paths: vec![
2176                         vec![RouteHop {
2177                                 pubkey: nodes[1].node.get_our_node_id(),
2178                                 node_features: nodes[1].node.node_features(),
2179                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2180                                 channel_features: nodes[1].node.channel_features(),
2181                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2182                                 cltv_expiry_delta: 100,
2183                         }],
2184                 ],
2185                 payment_params: Some(PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)),
2186         };
2187         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2188         // On retry, split the payment across both channels.
2189         route.paths.push(route.paths[0].clone());
2190         route.paths[0][0].short_channel_id = chans[1].short_channel_id.unwrap();
2191         route.paths[0][0].fee_msat = 50_000_000;
2192         route.paths[1][0].fee_msat = 50_000_001;
2193         nodes[0].router.expect_find_route(RouteParameters {
2194                         payment_params: route_params.payment_params.clone(),
2195                         final_value_msat: amt_msat, final_cltv_expiry_delta: TEST_FINAL_CLTV
2196                 }, Ok(route.clone()));
2197
2198         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2199         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2200         assert_eq!(htlc_msgs.len(), 2);
2201         check_added_monitors!(nodes[0], 2);
2202 }
2203
2204 #[test]
2205 fn no_extra_retries_on_back_to_back_fail() {
2206         // In a previous release, we had a race where we may exceed the payment retry count if we
2207         // get two failures in a row with the second having `all_paths_failed` set.
2208         // Generally, when we give up trying to retry a payment, we don't know for sure what the
2209         // current state of the ChannelManager event queue is. Specifically, we cannot be sure that
2210         // there are not multiple additional `PaymentPathFailed` or even `PaymentSent` events
2211         // pending which we will see later. Thus, when we previously removed the retry tracking map
2212         // entry after a `all_paths_failed` `PaymentPathFailed` event, we may have dropped the
2213         // retry entry even though more events for the same payment were still pending. This led to
2214         // us retrying a payment again even though we'd already given up on it.
2215         //
2216         // We now have a separate event - `PaymentFailed` which indicates no HTLCs remain and which
2217         // is used to remove the payment retry counter entries instead. This tests for the specific
2218         // excess-retry case while also testing `PaymentFailed` generation.
2219
2220         let chanmon_cfgs = create_chanmon_cfgs(3);
2221         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2222         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2223         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2224
2225         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2226         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2227
2228         let amt_msat = 200_000_000;
2229         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2230         #[cfg(feature = "std")]
2231         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2232         #[cfg(not(feature = "std"))]
2233         let payment_expiry_secs = 60 * 60;
2234         let mut invoice_features = InvoiceFeatures::empty();
2235         invoice_features.set_variable_length_onion_required();
2236         invoice_features.set_payment_secret_required();
2237         invoice_features.set_basic_mpp_optional();
2238         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2239                 .with_expiry_time(payment_expiry_secs as u64)
2240                 .with_features(invoice_features);
2241         let route_params = RouteParameters {
2242                 payment_params,
2243                 final_value_msat: amt_msat,
2244                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2245         };
2246
2247         let mut route = Route {
2248                 paths: vec![
2249                         vec![RouteHop {
2250                                 pubkey: nodes[1].node.get_our_node_id(),
2251                                 node_features: nodes[1].node.node_features(),
2252                                 short_channel_id: chan_1_scid,
2253                                 channel_features: nodes[1].node.channel_features(),
2254                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2255                                 cltv_expiry_delta: 100,
2256                         }, RouteHop {
2257                                 pubkey: nodes[2].node.get_our_node_id(),
2258                                 node_features: nodes[2].node.node_features(),
2259                                 short_channel_id: chan_2_scid,
2260                                 channel_features: nodes[2].node.channel_features(),
2261                                 fee_msat: 100_000_000,
2262                                 cltv_expiry_delta: 100,
2263                         }],
2264                         vec![RouteHop {
2265                                 pubkey: nodes[1].node.get_our_node_id(),
2266                                 node_features: nodes[1].node.node_features(),
2267                                 short_channel_id: chan_1_scid,
2268                                 channel_features: nodes[1].node.channel_features(),
2269                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2270                                 cltv_expiry_delta: 100,
2271                         }, RouteHop {
2272                                 pubkey: nodes[2].node.get_our_node_id(),
2273                                 node_features: nodes[2].node.node_features(),
2274                                 short_channel_id: chan_2_scid,
2275                                 channel_features: nodes[2].node.channel_features(),
2276                                 fee_msat: 100_000_000,
2277                                 cltv_expiry_delta: 100,
2278                         }]
2279                 ],
2280                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2281         };
2282         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2283         let mut second_payment_params = route_params.payment_params.clone();
2284         second_payment_params.previously_failed_channels = vec![chan_2_scid, chan_2_scid];
2285         // On retry, we'll only return one path
2286         route.paths.remove(1);
2287         route.paths[0][1].fee_msat = amt_msat;
2288         nodes[0].router.expect_find_route(RouteParameters {
2289                         payment_params: second_payment_params,
2290                         final_value_msat: amt_msat, final_cltv_expiry_delta: TEST_FINAL_CLTV,
2291                 }, Ok(route.clone()));
2292
2293         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2294         let htlc_updates = SendEvent::from_node(&nodes[0]);
2295         check_added_monitors!(nodes[0], 1);
2296         assert_eq!(htlc_updates.msgs.len(), 1);
2297
2298         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2299         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2300         check_added_monitors!(nodes[1], 1);
2301         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2302
2303         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2304         check_added_monitors!(nodes[0], 1);
2305         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2306
2307         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2308         check_added_monitors!(nodes[0], 1);
2309         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2310
2311         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2312         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2313         check_added_monitors!(nodes[1], 1);
2314         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2315
2316         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2317         check_added_monitors!(nodes[1], 1);
2318         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2319
2320         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2321         check_added_monitors!(nodes[0], 1);
2322
2323         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2324         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2325         check_added_monitors!(nodes[0], 1);
2326         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2327
2328         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2329         check_added_monitors!(nodes[1], 1);
2330         let bs_second_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2331
2332         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2333         check_added_monitors!(nodes[1], 1);
2334         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2335
2336         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.update_fail_htlcs[0]);
2337         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.commitment_signed);
2338         check_added_monitors!(nodes[0], 1);
2339
2340         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2341         check_added_monitors!(nodes[0], 1);
2342         let (as_third_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2343
2344         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2345         check_added_monitors!(nodes[1], 1);
2346         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2347         check_added_monitors!(nodes[1], 1);
2348         let bs_fourth_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2349
2350         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_fourth_raa);
2351         check_added_monitors!(nodes[0], 1);
2352
2353         // At this point A has sent two HTLCs which both failed due to lack of fee. It now has two
2354         // pending `PaymentPathFailed` events, one with `all_paths_failed` unset, and the second
2355         // with it set.
2356         //
2357         // Previously, we retried payments in an event consumer, which would retry each
2358         // `PaymentPathFailed` individually. In that setup, we had retried the payment in response to
2359         // the first `PaymentPathFailed`, then seen the second `PaymentPathFailed` with
2360         // `all_paths_failed` set and assumed the payment was completely failed. We ultimately fixed it
2361         // by adding the `PaymentFailed` event.
2362         //
2363         // Because we now retry payments as a batch, we simply return a single-path route in the
2364         // second, batched, request, have that fail, then complete the payment via `abandon_payment`.
2365         let mut events = nodes[0].node.get_and_clear_pending_events();
2366         assert_eq!(events.len(), 4);
2367         match events[0] {
2368                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2369                         assert_eq!(payment_hash, ev_payment_hash);
2370                         assert_eq!(payment_failed_permanently, false);
2371                 },
2372                 _ => panic!("Unexpected event"),
2373         }
2374         match events[1] {
2375                 Event::PendingHTLCsForwardable { .. } => {},
2376                 _ => panic!("Unexpected event"),
2377         }
2378         match events[2] {
2379                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2380                         assert_eq!(payment_hash, ev_payment_hash);
2381                         assert_eq!(payment_failed_permanently, false);
2382                 },
2383                 _ => panic!("Unexpected event"),
2384         }
2385         match events[3] {
2386                 Event::PendingHTLCsForwardable { .. } => {},
2387                 _ => panic!("Unexpected event"),
2388         }
2389
2390         nodes[0].node.process_pending_htlc_forwards();
2391         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2392         check_added_monitors!(nodes[0], 1);
2393
2394         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2395         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2396         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2397         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2398         commitment_signed_dance!(nodes[0], nodes[1], &bs_fail_update.commitment_signed, false, true);
2399
2400         let mut events = nodes[0].node.get_and_clear_pending_events();
2401         assert_eq!(events.len(), 1);
2402         match events[0] {
2403                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2404                         assert_eq!(payment_hash, ev_payment_hash);
2405                         assert_eq!(payment_failed_permanently, false);
2406                 },
2407                 _ => panic!("Unexpected event"),
2408         }
2409         nodes[0].node.abandon_payment(PaymentId(payment_hash.0));
2410         events = nodes[0].node.get_and_clear_pending_events();
2411         assert_eq!(events.len(), 1);
2412         match events[0] {
2413                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
2414                         assert_eq!(payment_hash, *ev_payment_hash);
2415                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2416                 },
2417                 _ => panic!("Unexpected event"),
2418         }
2419 }
2420
2421 #[test]
2422 fn test_simple_partial_retry() {
2423         // In the first version of the in-`ChannelManager` payment retries, retries were sent for the
2424         // full amount of the payment, rather than only the missing amount. Here we simply test for
2425         // this by sending a payment with two parts, failing one, and retrying the second. Note that
2426         // `TestRouter` will check that the `RouteParameters` (which contain the amount) matches the
2427         // request.
2428         let chanmon_cfgs = create_chanmon_cfgs(3);
2429         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2430         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2431         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2432
2433         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2434         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2435
2436         let amt_msat = 200_000_000;
2437         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
2438         #[cfg(feature = "std")]
2439         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2440         #[cfg(not(feature = "std"))]
2441         let payment_expiry_secs = 60 * 60;
2442         let mut invoice_features = InvoiceFeatures::empty();
2443         invoice_features.set_variable_length_onion_required();
2444         invoice_features.set_payment_secret_required();
2445         invoice_features.set_basic_mpp_optional();
2446         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2447                 .with_expiry_time(payment_expiry_secs as u64)
2448                 .with_features(invoice_features);
2449         let route_params = RouteParameters {
2450                 payment_params,
2451                 final_value_msat: amt_msat,
2452                 final_cltv_expiry_delta: TEST_FINAL_CLTV,
2453         };
2454
2455         let mut route = Route {
2456                 paths: vec![
2457                         vec![RouteHop {
2458                                 pubkey: nodes[1].node.get_our_node_id(),
2459                                 node_features: nodes[1].node.node_features(),
2460                                 short_channel_id: chan_1_scid,
2461                                 channel_features: nodes[1].node.channel_features(),
2462                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2463                                 cltv_expiry_delta: 100,
2464                         }, RouteHop {
2465                                 pubkey: nodes[2].node.get_our_node_id(),
2466                                 node_features: nodes[2].node.node_features(),
2467                                 short_channel_id: chan_2_scid,
2468                                 channel_features: nodes[2].node.channel_features(),
2469                                 fee_msat: 100_000_000,
2470                                 cltv_expiry_delta: 100,
2471                         }],
2472                         vec![RouteHop {
2473                                 pubkey: nodes[1].node.get_our_node_id(),
2474                                 node_features: nodes[1].node.node_features(),
2475                                 short_channel_id: chan_1_scid,
2476                                 channel_features: nodes[1].node.channel_features(),
2477                                 fee_msat: 100_000,
2478                                 cltv_expiry_delta: 100,
2479                         }, RouteHop {
2480                                 pubkey: nodes[2].node.get_our_node_id(),
2481                                 node_features: nodes[2].node.node_features(),
2482                                 short_channel_id: chan_2_scid,
2483                                 channel_features: nodes[2].node.channel_features(),
2484                                 fee_msat: 100_000_000,
2485                                 cltv_expiry_delta: 100,
2486                         }]
2487                 ],
2488                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2489         };
2490         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2491         let mut second_payment_params = route_params.payment_params.clone();
2492         second_payment_params.previously_failed_channels = vec![chan_2_scid];
2493         // On retry, we'll only be asked for one path (or 100k sats)
2494         route.paths.remove(0);
2495         nodes[0].router.expect_find_route(RouteParameters {
2496                         payment_params: second_payment_params,
2497                         final_value_msat: amt_msat / 2, final_cltv_expiry_delta: TEST_FINAL_CLTV,
2498                 }, Ok(route.clone()));
2499
2500         nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2501         let htlc_updates = SendEvent::from_node(&nodes[0]);
2502         check_added_monitors!(nodes[0], 1);
2503         assert_eq!(htlc_updates.msgs.len(), 1);
2504
2505         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2506         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2507         check_added_monitors!(nodes[1], 1);
2508         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2509
2510         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2511         check_added_monitors!(nodes[0], 1);
2512         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2513
2514         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2515         check_added_monitors!(nodes[0], 1);
2516         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2517
2518         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2519         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2520         check_added_monitors!(nodes[1], 1);
2521         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2522
2523         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2524         check_added_monitors!(nodes[1], 1);
2525         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2526
2527         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2528         check_added_monitors!(nodes[0], 1);
2529
2530         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2531         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2532         check_added_monitors!(nodes[0], 1);
2533         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2534
2535         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2536         check_added_monitors!(nodes[1], 1);
2537
2538         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2539         check_added_monitors!(nodes[1], 1);
2540
2541         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2542
2543         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2544         check_added_monitors!(nodes[0], 1);
2545
2546         let mut events = nodes[0].node.get_and_clear_pending_events();
2547         assert_eq!(events.len(), 2);
2548         match events[0] {
2549                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2550                         assert_eq!(payment_hash, ev_payment_hash);
2551                         assert_eq!(payment_failed_permanently, false);
2552                 },
2553                 _ => panic!("Unexpected event"),
2554         }
2555         match events[1] {
2556                 Event::PendingHTLCsForwardable { .. } => {},
2557                 _ => panic!("Unexpected event"),
2558         }
2559
2560         nodes[0].node.process_pending_htlc_forwards();
2561         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2562         check_added_monitors!(nodes[0], 1);
2563
2564         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2565         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2566
2567         expect_pending_htlcs_forwardable!(nodes[1]);
2568         check_added_monitors!(nodes[1], 1);
2569
2570         let bs_forward_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
2571         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[0]);
2572         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[1]);
2573         commitment_signed_dance!(nodes[2], nodes[1], &bs_forward_update.commitment_signed, false);
2574
2575         expect_pending_htlcs_forwardable!(nodes[2]);
2576         expect_payment_claimable!(nodes[2], payment_hash, payment_secret, amt_msat);
2577 }