_ Git - rust-lightning/blob - lightning/src/ln/payment_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Tests that test the payment retry logic in ChannelManager, including various edge-cases around
  11 //! serialization ordering between ChannelManager/ChannelMonitors and ensuring we can still retry
  12 //! payments thereafter.
  13
  14 use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Listen, Watch};
  15 use crate::chain::channelmonitor::{ANTI_REORG_DELAY, HTLC_FAIL_BACK_BUFFER, LATENCY_GRACE_PERIOD_BLOCKS};
  16 use crate::sign::EntropySource;
  17 use crate::chain::transaction::OutPoint;
  18 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentFailureReason};
  19 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
  20 use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, ChannelManager, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS, RecentPaymentDetails, RecipientOnionFields, HTLCForwardInfo, PendingHTLCRouting, PendingAddHTLCInfo};
  21 use crate::ln::features::Bolt11InvoiceFeatures;
  22 use crate::ln::{msgs, PaymentSecret, PaymentPreimage};
  23 use crate::ln::msgs::ChannelMessageHandler;
  24 use crate::ln::outbound_payment::Retry;
  25 use crate::routing::gossip::{EffectiveCapacity, RoutingFees};
  26 use crate::routing::router::{get_route, Path, PaymentParameters, Route, Router, RouteHint, RouteHintHop, RouteHop, RouteParameters, find_route};
  27 use crate::routing::scoring::ChannelUsage;
  28 use crate::util::test_utils;
  29 use crate::util::errors::APIError;
  30 use crate::util::ser::Writeable;
  31 use crate::util::string::UntrustedString;
  32
  33 use bitcoin::network::constants::Network;
  34
  35 use crate::prelude::*;
  36
  37 use crate::ln::functional_test_utils::*;
  38 use crate::routing::gossip::NodeId;
  39 #[cfg(feature = "std")]
  40 use {
  41         crate::util::time::tests::SinceEpoch,
  42         std::time::{SystemTime, Instant, Duration}
  43 };
  44
  45 #[test]
  46 fn mpp_failure() {
  47         let chanmon_cfgs = create_chanmon_cfgs(4);
  48         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
  49         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
  50         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
  51
  52         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
  53         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
  54         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
  55         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3).0.contents.short_channel_id;
  56
  57         let (mut route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
  58         let path = route.paths[0].clone();
  59         route.paths.push(path);
  60         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
  61         route.paths[0].hops[0].short_channel_id = chan_1_id;
  62         route.paths[0].hops[1].short_channel_id = chan_3_id;
  63         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
  64         route.paths[1].hops[0].short_channel_id = chan_2_id;
  65         route.paths[1].hops[1].short_channel_id = chan_4_id;
  66         send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
  67         fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
  68 }
  69
  70 #[test]
  71 fn mpp_retry() {
  72         let chanmon_cfgs = create_chanmon_cfgs(4);
  73         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
  74         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
  75         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
  76
  77         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
  78         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
  79         let (chan_3_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
  80         let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes(&nodes, 3, 2);
  81         // Rebalance
  82         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
  83
  84         let amt_msat = 1_000_000;
  85         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], amt_msat);
  86         let path = route.paths[0].clone();
  87         route.paths.push(path);
  88         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
  89         route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
  90         route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
  91         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
  92         route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
  93         route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
  94
  95         // Initiate the MPP payment.
  96         let payment_id = PaymentId(payment_hash.0);
  97         let mut route_params = RouteParameters {
  98                 payment_params: route.payment_params.clone().unwrap(),
  99                 final_value_msat: amt_msat,
 100         };
 101
 102         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
 103         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 104                 payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
 105         check_added_monitors!(nodes[0], 2); // one monitor per path
 106         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 107         assert_eq!(events.len(), 2);
 108
 109         // Pass half of the payment along the success path.
 110         let success_path_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 111         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), success_path_msgs, false, None);
 112
 113         // Add the HTLC along the first hop.
 114         let fail_path_msgs_1 = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 115         let (update_add, commitment_signed) = match fail_path_msgs_1 {
 116                 MessageSendEvent::UpdateHTLCs { node_id: _, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 117                         assert_eq!(update_add_htlcs.len(), 1);
 118                         assert!(update_fail_htlcs.is_empty());
 119                         assert!(update_fulfill_htlcs.is_empty());
 120                         assert!(update_fail_malformed_htlcs.is_empty());
 121                         assert!(update_fee.is_none());
 122                         (update_add_htlcs[0].clone(), commitment_signed.clone())
 123                 },
 124                 _ => panic!("Unexpected event"),
 125         };
 126         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
 127         commitment_signed_dance!(nodes[2], nodes[0], commitment_signed, false);
 128
 129         // Attempt to forward the payment and complete the 2nd path's failure.
 130         expect_pending_htlcs_forwardable!(&nodes[2]);
 131         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id }]);
 132         let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 133         assert!(htlc_updates.update_add_htlcs.is_empty());
 134         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 135         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 136         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 137         check_added_monitors!(nodes[2], 1);
 138         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 139         commitment_signed_dance!(nodes[0], nodes[2], htlc_updates.commitment_signed, false);
 140         let mut events = nodes[0].node.get_and_clear_pending_events();
 141         match events[1] {
 142                 Event::PendingHTLCsForwardable { .. } => {},
 143                 _ => panic!("Unexpected event")
 144         }
 145         events.remove(1);
 146         expect_payment_failed_conditions_event(events, payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 147
 148         // Rebalance the channel so the second half of the payment can succeed.
 149         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 150
 151         // Retry the second half of the payment and make sure it succeeds.
 152         route.paths.remove(0);
 153         route_params.final_value_msat = 1_000_000;
 154         route_params.payment_params.previously_failed_channels.push(chan_4_update.contents.short_channel_id);
 155         nodes[0].router.expect_find_route(route_params, Ok(route));
 156         nodes[0].node.process_pending_htlc_forwards();
 157         check_added_monitors!(nodes[0], 1);
 158         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 159         assert_eq!(events.len(), 1);
 160         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 161         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 162 }
 163
 164 fn do_mpp_receive_timeout(send_partial_mpp: bool) {
 165         let chanmon_cfgs = create_chanmon_cfgs(4);
 166         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 167         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 168         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 169
 170         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 171         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 172         let (chan_3_update, _, chan_3_id, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 173         let (chan_4_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 174
 175         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 100_000);
 176         let path = route.paths[0].clone();
 177         route.paths.push(path);
 178         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
 179         route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
 180         route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
 181         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
 182         route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
 183         route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
 184
 185         // Initiate the MPP payment.
 186         nodes[0].node.send_payment_with_route(&route, payment_hash,
 187                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 188         check_added_monitors!(nodes[0], 2); // one monitor per path
 189         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 190         assert_eq!(events.len(), 2);
 191
 192         // Pass half of the payment along the first path.
 193         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 194         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_1_msgs, false, None);
 195
 196         if send_partial_mpp {
 197                 // Time out the partial MPP
 198                 for _ in 0..MPP_TIMEOUT_TICKS {
 199                         nodes[3].node.timer_tick_occurred();
 200                 }
 201
 202                 // Failed HTLC from node 3 -> 1
 203                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 204                 let htlc_fail_updates_3_1 = get_htlc_update_msgs!(nodes[3], nodes[1].node.get_our_node_id());
 205                 assert_eq!(htlc_fail_updates_3_1.update_fail_htlcs.len(), 1);
 206                 nodes[1].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &htlc_fail_updates_3_1.update_fail_htlcs[0]);
 207                 check_added_monitors!(nodes[3], 1);
 208                 commitment_signed_dance!(nodes[1], nodes[3], htlc_fail_updates_3_1.commitment_signed, false);
 209
 210                 // Failed HTLC from node 1 -> 0
 211                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_3_id }]);
 212                 let htlc_fail_updates_1_0 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 213                 assert_eq!(htlc_fail_updates_1_0.update_fail_htlcs.len(), 1);
 214                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates_1_0.update_fail_htlcs[0]);
 215                 check_added_monitors!(nodes[1], 1);
 216                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates_1_0.commitment_signed, false);
 217
 218                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain().expected_htlc_error_data(23, &[][..]));
 219         } else {
 220                 // Pass half of the payment along the second path.
 221                 let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 222                 pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_2_msgs, true, None);
 223
 224                 // Even after MPP_TIMEOUT_TICKS we should not timeout the MPP if we have all the parts
 225                 for _ in 0..MPP_TIMEOUT_TICKS {
 226                         nodes[3].node.timer_tick_occurred();
 227                 }
 228
 229                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 230         }
 231 }
 232
 233 #[test]
 234 fn mpp_receive_timeout() {
 235         do_mpp_receive_timeout(true);
 236         do_mpp_receive_timeout(false);
 237 }
 238
 239 #[test]
 240 fn test_keysend_payments_to_public_node() {
 241         let chanmon_cfgs = create_chanmon_cfgs(2);
 242         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 243         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 244         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 245
 246         let _chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 10001);
 247         let network_graph = nodes[0].network_graph.clone();
 248         let payer_pubkey = nodes[0].node.get_our_node_id();
 249         let payee_pubkey = nodes[1].node.get_our_node_id();
 250         let route_params = RouteParameters {
 251                 payment_params: PaymentParameters::for_keysend(payee_pubkey, 40, false),
 252                 final_value_msat: 10000,
 253         };
 254         let scorer = test_utils::TestScorer::new();
 255         let random_seed_bytes = chanmon_cfgs[1].keys_manager.get_secure_random_bytes();
 256         let route = find_route(&payer_pubkey, &route_params, &network_graph, None, nodes[0].logger, &scorer, &(), &random_seed_bytes).unwrap();
 257
 258         let test_preimage = PaymentPreimage([42; 32]);
 259         let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage),
 260                 RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0)).unwrap();
 261         check_added_monitors!(nodes[0], 1);
 262         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 263         assert_eq!(events.len(), 1);
 264         let event = events.pop().unwrap();
 265         let path = vec![&nodes[1]];
 266         pass_along_path(&nodes[0], &path, 10000, payment_hash, None, event, true, Some(test_preimage));
 267         claim_payment(&nodes[0], &path, test_preimage);
 268 }
 269
 270 #[test]
 271 fn test_keysend_payments_to_private_node() {
 272         let chanmon_cfgs = create_chanmon_cfgs(2);
 273         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 274         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 275         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 276
 277         let payer_pubkey = nodes[0].node.get_our_node_id();
 278         let payee_pubkey = nodes[1].node.get_our_node_id();
 279
 280         let _chan = create_chan_between_nodes(&nodes[0], &nodes[1]);
 281         let route_params = RouteParameters {
 282                 payment_params: PaymentParameters::for_keysend(payee_pubkey, 40, false),
 283                 final_value_msat: 10000,
 284         };
 285         let network_graph = nodes[0].network_graph.clone();
 286         let first_hops = nodes[0].node.list_usable_channels();
 287         let scorer = test_utils::TestScorer::new();
 288         let random_seed_bytes = chanmon_cfgs[1].keys_manager.get_secure_random_bytes();
 289         let route = find_route(
 290                 &payer_pubkey, &route_params, &network_graph, Some(&first_hops.iter().collect::<Vec<_>>()),
 291                 nodes[0].logger, &scorer, &(), &random_seed_bytes
 292         ).unwrap();
 293
 294         let test_preimage = PaymentPreimage([42; 32]);
 295         let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage),
 296                 RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0)).unwrap();
 297         check_added_monitors!(nodes[0], 1);
 298         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 299         assert_eq!(events.len(), 1);
 300         let event = events.pop().unwrap();
 301         let path = vec![&nodes[1]];
 302         pass_along_path(&nodes[0], &path, 10000, payment_hash, None, event, true, Some(test_preimage));
 303         claim_payment(&nodes[0], &path, test_preimage);
 304 }
 305
 306 #[test]
 307 fn test_mpp_keysend() {
 308         let mut mpp_keysend_config = test_default_channel_config();
 309         mpp_keysend_config.accept_mpp_keysend = true;
 310         let chanmon_cfgs = create_chanmon_cfgs(4);
 311         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 312         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, Some(mpp_keysend_config)]);
 313         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 314
 315         create_announced_chan_between_nodes(&nodes, 0, 1);
 316         create_announced_chan_between_nodes(&nodes, 0, 2);
 317         create_announced_chan_between_nodes(&nodes, 1, 3);
 318         create_announced_chan_between_nodes(&nodes, 2, 3);
 319         let network_graph = nodes[0].network_graph.clone();
 320
 321         let payer_pubkey = nodes[0].node.get_our_node_id();
 322         let payee_pubkey = nodes[3].node.get_our_node_id();
 323         let recv_value = 15_000_000;
 324         let route_params = RouteParameters {
 325                 payment_params: PaymentParameters::for_keysend(payee_pubkey, 40, true),
 326                 final_value_msat: recv_value,
 327         };
 328         let scorer = test_utils::TestScorer::new();
 329         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
 330         let route = find_route(&payer_pubkey, &route_params, &network_graph, None, nodes[0].logger,
 331                 &scorer, &(), &random_seed_bytes).unwrap();
 332
 333         let payment_preimage = PaymentPreimage([42; 32]);
 334         let payment_secret = PaymentSecret(payment_preimage.0);
 335         let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage),
 336                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_preimage.0)).unwrap();
 337         check_added_monitors!(nodes[0], 2);
 338
 339         let expected_route: &[&[&Node]] = &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]];
 340         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 341         assert_eq!(events.len(), 2);
 342
 343         let ev = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 344         pass_along_path(&nodes[0], expected_route[0], recv_value, payment_hash.clone(),
 345                 Some(payment_secret), ev.clone(), false, Some(payment_preimage));
 346
 347         let ev = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 348         pass_along_path(&nodes[0], expected_route[1], recv_value, payment_hash.clone(),
 349                 Some(payment_secret), ev.clone(), true, Some(payment_preimage));
 350         claim_payment_along_route(&nodes[0], expected_route, false, payment_preimage);
 351 }
 352
 353 #[test]
 354 fn test_reject_mpp_keysend_htlc() {
 355         // This test enforces that we reject MPP keysend HTLCs if our config states we don't support
 356         // MPP keysend. When receiving a payment, if we don't support MPP keysend we'll reject the
 357         // payment if it's keysend and has a payment secret, never reaching our payment validation
 358         // logic. To check that we enforce rejecting MPP keysends in our payment logic, here we send
 359         // keysend payments without payment secrets, then modify them by adding payment secrets in the
 360         // final node in between receiving the HTLCs and actually processing them.
 361         let mut reject_mpp_keysend_cfg = test_default_channel_config();
 362         reject_mpp_keysend_cfg.accept_mpp_keysend = false;
 363
 364         let chanmon_cfgs = create_chanmon_cfgs(4);
 365         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 366         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, Some(reject_mpp_keysend_cfg)]);
 367         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 368         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
 369         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
 370         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
 371         let (update_a, _, chan_4_channel_id, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 372         let chan_4_id = update_a.contents.short_channel_id;
 373         let amount = 40_000;
 374         let (mut route, payment_hash, payment_preimage, _) = get_route_and_payment_hash!(nodes[0], nodes[3], amount);
 375
 376         // Pay along nodes[1]
 377         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
 378         route.paths[0].hops[0].short_channel_id = chan_1_id;
 379         route.paths[0].hops[1].short_channel_id = chan_3_id;
 380
 381         let payment_id_0 = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 382         nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), RecipientOnionFields::spontaneous_empty(), payment_id_0).unwrap();
 383         check_added_monitors!(nodes[0], 1);
 384
 385         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 386         let update_add_0 = update_0.update_add_htlcs[0].clone();
 387         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_0);
 388         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
 389         expect_pending_htlcs_forwardable!(nodes[1]);
 390
 391         check_added_monitors!(&nodes[1], 1);
 392         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[3].node.get_our_node_id());
 393         let update_add_1 = update_1.update_add_htlcs[0].clone();
 394         nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &update_add_1);
 395         commitment_signed_dance!(nodes[3], nodes[1], update_1.commitment_signed, false, true);
 396
 397         assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
 398         for (_, pending_forwards) in nodes[3].node.forward_htlcs.lock().unwrap().iter_mut() {
 399                 for f in pending_forwards.iter_mut() {
 400                         match f {
 401                                 &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) => {
 402                                         match forward_info.routing {
 403                                                 PendingHTLCRouting::ReceiveKeysend { ref mut payment_data, .. } => {
 404                                                         *payment_data = Some(msgs::FinalOnionHopData {
 405                                                                 payment_secret: PaymentSecret([42; 32]),
 406                                                                 total_msat: amount * 2,
 407                                                         });
 408                                                 },
 409                                                 _ => panic!("Expected PendingHTLCRouting::ReceiveKeysend"),
 410                                         }
 411                                 },
 412                                 _ => {},
 413                         }
 414                 }
 415         }
 416         expect_pending_htlcs_forwardable!(nodes[3]);
 417
 418         // Pay along nodes[2]
 419         route.paths[0].hops[0].pubkey = nodes[2].node.get_our_node_id();
 420         route.paths[0].hops[0].short_channel_id = chan_2_id;
 421         route.paths[0].hops[1].short_channel_id = chan_4_id;
 422
 423         let payment_id_1 = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 424         nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), RecipientOnionFields::spontaneous_empty(), payment_id_1).unwrap();
 425         check_added_monitors!(nodes[0], 1);
 426
 427         let update_2 = get_htlc_update_msgs!(nodes[0], nodes[2].node.get_our_node_id());
 428         let update_add_2 = update_2.update_add_htlcs[0].clone();
 429         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_2);
 430         commitment_signed_dance!(nodes[2], nodes[0], &update_2.commitment_signed, false, true);
 431         expect_pending_htlcs_forwardable!(nodes[2]);
 432
 433         check_added_monitors!(&nodes[2], 1);
 434         let update_3 = get_htlc_update_msgs!(nodes[2], nodes[3].node.get_our_node_id());
 435         let update_add_3 = update_3.update_add_htlcs[0].clone();
 436         nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &update_add_3);
 437         commitment_signed_dance!(nodes[3], nodes[2], update_3.commitment_signed, false, true);
 438
 439         assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
 440         for (_, pending_forwards) in nodes[3].node.forward_htlcs.lock().unwrap().iter_mut() {
 441                 for f in pending_forwards.iter_mut() {
 442                         match f {
 443                                 &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) => {
 444                                         match forward_info.routing {
 445                                                 PendingHTLCRouting::ReceiveKeysend { ref mut payment_data, .. } => {
 446                                                         *payment_data = Some(msgs::FinalOnionHopData {
 447                                                                 payment_secret: PaymentSecret([42; 32]),
 448                                                                 total_msat: amount * 2,
 449                                                         });
 450                                                 },
 451                                                 _ => panic!("Expected PendingHTLCRouting::ReceiveKeysend"),
 452                                         }
 453                                 },
 454                                 _ => {},
 455                         }
 456                 }
 457         }
 458         expect_pending_htlcs_forwardable!(nodes[3]);
 459         check_added_monitors!(nodes[3], 1);
 460
 461         // Fail back along nodes[2]
 462         let update_fail_0 = get_htlc_update_msgs!(&nodes[3], &nodes[2].node.get_our_node_id());
 463         nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &update_fail_0.update_fail_htlcs[0]);
 464         commitment_signed_dance!(nodes[2], nodes[3], update_fail_0.commitment_signed, false);
 465         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_channel_id }]);
 466         check_added_monitors!(nodes[2], 1);
 467
 468         let update_fail_1 = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 469         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_1.update_fail_htlcs[0]);
 470         commitment_signed_dance!(nodes[0], nodes[2], update_fail_1.commitment_signed, false);
 471
 472         expect_payment_failed_conditions(&nodes[0], payment_hash, true, PaymentFailedConditions::new());
 473         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 474 }
 475
 476
 477 #[test]
 478 fn no_pending_leak_on_initial_send_failure() {
 479         // In an earlier version of our payment tracking, we'd have a retry entry even when the initial
 480         // HTLC for payment failed to send due to local channel errors (e.g. peer disconnected). In this
 481         // case, the user wouldn't have a PaymentId to retry the payment with, but we'd think we have a
 482         // pending payment forever and never time it out.
 483         // Here we test exactly that - retrying a payment when a peer was disconnected on the first
 484         // try, and then check that no pending payment is being tracked.
 485         let chanmon_cfgs = create_chanmon_cfgs(2);
 486         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 487         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 488         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 489
 490         create_announced_chan_between_nodes(&nodes, 0, 1);
 491
 492         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 493
 494         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 495         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 496
 497         unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash,
 498                         RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
 499                 ), true, APIError::ChannelUnavailable { ref err },
 500                 assert_eq!(err, "Peer for first hop currently disconnected"));
 501
 502         assert!(!nodes[0].node.has_pending_payments());
 503 }
 504
 505 fn do_retry_with_no_persist(confirm_before_reload: bool) {
 506         // If we send a pending payment and `send_payment` returns success, we should always either
 507         // return a payment failure event or a payment success event, and on failure the payment should
 508         // be retryable.
 509         //
 510         // In order to do so when the ChannelManager isn't immediately persisted (which is normal - its
 511         // always persisted asynchronously), the ChannelManager has to reload some payment data from
 512         // ChannelMonitor(s) in some cases. This tests that reloading.
 513         //
 514         // `confirm_before_reload` confirms the channel-closing commitment transaction on-chain prior
 515         // to reloading the ChannelManager, increasing test coverage in ChannelMonitor HTLC tracking
 516         // which has separate codepaths for "commitment transaction already confirmed" and not.
 517         let chanmon_cfgs = create_chanmon_cfgs(3);
 518         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 519         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 520         let persister: test_utils::TestPersister;
 521         let new_chain_monitor: test_utils::TestChainMonitor;
 522         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 523         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 524
 525         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 526         let (_, _, chan_id_2, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
 527
 528         // Serialize the ChannelManager prior to sending payments
 529         let nodes_0_serialized = nodes[0].node.encode();
 530
 531         // Send two payments - one which will get to nodes[2] and will be claimed, one which we'll time
 532         // out and retry.
 533         let amt_msat = 1_000_000;
 534         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
 535         let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000);
 536         let route_params = RouteParameters {
 537                 payment_params: route.payment_params.clone().unwrap(),
 538                 final_value_msat: amt_msat,
 539         };
 540         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 541                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
 542         check_added_monitors!(nodes[0], 1);
 543
 544         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 545         assert_eq!(events.len(), 1);
 546         let payment_event = SendEvent::from_event(events.pop().unwrap());
 547         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 548
 549         // We relay the payment to nodes[1] while its disconnected from nodes[2], causing the payment
 550         // to be returned immediately to nodes[0], without having nodes[2] fail the inbound payment
 551         // which would prevent retry.
 552         nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id());
 553         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 554
 555         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 556         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true);
 557         // nodes[1] now immediately fails the HTLC as the next-hop channel is disconnected
 558         let _ = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 559
 560         reconnect_nodes(ReconnectArgs::new(&nodes[1], &nodes[2]));
 561
 562         let as_commitment_tx = get_local_commitment_txn!(nodes[0], chan_id)[0].clone();
 563         if confirm_before_reload {
 564                 mine_transaction(&nodes[0], &as_commitment_tx);
 565                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 566         }
 567
 568         // The ChannelMonitor should always be the latest version, as we're required to persist it
 569         // during the `commitment_signed_dance!()`.
 570         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 571         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 572
 573         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 574         // force-close the channel.
 575         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 576         assert!(nodes[0].node.list_channels().is_empty());
 577         assert!(nodes[0].node.has_pending_payments());
 578         nodes[0].node.timer_tick_occurred();
 579         if !confirm_before_reload {
 580                 let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 581                 assert_eq!(as_broadcasted_txn.len(), 1);
 582                 assert_eq!(as_broadcasted_txn[0].txid(), as_commitment_tx.txid());
 583         } else {
 584                 assert!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
 585         }
 586         check_added_monitors!(nodes[0], 1);
 587
 588         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 589         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 590                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 591         }, true).unwrap();
 592         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 593
 594         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 595         // error, as the channel has hit the chain.
 596         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 597                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 598         }, false).unwrap();
 599         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 600         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 601         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 602         assert_eq!(as_err.len(), 1);
 603         match as_err[0] {
 604                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 605                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 606                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 607                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) });
 608                         check_added_monitors!(nodes[1], 1);
 609                         assert_eq!(nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 610                 },
 611                 _ => panic!("Unexpected event"),
 612         }
 613         check_closed_broadcast!(nodes[1], false);
 614
 615         // Now claim the first payment, which should allow nodes[1] to claim the payment on-chain when
 616         // we close in a moment.
 617         nodes[2].node.claim_funds(payment_preimage_1);
 618         check_added_monitors!(nodes[2], 1);
 619         expect_payment_claimed!(nodes[2], payment_hash_1, 1_000_000);
 620
 621         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 622         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 623         check_added_monitors!(nodes[1], 1);
 624         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 625         expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], None, false, false);
 626
 627         if confirm_before_reload {
 628                 let best_block = nodes[0].blocks.lock().unwrap().last().unwrap().clone();
 629                 nodes[0].node.best_block_updated(&best_block.0.header, best_block.1);
 630         }
 631
 632         // Create a new channel on which to retry the payment before we fail the payment via the
 633         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 634         // connecting several blocks while creating the channel (implying time has passed).
 635         create_announced_chan_between_nodes(&nodes, 0, 1);
 636         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 637
 638         mine_transaction(&nodes[1], &as_commitment_tx);
 639         let bs_htlc_claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 640         assert_eq!(bs_htlc_claim_txn.len(), 1);
 641         check_spends!(bs_htlc_claim_txn[0], as_commitment_tx);
 642
 643         if !confirm_before_reload {
 644                 mine_transaction(&nodes[0], &as_commitment_tx);
 645         }
 646         mine_transaction(&nodes[0], &bs_htlc_claim_txn[0]);
 647         expect_payment_sent!(nodes[0], payment_preimage_1);
 648         connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
 649         let (first_htlc_timeout_tx, second_htlc_timeout_tx) = {
 650                 let mut txn = nodes[0].tx_broadcaster.unique_txn_broadcast();
 651                 assert_eq!(txn.len(), 2);
 652                 (txn.remove(0), txn.remove(0))
 653         };
 654         check_spends!(first_htlc_timeout_tx, as_commitment_tx);
 655         check_spends!(second_htlc_timeout_tx, as_commitment_tx);
 656         if first_htlc_timeout_tx.input[0].previous_output == bs_htlc_claim_txn[0].input[0].previous_output {
 657                 confirm_transaction(&nodes[0], &second_htlc_timeout_tx);
 658         } else {
 659                 confirm_transaction(&nodes[0], &first_htlc_timeout_tx);
 660         }
 661         nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 662         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new());
 663
 664         // Finally, retry the payment (which was reloaded from the ChannelMonitor when nodes[0] was
 665         // reloaded) via a route over the new channel, which work without issue and eventually be
 666         // received and claimed at the recipient just like any other payment.
 667         let (mut new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 668
 669         // Update the fee on the middle hop to ensure PaymentSent events have the correct (retried) fee
 670         // and not the original fee. We also update node[1]'s relevant config as
 671         // do_claim_payment_along_route expects us to never overpay.
 672         {
 673                 let per_peer_state = nodes[1].node.per_peer_state.read().unwrap();
 674                 let mut peer_state = per_peer_state.get(&nodes[2].node.get_our_node_id())
 675                         .unwrap().lock().unwrap();
 676                 let mut channel = peer_state.channel_by_id.get_mut(&chan_id_2).unwrap();
 677                 let mut new_config = channel.context.config();
 678                 new_config.forwarding_fee_base_msat += 100_000;
 679                 channel.context.update_config(&new_config);
 680                 new_route.paths[0].hops[0].fee_msat += 100_000;
 681         }
 682
 683         // Force expiration of the channel's previous config.
 684         for _ in 0..EXPIRE_PREV_CONFIG_TICKS {
 685                 nodes[1].node.timer_tick_occurred();
 686         }
 687
 688         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash, // Shouldn't be allowed to retry a fulfilled payment
 689                 RecipientOnionFields::secret_only(payment_secret), payment_id_1).is_err());
 690         nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 691                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 692         check_added_monitors!(nodes[0], 1);
 693         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 694         assert_eq!(events.len(), 1);
 695         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 696         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
 697         expect_payment_sent!(nodes[0], payment_preimage, Some(new_route.paths[0].hops[0].fee_msat));
 698 }
 699
 700 #[test]
 701 fn retry_with_no_persist() {
 702         do_retry_with_no_persist(true);
 703         do_retry_with_no_persist(false);
 704 }
 705
 706 fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 707         // Test that an off-chain completed payment is not retryable on restart. This was previously
 708         // broken for dust payments, but we test for both dust and non-dust payments.
 709         //
 710         // `use_dust` switches to using a dust HTLC, which results in the HTLC not having an on-chain
 711         // output at all.
 712         let chanmon_cfgs = create_chanmon_cfgs(3);
 713         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 714
 715         let mut manually_accept_config = test_default_channel_config();
 716         manually_accept_config.manually_accept_inbound_channels = true;
 717
 718         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(manually_accept_config), None]);
 719
 720         let first_persister: test_utils::TestPersister;
 721         let first_new_chain_monitor: test_utils::TestChainMonitor;
 722         let first_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 723         let second_persister: test_utils::TestPersister;
 724         let second_new_chain_monitor: test_utils::TestChainMonitor;
 725         let second_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 726         let third_persister: test_utils::TestPersister;
 727         let third_new_chain_monitor: test_utils::TestChainMonitor;
 728         let third_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 729
 730         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 731
 732         // Because we set nodes[1] to manually accept channels, just open a 0-conf channel.
 733         let (funding_tx, chan_id) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 734         confirm_transaction(&nodes[0], &funding_tx);
 735         confirm_transaction(&nodes[1], &funding_tx);
 736         // Ignore the announcement_signatures messages
 737         nodes[0].node.get_and_clear_pending_msg_events();
 738         nodes[1].node.get_and_clear_pending_msg_events();
 739         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 740
 741         // Serialize the ChannelManager prior to sending payments
 742         let mut nodes_0_serialized = nodes[0].node.encode();
 743
 744         let route = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 }).0;
 745         let (payment_preimage, payment_hash, payment_secret, payment_id) = send_along_route(&nodes[0], route, &[&nodes[1], &nodes[2]], if use_dust { 1_000 } else { 1_000_000 });
 746
 747         // The ChannelMonitor should always be the latest version, as we're required to persist it
 748         // during the `commitment_signed_dance!()`.
 749         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 750
 751         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized], first_persister, first_new_chain_monitor, first_nodes_0_deserialized);
 752         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 753
 754         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 755         // force-close the channel.
 756         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 757         nodes[0].node.timer_tick_occurred();
 758         assert!(nodes[0].node.list_channels().is_empty());
 759         assert!(nodes[0].node.has_pending_payments());
 760         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 761         check_added_monitors!(nodes[0], 1);
 762
 763         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 764                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 765         }, true).unwrap();
 766         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 767
 768         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 769         // error, as the channel has hit the chain.
 770         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 771                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 772         }, false).unwrap();
 773         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 774         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 775         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 776         assert_eq!(as_err.len(), 1);
 777         let bs_commitment_tx;
 778         match as_err[0] {
 779                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 780                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 781                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 782                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) });
 783                         check_added_monitors!(nodes[1], 1);
 784                         bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 785                 },
 786                 _ => panic!("Unexpected event"),
 787         }
 788         check_closed_broadcast!(nodes[1], false);
 789
 790         // Now fail back the payment from nodes[2] to nodes[1]. This doesn't really matter as the
 791         // previous hop channel is already on-chain, but it makes nodes[2] willing to see additional
 792         // incoming HTLCs with the same payment hash later.
 793         nodes[2].node.fail_htlc_backwards(&payment_hash);
 794         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
 795         check_added_monitors!(nodes[2], 1);
 796
 797         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 798         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fail_htlcs[0]);
 799         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 800         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1],
 801                 [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 802
 803         // Connect the HTLC-Timeout transaction, timing out the HTLC on both nodes (but not confirming
 804         // the HTLC-Timeout transaction beyond 1 conf). For dust HTLCs, the HTLC is considered resolved
 805         // after the commitment transaction, so always connect the commitment transaction.
 806         mine_transaction(&nodes[0], &bs_commitment_tx[0]);
 807         mine_transaction(&nodes[1], &bs_commitment_tx[0]);
 808         if !use_dust {
 809                 connect_blocks(&nodes[0], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
 810                 connect_blocks(&nodes[1], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
 811                 let as_htlc_timeout = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 812                 check_spends!(as_htlc_timeout[0], bs_commitment_tx[0]);
 813                 assert_eq!(as_htlc_timeout.len(), 1);
 814
 815                 mine_transaction(&nodes[0], &as_htlc_timeout[0]);
 816                 // nodes[0] may rebroadcast (or RBF-bump) its HTLC-Timeout, so wipe the announced set.
 817                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 818                 mine_transaction(&nodes[1], &as_htlc_timeout[0]);
 819         }
 820
 821         // Create a new channel on which to retry the payment before we fail the payment via the
 822         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 823         // connecting several blocks while creating the channel (implying time has passed).
 824         // We do this with a zero-conf channel to avoid connecting blocks as a side-effect.
 825         let (_, chan_id_3) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 826         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 827
 828         // If we attempt to retry prior to the HTLC-Timeout (or commitment transaction, for dust HTLCs)
 829         // confirming, we will fail as it's considered still-pending...
 830         let (new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 });
 831         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 832                 Err(PaymentSendFailure::DuplicatePayment) => {},
 833                 _ => panic!("Unexpected error")
 834         }
 835         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 836
 837         // After ANTI_REORG_DELAY confirmations, the HTLC should be failed and we can try the payment
 838         // again. We serialize the node first as we'll then test retrying the HTLC after a restart
 839         // (which should also still work).
 840         connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
 841         connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 842         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new());
 843
 844         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 845         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 846         nodes_0_serialized = nodes[0].node.encode();
 847
 848         // After the payment failed, we're free to send it again.
 849         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 850                 RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
 851         assert!(!nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 852
 853         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], second_persister, second_new_chain_monitor, second_nodes_0_deserialized);
 854         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 855
 856         nodes[0].node.test_process_background_events();
 857         check_added_monitors(&nodes[0], 1);
 858
 859         let mut reconnect_args = ReconnectArgs::new(&nodes[0], &nodes[1]);
 860         reconnect_args.send_channel_ready = (true, true);
 861         reconnect_nodes(reconnect_args);
 862
 863         // Now resend the payment, delivering the HTLC and actually claiming it this time. This ensures
 864         // the payment is not (spuriously) listed as still pending.
 865         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 866                 RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
 867         check_added_monitors!(nodes[0], 1);
 868         pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], if use_dust { 1_000 } else { 1_000_000 }, payment_hash, payment_secret);
 869         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
 870
 871         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 872                 Err(PaymentSendFailure::DuplicatePayment) => {},
 873                 _ => panic!("Unexpected error")
 874         }
 875         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 876
 877         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 878         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 879         nodes_0_serialized = nodes[0].node.encode();
 880
 881         // Check that after reload we can send the payment again (though we shouldn't, since it was
 882         // claimed previously).
 883         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], third_persister, third_new_chain_monitor, third_nodes_0_deserialized);
 884         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 885
 886         nodes[0].node.test_process_background_events();
 887         check_added_monitors(&nodes[0], 1);
 888
 889         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
 890
 891         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 892                 Err(PaymentSendFailure::DuplicatePayment) => {},
 893                 _ => panic!("Unexpected error")
 894         }
 895         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 896 }
 897
 898 #[test]
 899 fn test_completed_payment_not_retryable_on_reload() {
 900         do_test_completed_payment_not_retryable_on_reload(true);
 901         do_test_completed_payment_not_retryable_on_reload(false);
 902 }
 903
 904
 905 fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, confirm_commitment_tx: bool, payment_timeout: bool) {
 906         // When a Channel is closed, any outbound HTLCs which were relayed through it are simply
 907         // dropped when the Channel is. From there, the ChannelManager relies on the ChannelMonitor
 908         // having a copy of the relevant fail-/claim-back data and processes the HTLC fail/claim when
 909         // the ChannelMonitor tells it to.
 910         //
 911         // If, due to an on-chain event, an HTLC is failed/claimed, we should avoid providing the
 912         // ChannelManager the HTLC event until after the monitor is re-persisted. This should prevent a
 913         // duplicate HTLC fail/claim (e.g. via a PaymentPathFailed event).
 914         let chanmon_cfgs = create_chanmon_cfgs(2);
 915         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 916         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 917         let persister: test_utils::TestPersister;
 918         let new_chain_monitor: test_utils::TestChainMonitor;
 919         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 920         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 921
 922         let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1);
 923
 924         // Route a payment, but force-close the channel before the HTLC fulfill message arrives at
 925         // nodes[0].
 926         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 10_000_000);
 927         nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap();
 928         check_closed_broadcast!(nodes[0], true);
 929         check_added_monitors!(nodes[0], 1);
 930         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 931
 932         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 933         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 934
 935         // Connect blocks until the CLTV timeout is up so that we get an HTLC-Timeout transaction
 936         connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
 937         let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 938         assert_eq!(node_txn.len(), 3);
 939         assert_eq!(node_txn[0].txid(), node_txn[1].txid());
 940         check_spends!(node_txn[1], funding_tx);
 941         check_spends!(node_txn[2], node_txn[1]);
 942         let timeout_txn = vec![node_txn[2].clone()];
 943
 944         nodes[1].node.claim_funds(payment_preimage);
 945         check_added_monitors!(nodes[1], 1);
 946         expect_payment_claimed!(nodes[1], payment_hash, 10_000_000);
 947
 948         connect_block(&nodes[1], &create_dummy_block(nodes[1].best_block_hash(), 42, vec![node_txn[1].clone()]));
 949         check_closed_broadcast!(nodes[1], true);
 950         check_added_monitors!(nodes[1], 1);
 951         check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 952         let claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 953         assert_eq!(claim_txn.len(), 1);
 954         check_spends!(claim_txn[0], node_txn[1]);
 955
 956         connect_block(&nodes[0], &create_dummy_block(nodes[0].best_block_hash(), 42, vec![node_txn[1].clone()]));
 957
 958         if confirm_commitment_tx {
 959                 connect_blocks(&nodes[0], BREAKDOWN_TIMEOUT as u32 - 1);
 960         }
 961
 962         let claim_block = create_dummy_block(nodes[0].best_block_hash(), 42, if payment_timeout { timeout_txn } else { vec![claim_txn[0].clone()] });
 963
 964         if payment_timeout {
 965                 assert!(confirm_commitment_tx); // Otherwise we're spending below our CSV!
 966                 connect_block(&nodes[0], &claim_block);
 967                 connect_blocks(&nodes[0], ANTI_REORG_DELAY - 2);
 968         }
 969
 970         // Now connect the HTLC claim transaction with the ChainMonitor-generated ChannelMonitor update
 971         // returning InProgress. This should cause the claim event to never make its way to the
 972         // ChannelManager.
 973         chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap().clear();
 974         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
 975
 976         if payment_timeout {
 977                 connect_blocks(&nodes[0], 1);
 978         } else {
 979                 connect_block(&nodes[0], &claim_block);
 980         }
 981
 982         let funding_txo = OutPoint { txid: funding_tx.txid(), index: 0 };
 983         let mon_updates: Vec<_> = chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap()
 984                 .get_mut(&funding_txo).unwrap().drain().collect();
 985         // If we are using chain::Confirm instead of chain::Listen, we will get the same update twice.
 986         // If we're testing connection idempotency we may get substantially more.
 987         assert!(mon_updates.len() >= 1);
 988         assert!(nodes[0].chain_monitor.release_pending_monitor_events().is_empty());
 989         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 990
 991         // If we persist the ChannelManager here, we should get the PaymentSent event after
 992         // deserialization.
 993         let mut chan_manager_serialized = Vec::new();
 994         if !persist_manager_post_event {
 995                 chan_manager_serialized = nodes[0].node.encode();
 996         }
 997
 998         // Now persist the ChannelMonitor and inform the ChainMonitor that we're done, generating the
 999         // payment sent event.
1000         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1001         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
1002         for update in mon_updates {
1003                 nodes[0].chain_monitor.chain_monitor.channel_monitor_updated(funding_txo, update).unwrap();
1004         }
1005         if payment_timeout {
1006                 expect_payment_failed!(nodes[0], payment_hash, false);
1007         } else {
1008                 expect_payment_sent!(nodes[0], payment_preimage);
1009         }
1010
1011         // If we persist the ChannelManager after we get the PaymentSent event, we shouldn't get it
1012         // twice.
1013         if persist_manager_post_event {
1014                 chan_manager_serialized = nodes[0].node.encode();
1015         }
1016
1017         // Now reload nodes[0]...
1018         reload_node!(nodes[0], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
1019
1020         if persist_manager_post_event {
1021                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1022         } else if payment_timeout {
1023                 expect_payment_failed!(nodes[0], payment_hash, false);
1024         } else {
1025                 expect_payment_sent!(nodes[0], payment_preimage);
1026         }
1027
1028         // Note that if we re-connect the block which exposed nodes[0] to the payment preimage (but
1029         // which the current ChannelMonitor has not seen), the ChannelManager's de-duplication of
1030         // payment events should kick in, leaving us with no pending events here.
1031         let height = nodes[0].blocks.lock().unwrap().len() as u32 - 1;
1032         nodes[0].chain_monitor.chain_monitor.block_connected(&claim_block, height);
1033         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1034         check_added_monitors(&nodes[0], 1);
1035 }
1036
1037 #[test]
1038 fn test_dup_htlc_onchain_fails_on_reload() {
1039         do_test_dup_htlc_onchain_fails_on_reload(true, true, true);
1040         do_test_dup_htlc_onchain_fails_on_reload(true, true, false);
1041         do_test_dup_htlc_onchain_fails_on_reload(true, false, false);
1042         do_test_dup_htlc_onchain_fails_on_reload(false, true, true);
1043         do_test_dup_htlc_onchain_fails_on_reload(false, true, false);
1044         do_test_dup_htlc_onchain_fails_on_reload(false, false, false);
1045 }
1046
1047 #[test]
1048 fn test_fulfill_restart_failure() {
1049         // When we receive an update_fulfill_htlc message, we immediately consider the HTLC fully
1050         // fulfilled. At this point, the peer can reconnect and decide to either fulfill the HTLC
1051         // again, or fail it, giving us free money.
1052         //
1053         // Of course probably they won't fail it and give us free money, but because we have code to
1054         // handle it, we should test the logic for it anyway. We do that here.
1055         let chanmon_cfgs = create_chanmon_cfgs(2);
1056         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1057         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1058         let persister: test_utils::TestPersister;
1059         let new_chain_monitor: test_utils::TestChainMonitor;
1060         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
1061         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1062
1063         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1064         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 100_000);
1065
1066         // The simplest way to get a failure after a fulfill is to reload nodes[1] from a state
1067         // pre-fulfill, which we do by serializing it here.
1068         let chan_manager_serialized = nodes[1].node.encode();
1069         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id).encode();
1070
1071         nodes[1].node.claim_funds(payment_preimage);
1072         check_added_monitors!(nodes[1], 1);
1073         expect_payment_claimed!(nodes[1], payment_hash, 100_000);
1074
1075         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1076         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
1077         expect_payment_sent_without_paths!(nodes[0], payment_preimage);
1078
1079         // Now reload nodes[1]...
1080         reload_node!(nodes[1], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1081
1082         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1083         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1084
1085         nodes[1].node.fail_htlc_backwards(&payment_hash);
1086         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]);
1087         check_added_monitors!(nodes[1], 1);
1088         let htlc_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1089         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates.update_fail_htlcs[0]);
1090         commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates.commitment_signed, false);
1091         // nodes[0] shouldn't generate any events here, while it just got a payment failure completion
1092         // it had already considered the payment fulfilled, and now they just got free money.
1093         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1094 }
1095
1096 #[test]
1097 fn get_ldk_payment_preimage() {
1098         // Ensure that `ChannelManager::get_payment_preimage` can successfully be used to claim a payment.
1099         let chanmon_cfgs = create_chanmon_cfgs(2);
1100         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1101         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1102         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1103         create_announced_chan_between_nodes(&nodes, 0, 1);
1104
1105         let amt_msat = 60_000;
1106         let expiry_secs = 60 * 60;
1107         let (payment_hash, payment_secret) = nodes[1].node.create_inbound_payment(Some(amt_msat), expiry_secs, None).unwrap();
1108
1109         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
1110                 .with_bolt11_features(nodes[1].node.invoice_features()).unwrap();
1111         let scorer = test_utils::TestScorer::new();
1112         let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet);
1113         let random_seed_bytes = keys_manager.get_secure_random_bytes();
1114         let route = get_route(
1115                 &nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(),
1116                 Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()),
1117                 amt_msat, nodes[0].logger, &scorer, &(), &random_seed_bytes).unwrap();
1118         nodes[0].node.send_payment_with_route(&route, payment_hash,
1119                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
1120         check_added_monitors!(nodes[0], 1);
1121
1122         // Make sure to use `get_payment_preimage`
1123         let payment_preimage = nodes[1].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1124         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1125         assert_eq!(events.len(), 1);
1126         pass_along_path(&nodes[0], &[&nodes[1]], amt_msat, payment_hash, Some(payment_secret), events.pop().unwrap(), true, Some(payment_preimage));
1127         claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, payment_preimage);
1128 }
1129
1130 #[test]
1131 fn sent_probe_is_probe_of_sending_node() {
1132         let chanmon_cfgs = create_chanmon_cfgs(3);
1133         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1134         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1135         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1136
1137         create_announced_chan_between_nodes(&nodes, 0, 1);
1138         create_announced_chan_between_nodes(&nodes, 1, 2);
1139
1140         // First check we refuse to build a single-hop probe
1141         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[1], 100_000);
1142         assert!(nodes[0].node.send_probe(route.paths[0].clone()).is_err());
1143
1144         // Then build an actual two-hop probing path
1145         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
1146
1147         match nodes[0].node.send_probe(route.paths[0].clone()) {
1148                 Ok((payment_hash, payment_id)) => {
1149                         assert!(nodes[0].node.payment_is_probe(&payment_hash, &payment_id));
1150                         assert!(!nodes[1].node.payment_is_probe(&payment_hash, &payment_id));
1151                         assert!(!nodes[2].node.payment_is_probe(&payment_hash, &payment_id));
1152                 },
1153                 _ => panic!(),
1154         }
1155
1156         get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1157         check_added_monitors!(nodes[0], 1);
1158 }
1159
1160 #[test]
1161 fn successful_probe_yields_event() {
1162         let chanmon_cfgs = create_chanmon_cfgs(3);
1163         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1164         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1165         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1166
1167         create_announced_chan_between_nodes(&nodes, 0, 1);
1168         create_announced_chan_between_nodes(&nodes, 1, 2);
1169
1170         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
1171
1172         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1173
1174         // node[0] -- update_add_htlcs -> node[1]
1175         check_added_monitors!(nodes[0], 1);
1176         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1177         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1178         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1179         check_added_monitors!(nodes[1], 0);
1180         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1181         expect_pending_htlcs_forwardable!(nodes[1]);
1182
1183         // node[1] -- update_add_htlcs -> node[2]
1184         check_added_monitors!(nodes[1], 1);
1185         let updates = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1186         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1187         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &probe_event.msgs[0]);
1188         check_added_monitors!(nodes[2], 0);
1189         commitment_signed_dance!(nodes[2], nodes[1], probe_event.commitment_msg, true, true);
1190
1191         // node[1] <- update_fail_htlcs -- node[2]
1192         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1193         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1194         check_added_monitors!(nodes[1], 0);
1195         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, true);
1196
1197         // node[0] <- update_fail_htlcs -- node[1]
1198         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1199         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1200         check_added_monitors!(nodes[0], 0);
1201         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1202
1203         let mut events = nodes[0].node.get_and_clear_pending_events();
1204         assert_eq!(events.len(), 1);
1205         match events.drain(..).next().unwrap() {
1206                 crate::events::Event::ProbeSuccessful { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1207                         assert_eq!(payment_id, ev_pid);
1208                         assert_eq!(payment_hash, ev_ph);
1209                 },
1210                 _ => panic!(),
1211         };
1212         assert!(!nodes[0].node.has_pending_payments());
1213 }
1214
1215 #[test]
1216 fn failed_probe_yields_event() {
1217         let chanmon_cfgs = create_chanmon_cfgs(3);
1218         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1219         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1220         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1221
1222         create_announced_chan_between_nodes(&nodes, 0, 1);
1223         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 100000, 90000000);
1224
1225         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1226
1227         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 9_998_000);
1228
1229         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1230
1231         // node[0] -- update_add_htlcs -> node[1]
1232         check_added_monitors!(nodes[0], 1);
1233         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1234         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1235         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1236         check_added_monitors!(nodes[1], 0);
1237         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1238         expect_pending_htlcs_forwardable!(nodes[1]);
1239
1240         // node[0] <- update_fail_htlcs -- node[1]
1241         check_added_monitors!(nodes[1], 1);
1242         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1243         // Skip the PendingHTLCsForwardable event
1244         let _events = nodes[1].node.get_and_clear_pending_events();
1245         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1246         check_added_monitors!(nodes[0], 0);
1247         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1248
1249         let mut events = nodes[0].node.get_and_clear_pending_events();
1250         assert_eq!(events.len(), 1);
1251         match events.drain(..).next().unwrap() {
1252                 crate::events::Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1253                         assert_eq!(payment_id, ev_pid);
1254                         assert_eq!(payment_hash, ev_ph);
1255                 },
1256                 _ => panic!(),
1257         };
1258         assert!(!nodes[0].node.has_pending_payments());
1259 }
1260
1261 #[test]
1262 fn onchain_failed_probe_yields_event() {
1263         // Tests that an attempt to probe over a channel that is eventaully closed results in a failure
1264         // event.
1265         let chanmon_cfgs = create_chanmon_cfgs(3);
1266         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1267         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1268         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1269
1270         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1271         create_announced_chan_between_nodes(&nodes, 1, 2);
1272
1273         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1274
1275         // Send a dust HTLC, which will be treated as if it timed out once the channel hits the chain.
1276         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 1_000);
1277         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1278
1279         // node[0] -- update_add_htlcs -> node[1]
1280         check_added_monitors!(nodes[0], 1);
1281         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1282         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1283         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1284         check_added_monitors!(nodes[1], 0);
1285         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1286         expect_pending_htlcs_forwardable!(nodes[1]);
1287
1288         check_added_monitors!(nodes[1], 1);
1289         let _ = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1290
1291         // Don't bother forwarding the HTLC onwards and just confirm the force-close transaction on
1292         // Node A, which after 6 confirmations should result in a probe failure event.
1293         let bs_txn = get_local_commitment_txn!(nodes[1], chan_id);
1294         confirm_transaction(&nodes[0], &bs_txn[0]);
1295         check_closed_broadcast!(&nodes[0], true);
1296         check_added_monitors!(nodes[0], 1);
1297
1298         let mut events = nodes[0].node.get_and_clear_pending_events();
1299         assert_eq!(events.len(), 2);
1300         let mut found_probe_failed = false;
1301         for event in events.drain(..) {
1302                 match event {
1303                         Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1304                                 assert_eq!(payment_id, ev_pid);
1305                                 assert_eq!(payment_hash, ev_ph);
1306                                 found_probe_failed = true;
1307                         },
1308                         Event::ChannelClosed { .. } => {},
1309                         _ => panic!(),
1310                 }
1311         }
1312         assert!(found_probe_failed);
1313         assert!(!nodes[0].node.has_pending_payments());
1314 }
1315
1316 #[test]
1317 fn claimed_send_payment_idempotent() {
1318         // Tests that `send_payment` (and friends) are (reasonably) idempotent.
1319         let chanmon_cfgs = create_chanmon_cfgs(2);
1320         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1321         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1322         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1323
1324         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1325
1326         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1327         let (first_payment_preimage, _, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1328
1329         macro_rules! check_send_rejected {
1330                 () => {
1331                         // If we try to resend a new payment with a different payment_hash but with the same
1332                         // payment_id, it should be rejected.
1333                         let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1334                                 RecipientOnionFields::secret_only(second_payment_secret), payment_id);
1335                         match send_result {
1336                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1337                                 _ => panic!("Unexpected send result: {:?}", send_result),
1338                         }
1339
1340                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1341                         // also be rejected.
1342                         let send_result = nodes[0].node.send_spontaneous_payment(
1343                                 &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
1344                         match send_result {
1345                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1346                                 _ => panic!("Unexpected send result: {:?}", send_result),
1347                         }
1348                 }
1349         }
1350
1351         check_send_rejected!();
1352
1353         // Claim the payment backwards, but note that the PaymentSent event is still pending and has
1354         // not been seen by the user. At this point, from the user perspective nothing has changed, so
1355         // we must remain just as idempotent as we were before.
1356         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, first_payment_preimage);
1357
1358         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1359                 nodes[0].node.timer_tick_occurred();
1360         }
1361
1362         check_send_rejected!();
1363
1364         // Once the user sees and handles the `PaymentSent` event, we expect them to no longer call
1365         // `send_payment`, and our idempotency guarantees are off - they should have atomically marked
1366         // the payment complete. However, they could have called `send_payment` while the event was
1367         // being processed, leading to a race in our idempotency guarantees. Thus, even immediately
1368         // after the event is handled a duplicate payment should sitll be rejected.
1369         expect_payment_sent!(&nodes[0], first_payment_preimage, Some(0));
1370         check_send_rejected!();
1371
1372         // If relatively little time has passed, a duplicate payment should still fail.
1373         nodes[0].node.timer_tick_occurred();
1374         check_send_rejected!();
1375
1376         // However, after some time has passed (at least more than the one timer tick above), a
1377         // duplicate payment should go through, as ChannelManager should no longer have any remaining
1378         // references to the old payment data.
1379         for _ in 0..IDEMPOTENCY_TIMEOUT_TICKS {
1380                 nodes[0].node.timer_tick_occurred();
1381         }
1382
1383         nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1384                 RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
1385         check_added_monitors!(nodes[0], 1);
1386         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1387         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1388 }
1389
1390 #[test]
1391 fn abandoned_send_payment_idempotent() {
1392         // Tests that `send_payment` (and friends) allow duplicate PaymentIds immediately after
1393         // abandon_payment.
1394         let chanmon_cfgs = create_chanmon_cfgs(2);
1395         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1396         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1397         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1398
1399         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1400
1401         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1402         let (_, first_payment_hash, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1403
1404         macro_rules! check_send_rejected {
1405                 () => {
1406                         // If we try to resend a new payment with a different payment_hash but with the same
1407                         // payment_id, it should be rejected.
1408                         let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1409                                 RecipientOnionFields::secret_only(second_payment_secret), payment_id);
1410                         match send_result {
1411                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1412                                 _ => panic!("Unexpected send result: {:?}", send_result),
1413                         }
1414
1415                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1416                         // also be rejected.
1417                         let send_result = nodes[0].node.send_spontaneous_payment(
1418                                 &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
1419                         match send_result {
1420                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1421                                 _ => panic!("Unexpected send result: {:?}", send_result),
1422                         }
1423                 }
1424         }
1425
1426         check_send_rejected!();
1427
1428         nodes[1].node.fail_htlc_backwards(&first_payment_hash);
1429         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash: first_payment_hash }]);
1430
1431         // Until we abandon the payment upon path failure, no matter how many timer ticks pass, we still cannot reuse the
1432         // PaymentId.
1433         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1434                 nodes[0].node.timer_tick_occurred();
1435         }
1436         check_send_rejected!();
1437
1438         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash, PaymentFailureReason::RecipientRejected);
1439
1440         // However, we can reuse the PaymentId immediately after we `abandon_payment` upon passing the
1441         // failed payment back.
1442         nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1443                 RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
1444         check_added_monitors!(nodes[0], 1);
1445         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1446         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1447 }
1448
1449 #[derive(PartialEq)]
1450 enum InterceptTest {
1451         Forward,
1452         Fail,
1453         Timeout,
1454 }
1455
1456 #[test]
1457 fn test_trivial_inflight_htlc_tracking(){
1458         // In this test, we test three scenarios:
1459         // (1) Sending + claiming a payment successfully should return `None` when querying InFlightHtlcs
1460         // (2) Sending a payment without claiming it should return the payment's value (500000) when querying InFlightHtlcs
1461         // (3) After we claim the payment sent in (2), InFlightHtlcs should return `None` for the query.
1462         let chanmon_cfgs = create_chanmon_cfgs(3);
1463         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1464         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1465         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1466
1467         let (_, _, chan_1_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
1468         let (_, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
1469
1470         // Send and claim the payment. Inflight HTLCs should be empty.
1471         let payment_hash = send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 500000).1;
1472         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1473         {
1474                 let mut node_0_per_peer_lock;
1475                 let mut node_0_peer_state_lock;
1476                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1477
1478                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1479                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1480                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1481                         channel_1.context.get_short_channel_id().unwrap()
1482                 );
1483                 assert_eq!(chan_1_used_liquidity, None);
1484         }
1485         {
1486                 let mut node_1_per_peer_lock;
1487                 let mut node_1_peer_state_lock;
1488                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1489
1490                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1491                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1492                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1493                         channel_2.context.get_short_channel_id().unwrap()
1494                 );
1495
1496                 assert_eq!(chan_2_used_liquidity, None);
1497         }
1498         let pending_payments = nodes[0].node.list_recent_payments();
1499         assert_eq!(pending_payments.len(), 1);
1500         assert_eq!(pending_payments[0], RecentPaymentDetails::Fulfilled { payment_hash: Some(payment_hash) });
1501
1502         // Remove fulfilled payment
1503         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1504                 nodes[0].node.timer_tick_occurred();
1505         }
1506
1507         // Send the payment, but do not claim it. Our inflight HTLCs should contain the pending payment.
1508         let (payment_preimage, payment_hash,  _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 500000);
1509         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1510         {
1511                 let mut node_0_per_peer_lock;
1512                 let mut node_0_peer_state_lock;
1513                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1514
1515                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1516                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1517                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1518                         channel_1.context.get_short_channel_id().unwrap()
1519                 );
1520                 // First hop accounts for expected 1000 msat fee
1521                 assert_eq!(chan_1_used_liquidity, Some(501000));
1522         }
1523         {
1524                 let mut node_1_per_peer_lock;
1525                 let mut node_1_peer_state_lock;
1526                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1527
1528                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1529                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1530                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1531                         channel_2.context.get_short_channel_id().unwrap()
1532                 );
1533
1534                 assert_eq!(chan_2_used_liquidity, Some(500000));
1535         }
1536         let pending_payments = nodes[0].node.list_recent_payments();
1537         assert_eq!(pending_payments.len(), 1);
1538         assert_eq!(pending_payments[0], RecentPaymentDetails::Pending { payment_hash, total_msat: 500000 });
1539
1540         // Now, let's claim the payment. This should result in the used liquidity to return `None`.
1541         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
1542
1543         // Remove fulfilled payment
1544         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1545                 nodes[0].node.timer_tick_occurred();
1546         }
1547
1548         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1549         {
1550                 let mut node_0_per_peer_lock;
1551                 let mut node_0_peer_state_lock;
1552                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1553
1554                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1555                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1556                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1557                         channel_1.context.get_short_channel_id().unwrap()
1558                 );
1559                 assert_eq!(chan_1_used_liquidity, None);
1560         }
1561         {
1562                 let mut node_1_per_peer_lock;
1563                 let mut node_1_peer_state_lock;
1564                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1565
1566                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1567                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1568                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1569                         channel_2.context.get_short_channel_id().unwrap()
1570                 );
1571                 assert_eq!(chan_2_used_liquidity, None);
1572         }
1573
1574         let pending_payments = nodes[0].node.list_recent_payments();
1575         assert_eq!(pending_payments.len(), 0);
1576 }
1577
1578 #[test]
1579 fn test_holding_cell_inflight_htlcs() {
1580         let chanmon_cfgs = create_chanmon_cfgs(2);
1581         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1582         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1583         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1584         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1585
1586         let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
1587         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1588
1589         // Queue up two payments - one will be delivered right away, one immediately goes into the
1590         // holding cell as nodes[0] is AwaitingRAA.
1591         {
1592                 nodes[0].node.send_payment_with_route(&route, payment_hash_1,
1593                         RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
1594                 check_added_monitors!(nodes[0], 1);
1595                 nodes[0].node.send_payment_with_route(&route, payment_hash_2,
1596                         RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
1597                 check_added_monitors!(nodes[0], 0);
1598         }
1599
1600         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1601
1602         {
1603                 let mut node_0_per_peer_lock;
1604                 let mut node_0_peer_state_lock;
1605                 let channel =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, channel_id);
1606
1607                 let used_liquidity = inflight_htlcs.used_liquidity_msat(
1608                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1609                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1610                         channel.context.get_short_channel_id().unwrap()
1611                 );
1612
1613                 assert_eq!(used_liquidity, Some(2000000));
1614         }
1615
1616         // Clear pending events so test doesn't throw a "Had excess message on node..." error
1617         nodes[0].node.get_and_clear_pending_msg_events();
1618 }
1619
1620 #[test]
1621 fn intercepted_payment() {
1622         // Test that detecting an intercept scid on payment forward will signal LDK to generate an
1623         // intercept event, which the LSP can then use to either (a) open a JIT channel to forward the
1624         // payment or (b) fail the payment.
1625         do_test_intercepted_payment(InterceptTest::Forward);
1626         do_test_intercepted_payment(InterceptTest::Fail);
1627         // Make sure that intercepted payments will be automatically failed back if too many blocks pass.
1628         do_test_intercepted_payment(InterceptTest::Timeout);
1629 }
1630
1631 fn do_test_intercepted_payment(test: InterceptTest) {
1632         let chanmon_cfgs = create_chanmon_cfgs(3);
1633         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1634
1635         let mut zero_conf_chan_config = test_default_channel_config();
1636         zero_conf_chan_config.manually_accept_inbound_channels = true;
1637         let mut intercept_forwards_config = test_default_channel_config();
1638         intercept_forwards_config.accept_intercept_htlcs = true;
1639         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(zero_conf_chan_config)]);
1640
1641         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1642         let scorer = test_utils::TestScorer::new();
1643         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
1644
1645         let _ = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1646
1647         let amt_msat = 100_000;
1648         let intercept_scid = nodes[1].node.get_intercept_scid();
1649         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1650                 .with_route_hints(vec![
1651                         RouteHint(vec![RouteHintHop {
1652                                 src_node_id: nodes[1].node.get_our_node_id(),
1653                                 short_channel_id: intercept_scid,
1654                                 fees: RoutingFees {
1655                                         base_msat: 1000,
1656                                         proportional_millionths: 0,
1657                                 },
1658                                 cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
1659                                 htlc_minimum_msat: None,
1660                                 htlc_maximum_msat: None,
1661                         }])
1662                 ]).unwrap()
1663                 .with_bolt11_features(nodes[2].node.invoice_features()).unwrap();
1664         let route_params = RouteParameters {
1665                 payment_params,
1666                 final_value_msat: amt_msat,
1667         };
1668         let route = get_route(
1669                 &nodes[0].node.get_our_node_id(), &route_params.payment_params,
1670                 &nodes[0].network_graph.read_only(), None, route_params.final_value_msat,
1671                 nodes[0].logger, &scorer, &(), &random_seed_bytes,
1672         ).unwrap();
1673
1674         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
1675         nodes[0].node.send_payment_with_route(&route, payment_hash,
1676                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
1677         let payment_event = {
1678                 {
1679                         let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
1680                         assert_eq!(added_monitors.len(), 1);
1681                         added_monitors.clear();
1682                 }
1683                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1684                 assert_eq!(events.len(), 1);
1685                 SendEvent::from_event(events.remove(0))
1686         };
1687         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1688         commitment_signed_dance!(nodes[1], nodes[0], &payment_event.commitment_msg, false, true);
1689
1690         // Check that we generate the PaymentIntercepted event when an intercept forward is detected.
1691         let events = nodes[1].node.get_and_clear_pending_events();
1692         assert_eq!(events.len(), 1);
1693         let (intercept_id, expected_outbound_amount_msat) = match events[0] {
1694                 crate::events::Event::HTLCIntercepted {
1695                         intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, inbound_amount_msat, requested_next_hop_scid: short_channel_id
1696                 } => {
1697                         assert_eq!(pmt_hash, payment_hash);
1698                         assert_eq!(inbound_amount_msat, route.get_total_amount() + route.get_total_fees());
1699                         assert_eq!(short_channel_id, intercept_scid);
1700                         (intercept_id, expected_outbound_amount_msat)
1701                 },
1702                 _ => panic!()
1703         };
1704
1705         // Check for unknown channel id error.
1706         let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &[42; 32], nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1707         assert_eq!(unknown_chan_id_err , APIError::ChannelUnavailable  {
1708                 err: format!("Funded channel with id {} not found for the passed counterparty node_id {}. Channel may still be opening.",
1709                         log_bytes!([42; 32]), nodes[2].node.get_our_node_id()) });
1710
1711         if test == InterceptTest::Fail {
1712                 // Ensure we can fail the intercepted payment back.
1713                 nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap();
1714                 expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::UnknownNextHop { requested_forward_scid: intercept_scid }]);
1715                 nodes[1].node.process_pending_htlc_forwards();
1716                 let update_fail = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1717                 check_added_monitors!(&nodes[1], 1);
1718                 assert!(update_fail.update_fail_htlcs.len() == 1);
1719                 let fail_msg = update_fail.update_fail_htlcs[0].clone();
1720                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1721                 commitment_signed_dance!(nodes[0], nodes[1], update_fail.commitment_signed, false);
1722
1723                 // Ensure the payment fails with the expected error.
1724                 let fail_conditions = PaymentFailedConditions::new()
1725                         .blamed_scid(intercept_scid)
1726                         .blamed_chan_closed(true)
1727                         .expected_htlc_error_data(0x4000 | 10, &[]);
1728                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
1729         } else if test == InterceptTest::Forward {
1730                 // Check that we'll fail as expected when sending to a channel that isn't in `ChannelReady` yet.
1731                 let temp_chan_id = nodes[1].node.create_channel(nodes[2].node.get_our_node_id(), 100_000, 0, 42, None).unwrap();
1732                 let unusable_chan_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &temp_chan_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1733                 assert_eq!(unusable_chan_err , APIError::ChannelUnavailable {
1734                         err: format!("Funded channel with id {} not found for the passed counterparty node_id {}. Channel may still be opening.",
1735                                 log_bytes!(temp_chan_id), nodes[2].node.get_our_node_id()) });
1736                 assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
1737
1738                 // Open the just-in-time channel so the payment can then be forwarded.
1739                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1740
1741                 // Finally, forward the intercepted payment through and claim it.
1742                 nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap();
1743                 expect_pending_htlcs_forwardable!(nodes[1]);
1744
1745                 let payment_event = {
1746                         {
1747                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
1748                                 assert_eq!(added_monitors.len(), 1);
1749                                 added_monitors.clear();
1750                         }
1751                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
1752                         assert_eq!(events.len(), 1);
1753                         SendEvent::from_event(events.remove(0))
1754                 };
1755                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1756                 commitment_signed_dance!(nodes[2], nodes[1], &payment_event.commitment_msg, false, true);
1757                 expect_pending_htlcs_forwardable!(nodes[2]);
1758
1759                 let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1760                 expect_payment_claimable!(&nodes[2], payment_hash, payment_secret, amt_msat, Some(payment_preimage), nodes[2].node.get_our_node_id());
1761                 do_claim_payment_along_route(&nodes[0], &vec!(&vec!(&nodes[1], &nodes[2])[..]), false, payment_preimage);
1762                 let events = nodes[0].node.get_and_clear_pending_events();
1763                 assert_eq!(events.len(), 2);
1764                 match events[0] {
1765                         Event::PaymentSent { payment_preimage: ref ev_preimage, payment_hash: ref ev_hash, ref fee_paid_msat, .. } => {
1766                                 assert_eq!(payment_preimage, *ev_preimage);
1767                                 assert_eq!(payment_hash, *ev_hash);
1768                                 assert_eq!(fee_paid_msat, &Some(1000));
1769                         },
1770                         _ => panic!("Unexpected event")
1771                 }
1772                 match events[1] {
1773                         Event::PaymentPathSuccessful { payment_hash: hash, .. } => {
1774                                 assert_eq!(hash, Some(payment_hash));
1775                         },
1776                         _ => panic!("Unexpected event")
1777                 }
1778         } else if test == InterceptTest::Timeout {
1779                 let mut block = create_dummy_block(nodes[0].best_block_hash(), 42, Vec::new());
1780                 connect_block(&nodes[0], &block);
1781                 connect_block(&nodes[1], &block);
1782                 for _ in 0..TEST_FINAL_CLTV {
1783                         block.header.prev_blockhash = block.block_hash();
1784                         connect_block(&nodes[0], &block);
1785                         connect_block(&nodes[1], &block);
1786                 }
1787                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::InvalidForward { requested_forward_scid: intercept_scid }]);
1788                 check_added_monitors!(nodes[1], 1);
1789                 let htlc_timeout_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1790                 assert!(htlc_timeout_updates.update_add_htlcs.is_empty());
1791                 assert_eq!(htlc_timeout_updates.update_fail_htlcs.len(), 1);
1792                 assert!(htlc_timeout_updates.update_fail_malformed_htlcs.is_empty());
1793                 assert!(htlc_timeout_updates.update_fee.is_none());
1794
1795                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_timeout_updates.update_fail_htlcs[0]);
1796                 commitment_signed_dance!(nodes[0], nodes[1], htlc_timeout_updates.commitment_signed, false);
1797                 expect_payment_failed!(nodes[0], payment_hash, false, 0x2000 | 2, []);
1798
1799                 // Check for unknown intercept id error.
1800                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1801                 let unknown_intercept_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1802                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1803                 let unknown_intercept_id_err = nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap_err();
1804                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1805         }
1806 }
1807
1808 #[test]
1809 fn accept_underpaying_htlcs_config() {
1810         do_accept_underpaying_htlcs_config(1);
1811         do_accept_underpaying_htlcs_config(2);
1812         do_accept_underpaying_htlcs_config(3);
1813 }
1814
1815 fn do_accept_underpaying_htlcs_config(num_mpp_parts: usize) {
1816         let chanmon_cfgs = create_chanmon_cfgs(3);
1817         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1818         let mut intercept_forwards_config = test_default_channel_config();
1819         intercept_forwards_config.accept_intercept_htlcs = true;
1820         let mut underpay_config = test_default_channel_config();
1821         underpay_config.channel_config.accept_underpaying_htlcs = true;
1822         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(underpay_config)]);
1823         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1824
1825         let mut chan_ids = Vec::new();
1826         for _ in 0..num_mpp_parts {
1827                 let _ = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000, 0);
1828                 let channel_id = create_unannounced_chan_between_nodes_with_value(&nodes, 1, 2, 2_000_000, 0).0.channel_id;
1829                 chan_ids.push(channel_id);
1830         }
1831
1832         // Send the initial payment.
1833         let amt_msat = 900_000;
1834         let skimmed_fee_msat = 20;
1835         let mut route_hints = Vec::new();
1836         for _ in 0..num_mpp_parts {
1837                 route_hints.push(RouteHint(vec![RouteHintHop {
1838                         src_node_id: nodes[1].node.get_our_node_id(),
1839                         short_channel_id: nodes[1].node.get_intercept_scid(),
1840                         fees: RoutingFees {
1841                                 base_msat: 1000,
1842                                 proportional_millionths: 0,
1843                         },
1844                         cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
1845                         htlc_minimum_msat: None,
1846                         htlc_maximum_msat: Some(amt_msat / num_mpp_parts as u64 + 5),
1847                 }]));
1848         }
1849         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1850                 .with_route_hints(route_hints).unwrap()
1851                 .with_bolt11_features(nodes[2].node.invoice_features()).unwrap();
1852         let route_params = RouteParameters {
1853                 payment_params,
1854                 final_value_msat: amt_msat,
1855         };
1856         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
1857         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
1858                 PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
1859         check_added_monitors!(nodes[0], num_mpp_parts); // one monitor per path
1860         let mut events: Vec<SendEvent> = nodes[0].node.get_and_clear_pending_msg_events().into_iter().map(|e| SendEvent::from_event(e)).collect();
1861         assert_eq!(events.len(), num_mpp_parts);
1862
1863         // Forward the intercepted payments.
1864         for (idx, ev) in events.into_iter().enumerate() {
1865                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &ev.msgs[0]);
1866                 do_commitment_signed_dance(&nodes[1], &nodes[0], &ev.commitment_msg, false, true);
1867
1868                 let events = nodes[1].node.get_and_clear_pending_events();
1869                 assert_eq!(events.len(), 1);
1870                 let (intercept_id, expected_outbound_amt_msat) = match events[0] {
1871                         crate::events::Event::HTLCIntercepted {
1872                                 intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, ..
1873                         } => {
1874                                 assert_eq!(pmt_hash, payment_hash);
1875                                 (intercept_id, expected_outbound_amount_msat)
1876                         },
1877                         _ => panic!()
1878                 };
1879                 nodes[1].node.forward_intercepted_htlc(intercept_id, &chan_ids[idx],
1880                         nodes[2].node.get_our_node_id(), expected_outbound_amt_msat - skimmed_fee_msat).unwrap();
1881                 expect_pending_htlcs_forwardable!(nodes[1]);
1882                 let payment_event = {
1883                         {
1884                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
1885                                 assert_eq!(added_monitors.len(), 1);
1886                                 added_monitors.clear();
1887                         }
1888                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
1889                         assert_eq!(events.len(), 1);
1890                         SendEvent::from_event(events.remove(0))
1891                 };
1892                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1893                 do_commitment_signed_dance(&nodes[2], &nodes[1], &payment_event.commitment_msg, false, true);
1894                 if idx == num_mpp_parts - 1 {
1895                         expect_pending_htlcs_forwardable!(nodes[2]);
1896                 }
1897         }
1898
1899         // Claim the payment and check that the skimmed fee is as expected.
1900         let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1901         let events = nodes[2].node.get_and_clear_pending_events();
1902         assert_eq!(events.len(), 1);
1903         match events[0] {
1904                 crate::events::Event::PaymentClaimable {
1905                         ref payment_hash, ref purpose, amount_msat, counterparty_skimmed_fee_msat, receiver_node_id, ..
1906                 } => {
1907                         assert_eq!(payment_hash, payment_hash);
1908                         assert_eq!(amt_msat - skimmed_fee_msat * num_mpp_parts as u64, amount_msat);
1909                         assert_eq!(skimmed_fee_msat * num_mpp_parts as u64, counterparty_skimmed_fee_msat);
1910                         assert_eq!(nodes[2].node.get_our_node_id(), receiver_node_id.unwrap());
1911                         match purpose {
1912                                 crate::events::PaymentPurpose::InvoicePayment { payment_preimage: ev_payment_preimage,
1913                                         payment_secret: ev_payment_secret, .. } =>
1914                                 {
1915                                         assert_eq!(payment_preimage, ev_payment_preimage.unwrap());
1916                                         assert_eq!(payment_secret, *ev_payment_secret);
1917                                 },
1918                                 _ => panic!(),
1919                         }
1920                 },
1921                 _ => panic!("Unexpected event"),
1922         }
1923         let mut expected_paths_vecs = Vec::new();
1924         let mut expected_paths = Vec::new();
1925         for _ in 0..num_mpp_parts { expected_paths_vecs.push(vec!(&nodes[1], &nodes[2])); }
1926         for i in 0..num_mpp_parts { expected_paths.push(&expected_paths_vecs[i][..]); }
1927         let total_fee_msat = do_claim_payment_along_route_with_extra_penultimate_hop_fees(
1928                 &nodes[0], &expected_paths[..], &vec![skimmed_fee_msat as u32; num_mpp_parts][..], false,
1929                 payment_preimage);
1930         // The sender doesn't know that the penultimate hop took an extra fee.
1931         expect_payment_sent(&nodes[0], payment_preimage,
1932                 Some(Some(total_fee_msat - skimmed_fee_msat * num_mpp_parts as u64)), true);
1933 }
1934
1935 #[derive(PartialEq)]
1936 enum AutoRetry {
1937         Success,
1938         Spontaneous,
1939         FailAttempts,
1940         FailTimeout,
1941         FailOnRestart,
1942         FailOnRetry,
1943 }
1944
1945 #[test]
1946 fn automatic_retries() {
1947         do_automatic_retries(AutoRetry::Success);
1948         do_automatic_retries(AutoRetry::Spontaneous);
1949         do_automatic_retries(AutoRetry::FailAttempts);
1950         do_automatic_retries(AutoRetry::FailTimeout);
1951         do_automatic_retries(AutoRetry::FailOnRestart);
1952         do_automatic_retries(AutoRetry::FailOnRetry);
1953 }
1954 fn do_automatic_retries(test: AutoRetry) {
1955         // Test basic automatic payment retries in ChannelManager. See individual `test` variant comments
1956         // below.
1957         let chanmon_cfgs = create_chanmon_cfgs(3);
1958         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1959         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1960
1961         let persister;
1962         let new_chain_monitor;
1963         let node_0_deserialized;
1964
1965         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1966         let channel_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1967         let channel_id_2 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1968
1969         // Marshall data to send the payment
1970         #[cfg(feature = "std")]
1971         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1972         #[cfg(not(feature = "std"))]
1973         let payment_expiry_secs = 60 * 60;
1974         let amt_msat = 1000;
1975         let mut invoice_features = Bolt11InvoiceFeatures::empty();
1976         invoice_features.set_variable_length_onion_required();
1977         invoice_features.set_payment_secret_required();
1978         invoice_features.set_basic_mpp_optional();
1979         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1980                 .with_expiry_time(payment_expiry_secs as u64)
1981                 .with_bolt11_features(invoice_features).unwrap();
1982         let route_params = RouteParameters {
1983                 payment_params,
1984                 final_value_msat: amt_msat,
1985         };
1986         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
1987
1988         macro_rules! pass_failed_attempt_with_retry_along_path {
1989                 ($failing_channel_id: expr, $expect_pending_htlcs_forwardable: expr) => {
1990                         // Send a payment attempt that fails due to lack of liquidity on the second hop
1991                         check_added_monitors!(nodes[0], 1);
1992                         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1993                         let mut update_add = update_0.update_add_htlcs[0].clone();
1994                         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
1995                         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
1996                         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
1997                         nodes[1].node.process_pending_htlc_forwards();
1998                         expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1],
1999                                 vec![HTLCDestination::NextHopChannel {
2000                                         node_id: Some(nodes[2].node.get_our_node_id()),
2001                                         channel_id: $failing_channel_id,
2002                                 }]);
2003                         nodes[1].node.process_pending_htlc_forwards();
2004                         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2005                         check_added_monitors!(&nodes[1], 1);
2006                         assert!(update_1.update_fail_htlcs.len() == 1);
2007                         let fail_msg = update_1.update_fail_htlcs[0].clone();
2008                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
2009                         commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
2010
2011                         // Ensure the attempt fails and a new PendingHTLCsForwardable event is generated for the retry
2012                         let mut events = nodes[0].node.get_and_clear_pending_events();
2013                         assert_eq!(events.len(), 2);
2014                         match events[0] {
2015                                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2016                                         assert_eq!(payment_hash, ev_payment_hash);
2017                                         assert_eq!(payment_failed_permanently, false);
2018                                 },
2019                                 _ => panic!("Unexpected event"),
2020                         }
2021                         if $expect_pending_htlcs_forwardable {
2022                                 match events[1] {
2023                                         Event::PendingHTLCsForwardable { .. } => {},
2024                                         _ => panic!("Unexpected event"),
2025                                 }
2026                         } else {
2027                                 match events[1] {
2028                                         Event::PaymentFailed { payment_hash: ev_payment_hash, .. } => {
2029                                                 assert_eq!(payment_hash, ev_payment_hash);
2030                                         },
2031                                         _ => panic!("Unexpected event"),
2032                                 }
2033                         }
2034                 }
2035         }
2036
2037         if test == AutoRetry::Success {
2038                 // Test that we can succeed on the first retry.
2039                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2040                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2041                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2042
2043                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
2044                 // attempt, since the initial second hop channel will be excluded from pathfinding
2045                 create_announced_chan_between_nodes(&nodes, 1, 2);
2046
2047                 // We retry payments in `process_pending_htlc_forwards`
2048                 nodes[0].node.process_pending_htlc_forwards();
2049                 check_added_monitors!(nodes[0], 1);
2050                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2051                 assert_eq!(msg_events.len(), 1);
2052                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, Some(payment_secret), msg_events.pop().unwrap(), true, None);
2053                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
2054         } else if test == AutoRetry::Spontaneous {
2055                 nodes[0].node.send_spontaneous_payment_with_retry(Some(payment_preimage),
2056                         RecipientOnionFields::spontaneous_empty(), PaymentId(payment_hash.0), route_params,
2057                         Retry::Attempts(1)).unwrap();
2058                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2059
2060                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
2061                 // attempt, since the initial second hop channel will be excluded from pathfinding
2062                 create_announced_chan_between_nodes(&nodes, 1, 2);
2063
2064                 // We retry payments in `process_pending_htlc_forwards`
2065                 nodes[0].node.process_pending_htlc_forwards();
2066                 check_added_monitors!(nodes[0], 1);
2067                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2068                 assert_eq!(msg_events.len(), 1);
2069                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, None, msg_events.pop().unwrap(), true, Some(payment_preimage));
2070                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
2071         } else if test == AutoRetry::FailAttempts {
2072                 // Ensure ChannelManager will not retry a payment if it has run out of payment attempts.
2073                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2074                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2075                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2076
2077                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
2078                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
2079                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
2080
2081                 // We retry payments in `process_pending_htlc_forwards`
2082                 nodes[0].node.process_pending_htlc_forwards();
2083                 pass_failed_attempt_with_retry_along_path!(channel_id_3, false);
2084
2085                 // Ensure we won't retry a second time.
2086                 nodes[0].node.process_pending_htlc_forwards();
2087                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2088                 assert_eq!(msg_events.len(), 0);
2089         } else if test == AutoRetry::FailTimeout {
2090                 #[cfg(not(feature = "no-std"))] {
2091                         // Ensure ChannelManager will not retry a payment if it times out due to Retry::Timeout.
2092                         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2093                                 PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
2094                         pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2095
2096                         // Advance the time so the second attempt fails due to timeout.
2097                         SinceEpoch::advance(Duration::from_secs(61));
2098
2099                         // Make sure we don't retry again.
2100                         nodes[0].node.process_pending_htlc_forwards();
2101                         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2102                         assert_eq!(msg_events.len(), 0);
2103
2104                         let mut events = nodes[0].node.get_and_clear_pending_events();
2105                         assert_eq!(events.len(), 1);
2106                         match events[0] {
2107                                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2108                                         assert_eq!(payment_hash, *ev_payment_hash);
2109                                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2110                                         assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
2111                                 },
2112                                 _ => panic!("Unexpected event"),
2113                         }
2114                 }
2115         } else if test == AutoRetry::FailOnRestart {
2116                 // Ensure ChannelManager will not retry a payment after restart, even if there were retry
2117                 // attempts remaining prior to restart.
2118                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2119                         PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
2120                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2121
2122                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
2123                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
2124                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
2125
2126                 // Ensure the first retry attempt fails, with 1 retry attempt remaining
2127                 nodes[0].node.process_pending_htlc_forwards();
2128                 pass_failed_attempt_with_retry_along_path!(channel_id_3, true);
2129
2130                 // Restart the node and ensure that ChannelManager does not use its remaining retry attempt
2131                 let node_encoded = nodes[0].node.encode();
2132                 let chan_1_monitor_serialized = get_monitor!(nodes[0], channel_id_1).encode();
2133                 reload_node!(nodes[0], node_encoded, &[&chan_1_monitor_serialized], persister, new_chain_monitor, node_0_deserialized);
2134
2135                 let mut events = nodes[0].node.get_and_clear_pending_events();
2136                 expect_pending_htlcs_forwardable_from_events!(nodes[0], events, true);
2137                 // Make sure we don't retry again.
2138                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2139                 assert_eq!(msg_events.len(), 0);
2140
2141                 let mut events = nodes[0].node.get_and_clear_pending_events();
2142                 assert_eq!(events.len(), 1);
2143                 match events[0] {
2144                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2145                                 assert_eq!(payment_hash, *ev_payment_hash);
2146                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2147                                 assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
2148                         },
2149                         _ => panic!("Unexpected event"),
2150                 }
2151         } else if test == AutoRetry::FailOnRetry {
2152                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2153                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2154                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2155
2156                 // We retry payments in `process_pending_htlc_forwards`. Since our channel closed, we should
2157                 // fail to find a route.
2158                 nodes[0].node.process_pending_htlc_forwards();
2159                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2160                 assert_eq!(msg_events.len(), 0);
2161
2162                 let mut events = nodes[0].node.get_and_clear_pending_events();
2163                 assert_eq!(events.len(), 1);
2164                 match events[0] {
2165                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2166                                 assert_eq!(payment_hash, *ev_payment_hash);
2167                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2168                                 assert_eq!(PaymentFailureReason::RouteNotFound, ev_reason.unwrap());
2169                         },
2170                         _ => panic!("Unexpected event"),
2171                 }
2172         }
2173 }
2174
2175 #[test]
2176 fn auto_retry_partial_failure() {
2177         // Test that we'll retry appropriately on send partial failure and retry partial failure.
2178         let chanmon_cfgs = create_chanmon_cfgs(2);
2179         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2180         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2181         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2182
2183         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2184         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2185         let chan_3_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2186
2187         // Marshall data to send the payment
2188         let amt_msat = 20_000;
2189         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2190         #[cfg(feature = "std")]
2191         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2192         #[cfg(not(feature = "std"))]
2193         let payment_expiry_secs = 60 * 60;
2194         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2195         invoice_features.set_variable_length_onion_required();
2196         invoice_features.set_payment_secret_required();
2197         invoice_features.set_basic_mpp_optional();
2198         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2199                 .with_expiry_time(payment_expiry_secs as u64)
2200                 .with_bolt11_features(invoice_features).unwrap();
2201         let route_params = RouteParameters {
2202                 payment_params,
2203                 final_value_msat: amt_msat,
2204         };
2205
2206         // Ensure the first monitor update (for the initial send path1 over chan_1) succeeds, but the
2207         // second (for the initial send path2 over chan_2) fails.
2208         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
2209         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2210         // Ensure third monitor update (for the retry1's path1 over chan_1) succeeds, but the fourth (for
2211         // the retry1's path2 over chan_3) fails, and monitor updates succeed after that.
2212         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
2213         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2214         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
2215
2216         // Configure the initial send, retry1 and retry2's paths.
2217         let send_route = Route {
2218                 paths: vec![
2219                         Path { hops: vec![RouteHop {
2220                                 pubkey: nodes[1].node.get_our_node_id(),
2221                                 node_features: nodes[1].node.node_features(),
2222                                 short_channel_id: chan_1_id,
2223                                 channel_features: nodes[1].node.channel_features(),
2224                                 fee_msat: amt_msat / 2,
2225                                 cltv_expiry_delta: 100,
2226                         }], blinded_tail: None },
2227                         Path { hops: vec![RouteHop {
2228                                 pubkey: nodes[1].node.get_our_node_id(),
2229                                 node_features: nodes[1].node.node_features(),
2230                                 short_channel_id: chan_2_id,
2231                                 channel_features: nodes[1].node.channel_features(),
2232                                 fee_msat: amt_msat / 2,
2233                                 cltv_expiry_delta: 100,
2234                         }], blinded_tail: None },
2235                 ],
2236                 payment_params: Some(route_params.payment_params.clone()),
2237         };
2238         let retry_1_route = Route {
2239                 paths: vec![
2240                         Path { hops: vec![RouteHop {
2241                                 pubkey: nodes[1].node.get_our_node_id(),
2242                                 node_features: nodes[1].node.node_features(),
2243                                 short_channel_id: chan_1_id,
2244                                 channel_features: nodes[1].node.channel_features(),
2245                                 fee_msat: amt_msat / 4,
2246                                 cltv_expiry_delta: 100,
2247                         }], blinded_tail: None },
2248                         Path { hops: vec![RouteHop {
2249                                 pubkey: nodes[1].node.get_our_node_id(),
2250                                 node_features: nodes[1].node.node_features(),
2251                                 short_channel_id: chan_3_id,
2252                                 channel_features: nodes[1].node.channel_features(),
2253                                 fee_msat: amt_msat / 4,
2254                                 cltv_expiry_delta: 100,
2255                         }], blinded_tail: None },
2256                 ],
2257                 payment_params: Some(route_params.payment_params.clone()),
2258         };
2259         let retry_2_route = Route {
2260                 paths: vec![
2261                         Path { hops: vec![RouteHop {
2262                                 pubkey: nodes[1].node.get_our_node_id(),
2263                                 node_features: nodes[1].node.node_features(),
2264                                 short_channel_id: chan_1_id,
2265                                 channel_features: nodes[1].node.channel_features(),
2266                                 fee_msat: amt_msat / 4,
2267                                 cltv_expiry_delta: 100,
2268                         }], blinded_tail: None },
2269                 ],
2270                 payment_params: Some(route_params.payment_params.clone()),
2271         };
2272         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
2273         let mut payment_params = route_params.payment_params.clone();
2274         payment_params.previously_failed_channels.push(chan_2_id);
2275         nodes[0].router.expect_find_route(RouteParameters {
2276                         payment_params, final_value_msat: amt_msat / 2,
2277                 }, Ok(retry_1_route));
2278         let mut payment_params = route_params.payment_params.clone();
2279         payment_params.previously_failed_channels.push(chan_3_id);
2280         nodes[0].router.expect_find_route(RouteParameters {
2281                         payment_params, final_value_msat: amt_msat / 4,
2282                 }, Ok(retry_2_route));
2283
2284         // Send a payment that will partially fail on send, then partially fail on retry, then succeed.
2285         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2286                 PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap();
2287         let closed_chan_events = nodes[0].node.get_and_clear_pending_events();
2288         assert_eq!(closed_chan_events.len(), 4);
2289         match closed_chan_events[0] {
2290                 Event::ChannelClosed { .. } => {},
2291                 _ => panic!("Unexpected event"),
2292         }
2293         match closed_chan_events[1] {
2294                 Event::PaymentPathFailed { .. } => {},
2295                 _ => panic!("Unexpected event"),
2296         }
2297         match closed_chan_events[2] {
2298                 Event::ChannelClosed { .. } => {},
2299                 _ => panic!("Unexpected event"),
2300         }
2301         match closed_chan_events[3] {
2302                 Event::PaymentPathFailed { .. } => {},
2303                 _ => panic!("Unexpected event"),
2304         }
2305
2306         // Pass the first part of the payment along the path.
2307         check_added_monitors!(nodes[0], 5); // three outbound channel updates succeeded, two permanently failed
2308         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2309
2310         // First message is the first update_add, remaining messages are broadcasting channel updates and
2311         // errors for the permfailed channels
2312         assert_eq!(msg_events.len(), 5);
2313         let mut payment_event = SendEvent::from_event(msg_events.remove(0));
2314
2315         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2316         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
2317         check_added_monitors!(nodes[1], 1);
2318         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2319
2320         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2321         check_added_monitors!(nodes[0], 1);
2322         let as_second_htlc_updates = SendEvent::from_node(&nodes[0]);
2323
2324         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2325         check_added_monitors!(nodes[0], 1);
2326         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2327
2328         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2329         check_added_monitors!(nodes[1], 1);
2330
2331         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[0]);
2332         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[1]);
2333         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.commitment_msg);
2334         check_added_monitors!(nodes[1], 1);
2335         let (bs_second_raa, bs_second_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2336
2337         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2338         check_added_monitors!(nodes[0], 1);
2339
2340         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_cs);
2341         check_added_monitors!(nodes[0], 1);
2342         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2343
2344         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2345         check_added_monitors!(nodes[1], 1);
2346
2347         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
2348         nodes[1].node.process_pending_htlc_forwards();
2349         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, amt_msat);
2350         nodes[1].node.claim_funds(payment_preimage);
2351         expect_payment_claimed!(nodes[1], payment_hash, amt_msat);
2352         let bs_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2353         assert_eq!(bs_claim_update.update_fulfill_htlcs.len(), 1);
2354
2355         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_claim_update.update_fulfill_htlcs[0]);
2356         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_claim_update.commitment_signed);
2357         check_added_monitors!(nodes[0], 1);
2358         let (as_third_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2359
2360         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2361         check_added_monitors!(nodes[1], 4);
2362         let bs_second_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2363
2364         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2365         check_added_monitors!(nodes[1], 1);
2366         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2367
2368         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2369         check_added_monitors!(nodes[0], 1);
2370
2371         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[0]);
2372         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[1]);
2373         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.commitment_signed);
2374         check_added_monitors!(nodes[0], 1);
2375         let (as_fourth_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2376
2377         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_fourth_raa);
2378         check_added_monitors!(nodes[1], 1);
2379
2380         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2381         check_added_monitors!(nodes[1], 1);
2382         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2383
2384         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2385         check_added_monitors!(nodes[0], 1);
2386         expect_payment_sent!(nodes[0], payment_preimage);
2387 }
2388
2389 #[test]
2390 fn auto_retry_zero_attempts_send_error() {
2391         let chanmon_cfgs = create_chanmon_cfgs(2);
2392         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2393         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2394         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2395
2396         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2397         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2398
2399         // Marshall data to send the payment
2400         let amt_msat = 20_000;
2401         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2402         #[cfg(feature = "std")]
2403         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2404         #[cfg(not(feature = "std"))]
2405         let payment_expiry_secs = 60 * 60;
2406         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2407         invoice_features.set_variable_length_onion_required();
2408         invoice_features.set_payment_secret_required();
2409         invoice_features.set_basic_mpp_optional();
2410         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2411                 .with_expiry_time(payment_expiry_secs as u64)
2412                 .with_bolt11_features(invoice_features).unwrap();
2413         let route_params = RouteParameters {
2414                 payment_params,
2415                 final_value_msat: amt_msat,
2416         };
2417
2418         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2419         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2420                 PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
2421         assert_eq!(nodes[0].node.get_and_clear_pending_msg_events().len(), 2); // channel close messages
2422         let events = nodes[0].node.get_and_clear_pending_events();
2423         assert_eq!(events.len(), 3);
2424         if let Event::ChannelClosed { .. } = events[0] { } else { panic!(); }
2425         if let Event::PaymentPathFailed { .. } = events[1] { } else { panic!(); }
2426         if let Event::PaymentFailed { .. } = events[2] { } else { panic!(); }
2427         check_added_monitors!(nodes[0], 2);
2428 }
2429
2430 #[test]
2431 fn fails_paying_after_rejected_by_payee() {
2432         let chanmon_cfgs = create_chanmon_cfgs(2);
2433         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2434         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2435         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2436
2437         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2438
2439         // Marshall data to send the payment
2440         let amt_msat = 20_000;
2441         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2442         #[cfg(feature = "std")]
2443         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2444         #[cfg(not(feature = "std"))]
2445         let payment_expiry_secs = 60 * 60;
2446         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2447         invoice_features.set_variable_length_onion_required();
2448         invoice_features.set_payment_secret_required();
2449         invoice_features.set_basic_mpp_optional();
2450         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2451                 .with_expiry_time(payment_expiry_secs as u64)
2452                 .with_bolt11_features(invoice_features).unwrap();
2453         let route_params = RouteParameters {
2454                 payment_params,
2455                 final_value_msat: amt_msat,
2456         };
2457
2458         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2459                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2460         check_added_monitors!(nodes[0], 1);
2461         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
2462         assert_eq!(events.len(), 1);
2463         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
2464         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2465         check_added_monitors!(nodes[1], 0);
2466         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
2467         expect_pending_htlcs_forwardable!(nodes[1]);
2468         expect_payment_claimable!(&nodes[1], payment_hash, payment_secret, amt_msat);
2469
2470         nodes[1].node.fail_htlc_backwards(&payment_hash);
2471         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash }]);
2472         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
2473 }
2474
2475 #[test]
2476 fn retry_multi_path_single_failed_payment() {
2477         // Tests that we can/will retry after a single path of an MPP payment failed immediately
2478         let chanmon_cfgs = create_chanmon_cfgs(2);
2479         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2480         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2481         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2482
2483         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2484         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2485
2486         let amt_msat = 100_010_000;
2487
2488         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2489         #[cfg(feature = "std")]
2490         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2491         #[cfg(not(feature = "std"))]
2492         let payment_expiry_secs = 60 * 60;
2493         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2494         invoice_features.set_variable_length_onion_required();
2495         invoice_features.set_payment_secret_required();
2496         invoice_features.set_basic_mpp_optional();
2497         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2498                 .with_expiry_time(payment_expiry_secs as u64)
2499                 .with_bolt11_features(invoice_features).unwrap();
2500         let route_params = RouteParameters {
2501                 payment_params: payment_params.clone(),
2502                 final_value_msat: amt_msat,
2503         };
2504
2505         let chans = nodes[0].node.list_usable_channels();
2506         let mut route = Route {
2507                 paths: vec![
2508                         Path { hops: vec![RouteHop {
2509                                 pubkey: nodes[1].node.get_our_node_id(),
2510                                 node_features: nodes[1].node.node_features(),
2511                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2512                                 channel_features: nodes[1].node.channel_features(),
2513                                 fee_msat: 10_000,
2514                                 cltv_expiry_delta: 100,
2515                         }], blinded_tail: None },
2516                         Path { hops: vec![RouteHop {
2517                                 pubkey: nodes[1].node.get_our_node_id(),
2518                                 node_features: nodes[1].node.node_features(),
2519                                 short_channel_id: chans[1].short_channel_id.unwrap(),
2520                                 channel_features: nodes[1].node.channel_features(),
2521                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2522                                 cltv_expiry_delta: 100,
2523                         }], blinded_tail: None },
2524                 ],
2525                 payment_params: Some(payment_params),
2526         };
2527         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2528         // On retry, split the payment across both channels.
2529         route.paths[0].hops[0].fee_msat = 50_000_001;
2530         route.paths[1].hops[0].fee_msat = 50_000_000;
2531         let mut pay_params = route.payment_params.clone().unwrap();
2532         pay_params.previously_failed_channels.push(chans[1].short_channel_id.unwrap());
2533         nodes[0].router.expect_find_route(RouteParameters {
2534                         payment_params: pay_params,
2535                         // Note that the second request here requests the amount we originally failed to send,
2536                         // not the amount remaining on the full payment, which should be changed.
2537                         final_value_msat: 100_000_001,
2538                 }, Ok(route.clone()));
2539
2540         {
2541                 let scorer = chanmon_cfgs[0].scorer.lock().unwrap();
2542                 // The initial send attempt, 2 paths
2543                 scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 10_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2544                 scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 100_000_001, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2545                 // The retry, 2 paths. Ensure that the in-flight HTLC amount is factored in.
2546                 scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_001, inflight_htlc_msat: 10_000, effective_capacity: EffectiveCapacity::Unknown });
2547                 scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2548         }
2549
2550         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2551                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2552         let events = nodes[0].node.get_and_clear_pending_events();
2553         assert_eq!(events.len(), 1);
2554         match events[0] {
2555                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false,
2556                         failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }},
2557                         short_channel_id: Some(expected_scid), .. } =>
2558                 {
2559                         assert_eq!(payment_hash, ev_payment_hash);
2560                         assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
2561                 },
2562                 _ => panic!("Unexpected event"),
2563         }
2564         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2565         assert_eq!(htlc_msgs.len(), 2);
2566         check_added_monitors!(nodes[0], 2);
2567 }
2568
2569 #[test]
2570 fn immediate_retry_on_failure() {
2571         // Tests that we can/will retry immediately after a failure
2572         let chanmon_cfgs = create_chanmon_cfgs(2);
2573         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2574         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2575         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2576
2577         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2578         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2579
2580         let amt_msat = 100_000_001;
2581         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2582         #[cfg(feature = "std")]
2583         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2584         #[cfg(not(feature = "std"))]
2585         let payment_expiry_secs = 60 * 60;
2586         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2587         invoice_features.set_variable_length_onion_required();
2588         invoice_features.set_payment_secret_required();
2589         invoice_features.set_basic_mpp_optional();
2590         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2591                 .with_expiry_time(payment_expiry_secs as u64)
2592                 .with_bolt11_features(invoice_features).unwrap();
2593         let route_params = RouteParameters {
2594                 payment_params,
2595                 final_value_msat: amt_msat,
2596         };
2597
2598         let chans = nodes[0].node.list_usable_channels();
2599         let mut route = Route {
2600                 paths: vec![
2601                         Path { hops: vec![RouteHop {
2602                                 pubkey: nodes[1].node.get_our_node_id(),
2603                                 node_features: nodes[1].node.node_features(),
2604                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2605                                 channel_features: nodes[1].node.channel_features(),
2606                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2607                                 cltv_expiry_delta: 100,
2608                         }], blinded_tail: None },
2609                 ],
2610                 payment_params: Some(PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)),
2611         };
2612         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2613         // On retry, split the payment across both channels.
2614         route.paths.push(route.paths[0].clone());
2615         route.paths[0].hops[0].short_channel_id = chans[1].short_channel_id.unwrap();
2616         route.paths[0].hops[0].fee_msat = 50_000_000;
2617         route.paths[1].hops[0].fee_msat = 50_000_001;
2618         let mut pay_params = route_params.payment_params.clone();
2619         pay_params.previously_failed_channels.push(chans[0].short_channel_id.unwrap());
2620         nodes[0].router.expect_find_route(RouteParameters {
2621                         payment_params: pay_params, final_value_msat: amt_msat,
2622                 }, Ok(route.clone()));
2623
2624         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2625                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2626         let events = nodes[0].node.get_and_clear_pending_events();
2627         assert_eq!(events.len(), 1);
2628         match events[0] {
2629                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false,
2630                         failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }},
2631                         short_channel_id: Some(expected_scid), .. } =>
2632                 {
2633                         assert_eq!(payment_hash, ev_payment_hash);
2634                         assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
2635                 },
2636                 _ => panic!("Unexpected event"),
2637         }
2638         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2639         assert_eq!(htlc_msgs.len(), 2);
2640         check_added_monitors!(nodes[0], 2);
2641 }
2642
2643 #[test]
2644 fn no_extra_retries_on_back_to_back_fail() {
2645         // In a previous release, we had a race where we may exceed the payment retry count if we
2646         // get two failures in a row with the second indicating that all paths had failed (this field,
2647         // `all_paths_failed`, has since been removed).
2648         // Generally, when we give up trying to retry a payment, we don't know for sure what the
2649         // current state of the ChannelManager event queue is. Specifically, we cannot be sure that
2650         // there are not multiple additional `PaymentPathFailed` or even `PaymentSent` events
2651         // pending which we will see later. Thus, when we previously removed the retry tracking map
2652         // entry after a `all_paths_failed` `PaymentPathFailed` event, we may have dropped the
2653         // retry entry even though more events for the same payment were still pending. This led to
2654         // us retrying a payment again even though we'd already given up on it.
2655         //
2656         // We now have a separate event - `PaymentFailed` which indicates no HTLCs remain and which
2657         // is used to remove the payment retry counter entries instead. This tests for the specific
2658         // excess-retry case while also testing `PaymentFailed` generation.
2659
2660         let chanmon_cfgs = create_chanmon_cfgs(3);
2661         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2662         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2663         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2664
2665         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2666         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2667
2668         let amt_msat = 200_000_000;
2669         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2670         #[cfg(feature = "std")]
2671         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2672         #[cfg(not(feature = "std"))]
2673         let payment_expiry_secs = 60 * 60;
2674         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2675         invoice_features.set_variable_length_onion_required();
2676         invoice_features.set_payment_secret_required();
2677         invoice_features.set_basic_mpp_optional();
2678         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2679                 .with_expiry_time(payment_expiry_secs as u64)
2680                 .with_bolt11_features(invoice_features).unwrap();
2681         let route_params = RouteParameters {
2682                 payment_params,
2683                 final_value_msat: amt_msat,
2684         };
2685
2686         let mut route = Route {
2687                 paths: vec![
2688                         Path { hops: vec![RouteHop {
2689                                 pubkey: nodes[1].node.get_our_node_id(),
2690                                 node_features: nodes[1].node.node_features(),
2691                                 short_channel_id: chan_1_scid,
2692                                 channel_features: nodes[1].node.channel_features(),
2693                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2694                                 cltv_expiry_delta: 100,
2695                         }, RouteHop {
2696                                 pubkey: nodes[2].node.get_our_node_id(),
2697                                 node_features: nodes[2].node.node_features(),
2698                                 short_channel_id: chan_2_scid,
2699                                 channel_features: nodes[2].node.channel_features(),
2700                                 fee_msat: 100_000_000,
2701                                 cltv_expiry_delta: 100,
2702                         }], blinded_tail: None },
2703                         Path { hops: vec![RouteHop {
2704                                 pubkey: nodes[1].node.get_our_node_id(),
2705                                 node_features: nodes[1].node.node_features(),
2706                                 short_channel_id: chan_1_scid,
2707                                 channel_features: nodes[1].node.channel_features(),
2708                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2709                                 cltv_expiry_delta: 100,
2710                         }, RouteHop {
2711                                 pubkey: nodes[2].node.get_our_node_id(),
2712                                 node_features: nodes[2].node.node_features(),
2713                                 short_channel_id: chan_2_scid,
2714                                 channel_features: nodes[2].node.channel_features(),
2715                                 fee_msat: 100_000_000,
2716                                 cltv_expiry_delta: 100,
2717                         }], blinded_tail: None }
2718                 ],
2719                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2720         };
2721         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2722         let mut second_payment_params = route_params.payment_params.clone();
2723         second_payment_params.previously_failed_channels = vec![chan_2_scid, chan_2_scid];
2724         // On retry, we'll only return one path
2725         route.paths.remove(1);
2726         route.paths[0].hops[1].fee_msat = amt_msat;
2727         nodes[0].router.expect_find_route(RouteParameters {
2728                         payment_params: second_payment_params,
2729                         final_value_msat: amt_msat,
2730                 }, Ok(route.clone()));
2731
2732         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2733                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2734         let htlc_updates = SendEvent::from_node(&nodes[0]);
2735         check_added_monitors!(nodes[0], 1);
2736         assert_eq!(htlc_updates.msgs.len(), 1);
2737
2738         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2739         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2740         check_added_monitors!(nodes[1], 1);
2741         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2742
2743         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2744         check_added_monitors!(nodes[0], 1);
2745         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2746
2747         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2748         check_added_monitors!(nodes[0], 1);
2749         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2750
2751         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2752         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2753         check_added_monitors!(nodes[1], 1);
2754         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2755
2756         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2757         check_added_monitors!(nodes[1], 1);
2758         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2759
2760         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2761         check_added_monitors!(nodes[0], 1);
2762
2763         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2764         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2765         check_added_monitors!(nodes[0], 1);
2766         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2767
2768         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2769         check_added_monitors!(nodes[1], 1);
2770         let bs_second_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2771
2772         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2773         check_added_monitors!(nodes[1], 1);
2774         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2775
2776         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.update_fail_htlcs[0]);
2777         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.commitment_signed);
2778         check_added_monitors!(nodes[0], 1);
2779
2780         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2781         check_added_monitors!(nodes[0], 1);
2782         let (as_third_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2783
2784         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2785         check_added_monitors!(nodes[1], 1);
2786         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2787         check_added_monitors!(nodes[1], 1);
2788         let bs_fourth_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2789
2790         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_fourth_raa);
2791         check_added_monitors!(nodes[0], 1);
2792
2793         // At this point A has sent two HTLCs which both failed due to lack of fee. It now has two
2794         // pending `PaymentPathFailed` events, one with `all_paths_failed` unset, and the second
2795         // with it set.
2796         //
2797         // Previously, we retried payments in an event consumer, which would retry each
2798         // `PaymentPathFailed` individually. In that setup, we had retried the payment in response to
2799         // the first `PaymentPathFailed`, then seen the second `PaymentPathFailed` with
2800         // `all_paths_failed` set and assumed the payment was completely failed. We ultimately fixed it
2801         // by adding the `PaymentFailed` event.
2802         //
2803         // Because we now retry payments as a batch, we simply return a single-path route in the
2804         // second, batched, request, have that fail, ensure the payment was abandoned.
2805         let mut events = nodes[0].node.get_and_clear_pending_events();
2806         assert_eq!(events.len(), 3);
2807         match events[0] {
2808                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2809                         assert_eq!(payment_hash, ev_payment_hash);
2810                         assert_eq!(payment_failed_permanently, false);
2811                 },
2812                 _ => panic!("Unexpected event"),
2813         }
2814         match events[1] {
2815                 Event::PendingHTLCsForwardable { .. } => {},
2816                 _ => panic!("Unexpected event"),
2817         }
2818         match events[2] {
2819                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2820                         assert_eq!(payment_hash, ev_payment_hash);
2821                         assert_eq!(payment_failed_permanently, false);
2822                 },
2823                 _ => panic!("Unexpected event"),
2824         }
2825
2826         nodes[0].node.process_pending_htlc_forwards();
2827         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2828         check_added_monitors!(nodes[0], 1);
2829
2830         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2831         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2832         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2833         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2834         commitment_signed_dance!(nodes[0], nodes[1], &bs_fail_update.commitment_signed, false, true);
2835
2836         let mut events = nodes[0].node.get_and_clear_pending_events();
2837         assert_eq!(events.len(), 2);
2838         match events[0] {
2839                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2840                         assert_eq!(payment_hash, ev_payment_hash);
2841                         assert_eq!(payment_failed_permanently, false);
2842                 },
2843                 _ => panic!("Unexpected event"),
2844         }
2845         match events[1] {
2846                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2847                         assert_eq!(payment_hash, *ev_payment_hash);
2848                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2849                         assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
2850                 },
2851                 _ => panic!("Unexpected event"),
2852         }
2853 }
2854
2855 #[test]
2856 fn test_simple_partial_retry() {
2857         // In the first version of the in-`ChannelManager` payment retries, retries were sent for the
2858         // full amount of the payment, rather than only the missing amount. Here we simply test for
2859         // this by sending a payment with two parts, failing one, and retrying the second. Note that
2860         // `TestRouter` will check that the `RouteParameters` (which contain the amount) matches the
2861         // request.
2862         let chanmon_cfgs = create_chanmon_cfgs(3);
2863         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2864         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2865         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2866
2867         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2868         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2869
2870         let amt_msat = 200_000_000;
2871         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
2872         #[cfg(feature = "std")]
2873         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2874         #[cfg(not(feature = "std"))]
2875         let payment_expiry_secs = 60 * 60;
2876         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2877         invoice_features.set_variable_length_onion_required();
2878         invoice_features.set_payment_secret_required();
2879         invoice_features.set_basic_mpp_optional();
2880         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2881                 .with_expiry_time(payment_expiry_secs as u64)
2882                 .with_bolt11_features(invoice_features).unwrap();
2883         let route_params = RouteParameters {
2884                 payment_params,
2885                 final_value_msat: amt_msat,
2886         };
2887
2888         let mut route = Route {
2889                 paths: vec![
2890                         Path { hops: vec![RouteHop {
2891                                 pubkey: nodes[1].node.get_our_node_id(),
2892                                 node_features: nodes[1].node.node_features(),
2893                                 short_channel_id: chan_1_scid,
2894                                 channel_features: nodes[1].node.channel_features(),
2895                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2896                                 cltv_expiry_delta: 100,
2897                         }, RouteHop {
2898                                 pubkey: nodes[2].node.get_our_node_id(),
2899                                 node_features: nodes[2].node.node_features(),
2900                                 short_channel_id: chan_2_scid,
2901                                 channel_features: nodes[2].node.channel_features(),
2902                                 fee_msat: 100_000_000,
2903                                 cltv_expiry_delta: 100,
2904                         }], blinded_tail: None },
2905                         Path { hops: vec![RouteHop {
2906                                 pubkey: nodes[1].node.get_our_node_id(),
2907                                 node_features: nodes[1].node.node_features(),
2908                                 short_channel_id: chan_1_scid,
2909                                 channel_features: nodes[1].node.channel_features(),
2910                                 fee_msat: 100_000,
2911                                 cltv_expiry_delta: 100,
2912                         }, RouteHop {
2913                                 pubkey: nodes[2].node.get_our_node_id(),
2914                                 node_features: nodes[2].node.node_features(),
2915                                 short_channel_id: chan_2_scid,
2916                                 channel_features: nodes[2].node.channel_features(),
2917                                 fee_msat: 100_000_000,
2918                                 cltv_expiry_delta: 100,
2919                         }], blinded_tail: None }
2920                 ],
2921                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2922         };
2923         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2924         let mut second_payment_params = route_params.payment_params.clone();
2925         second_payment_params.previously_failed_channels = vec![chan_2_scid];
2926         // On retry, we'll only be asked for one path (or 100k sats)
2927         route.paths.remove(0);
2928         nodes[0].router.expect_find_route(RouteParameters {
2929                         payment_params: second_payment_params,
2930                         final_value_msat: amt_msat / 2,
2931                 }, Ok(route.clone()));
2932
2933         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2934                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2935         let htlc_updates = SendEvent::from_node(&nodes[0]);
2936         check_added_monitors!(nodes[0], 1);
2937         assert_eq!(htlc_updates.msgs.len(), 1);
2938
2939         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2940         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2941         check_added_monitors!(nodes[1], 1);
2942         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2943
2944         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2945         check_added_monitors!(nodes[0], 1);
2946         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2947
2948         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2949         check_added_monitors!(nodes[0], 1);
2950         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2951
2952         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2953         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2954         check_added_monitors!(nodes[1], 1);
2955         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2956
2957         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2958         check_added_monitors!(nodes[1], 1);
2959         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2960
2961         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2962         check_added_monitors!(nodes[0], 1);
2963
2964         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2965         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2966         check_added_monitors!(nodes[0], 1);
2967         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2968
2969         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2970         check_added_monitors!(nodes[1], 1);
2971
2972         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2973         check_added_monitors!(nodes[1], 1);
2974
2975         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2976
2977         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2978         check_added_monitors!(nodes[0], 1);
2979
2980         let mut events = nodes[0].node.get_and_clear_pending_events();
2981         assert_eq!(events.len(), 2);
2982         match events[0] {
2983                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2984                         assert_eq!(payment_hash, ev_payment_hash);
2985                         assert_eq!(payment_failed_permanently, false);
2986                 },
2987                 _ => panic!("Unexpected event"),
2988         }
2989         match events[1] {
2990                 Event::PendingHTLCsForwardable { .. } => {},
2991                 _ => panic!("Unexpected event"),
2992         }
2993
2994         nodes[0].node.process_pending_htlc_forwards();
2995         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2996         check_added_monitors!(nodes[0], 1);
2997
2998         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2999         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
3000
3001         expect_pending_htlcs_forwardable!(nodes[1]);
3002         check_added_monitors!(nodes[1], 1);
3003
3004         let bs_forward_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
3005         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[0]);
3006         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[1]);
3007         commitment_signed_dance!(nodes[2], nodes[1], &bs_forward_update.commitment_signed, false);
3008
3009         expect_pending_htlcs_forwardable!(nodes[2]);
3010         expect_payment_claimable!(nodes[2], payment_hash, payment_secret, amt_msat);
3011 }
3012
3013 #[test]
3014 #[cfg(feature = "std")]
3015 fn test_threaded_payment_retries() {
3016         // In the first version of the in-`ChannelManager` payment retries, retries weren't limited to
3017         // a single thread and would happily let multiple threads run retries at the same time. Because
3018         // retries are done by first calculating the amount we need to retry, then dropping the
3019         // relevant lock, then actually sending, we would happily let multiple threads retry the same
3020         // amount at the same time, overpaying our original HTLC!
3021         let chanmon_cfgs = create_chanmon_cfgs(4);
3022         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3023         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3024         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3025
3026         // There is one mitigating guardrail when retrying payments - we can never over-pay by more
3027         // than 10% of the original value. Thus, we want all our retries to be below that. In order to
3028         // keep things simple, we route one HTLC for 0.1% of the payment over channel 1 and the rest
3029         // out over channel 3+4. This will let us ignore 99% of the payment value and deal with only
3030         // our channel.
3031         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
3032         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 10_000_000, 0);
3033         let chan_3_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 10_000_000, 0).0.contents.short_channel_id;
3034         let chan_4_scid = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 10_000_000, 0).0.contents.short_channel_id;
3035
3036         let amt_msat = 100_000_000;
3037         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
3038         #[cfg(feature = "std")]
3039         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
3040         #[cfg(not(feature = "std"))]
3041         let payment_expiry_secs = 60 * 60;
3042         let mut invoice_features = Bolt11InvoiceFeatures::empty();
3043         invoice_features.set_variable_length_onion_required();
3044         invoice_features.set_payment_secret_required();
3045         invoice_features.set_basic_mpp_optional();
3046         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
3047                 .with_expiry_time(payment_expiry_secs as u64)
3048                 .with_bolt11_features(invoice_features).unwrap();
3049         let mut route_params = RouteParameters {
3050                 payment_params,
3051                 final_value_msat: amt_msat,
3052         };
3053
3054         let mut route = Route {
3055                 paths: vec![
3056                         Path { hops: vec![RouteHop {
3057                                 pubkey: nodes[1].node.get_our_node_id(),
3058                                 node_features: nodes[1].node.node_features(),
3059                                 short_channel_id: chan_1_scid,
3060                                 channel_features: nodes[1].node.channel_features(),
3061                                 fee_msat: 0,
3062                                 cltv_expiry_delta: 100,
3063                         }, RouteHop {
3064                                 pubkey: nodes[3].node.get_our_node_id(),
3065                                 node_features: nodes[2].node.node_features(),
3066                                 short_channel_id: 42, // Set a random SCID which nodes[1] will fail as unknown
3067                                 channel_features: nodes[2].node.channel_features(),
3068                                 fee_msat: amt_msat / 1000,
3069                                 cltv_expiry_delta: 100,
3070                         }], blinded_tail: None },
3071                         Path { hops: vec![RouteHop {
3072                                 pubkey: nodes[2].node.get_our_node_id(),
3073                                 node_features: nodes[2].node.node_features(),
3074                                 short_channel_id: chan_3_scid,
3075                                 channel_features: nodes[2].node.channel_features(),
3076                                 fee_msat: 100_000,
3077                                 cltv_expiry_delta: 100,
3078                         }, RouteHop {
3079                                 pubkey: nodes[3].node.get_our_node_id(),
3080                                 node_features: nodes[3].node.node_features(),
3081                                 short_channel_id: chan_4_scid,
3082                                 channel_features: nodes[3].node.channel_features(),
3083                                 fee_msat: amt_msat - amt_msat / 1000,
3084                                 cltv_expiry_delta: 100,
3085                         }], blinded_tail: None }
3086                 ],
3087                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
3088         };
3089         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3090
3091         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3092                 PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(0xdeadbeef)).unwrap();
3093         check_added_monitors!(nodes[0], 2);
3094         let mut send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
3095         assert_eq!(send_msg_events.len(), 2);
3096         send_msg_events.retain(|msg|
3097                 if let MessageSendEvent::UpdateHTLCs { node_id, .. } = msg {
3098                         // Drop the commitment update for nodes[2], we can just let that one sit pending
3099                         // forever.
3100                         *node_id == nodes[1].node.get_our_node_id()
3101                 } else { panic!(); }
3102         );
3103
3104         // from here on out, the retry `RouteParameters` amount will be amt/1000
3105         route_params.final_value_msat /= 1000;
3106         route.paths.pop();
3107
3108         let end_time = Instant::now() + Duration::from_secs(1);
3109         macro_rules! thread_body { () => { {
3110                 // We really want std::thread::scope, but its not stable until 1.63. Until then, we get unsafe.
3111                 let node_ref = NodePtr::from_node(&nodes[0]);
3112                 move || {
3113                         let node_a = unsafe { &*node_ref.0 };
3114                         while Instant::now() < end_time {
3115                                 node_a.node.get_and_clear_pending_events(); // wipe the PendingHTLCsForwardable
3116                                 // Ignore if we have any pending events, just always pretend we just got a
3117                                 // PendingHTLCsForwardable
3118                                 node_a.node.process_pending_htlc_forwards();
3119                         }
3120                 }
3121         } } }
3122         let mut threads = Vec::new();
3123         for _ in 0..16 { threads.push(std::thread::spawn(thread_body!())); }
3124
3125         // Back in the main thread, poll pending messages and make sure that we never have more than
3126         // one HTLC pending at a time. Note that the commitment_signed_dance will fail horribly if
3127         // there are HTLC messages shoved in while its running. This allows us to test that we never
3128         // generate an additional update_add_htlc until we've fully failed the first.
3129         let mut previously_failed_channels = Vec::new();
3130         loop {
3131                 assert_eq!(send_msg_events.len(), 1);
3132                 let send_event = SendEvent::from_event(send_msg_events.pop().unwrap());
3133                 assert_eq!(send_event.msgs.len(), 1);
3134
3135                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
3136                 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
3137
3138                 // Note that we only push one route into `expect_find_route` at a time, because that's all
3139                 // the retries (should) need. If the bug is reintroduced "real" routes may be selected, but
3140                 // we should still ultimately fail for the same reason - because we're trying to send too
3141                 // many HTLCs at once.
3142                 let mut new_route_params = route_params.clone();
3143                 previously_failed_channels.push(route.paths[0].hops[1].short_channel_id);
3144                 new_route_params.payment_params.previously_failed_channels = previously_failed_channels.clone();
3145                 route.paths[0].hops[1].short_channel_id += 1;
3146                 nodes[0].router.expect_find_route(new_route_params, Ok(route.clone()));
3147
3148                 let bs_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3149                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_updates.update_fail_htlcs[0]);
3150                 // The "normal" commitment_signed_dance delivers the final RAA and then calls
3151                 // `check_added_monitors` to ensure only the one RAA-generated monitor update was created.
3152                 // This races with our other threads which may generate an add-HTLCs commitment update via
3153                 // `process_pending_htlc_forwards`. Instead, we defer the monitor update check until after
3154                 // *we've* called `process_pending_htlc_forwards` when its guaranteed to have two updates.
3155                 let last_raa = commitment_signed_dance!(nodes[0], nodes[1], bs_fail_updates.commitment_signed, false, true, false, true);
3156                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &last_raa);
3157
3158                 let cur_time = Instant::now();
3159                 if cur_time > end_time {
3160                         for thread in threads.drain(..) { thread.join().unwrap(); }
3161                 }
3162
3163                 // Make sure we have some events to handle when we go around...
3164                 nodes[0].node.get_and_clear_pending_events(); // wipe the PendingHTLCsForwardable
3165                 nodes[0].node.process_pending_htlc_forwards();
3166                 send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
3167                 check_added_monitors!(nodes[0], 2);
3168
3169                 if cur_time > end_time {
3170                         break;
3171                 }
3172         }
3173 }
3174
3175 fn do_no_missing_sent_on_midpoint_reload(persist_manager_with_payment: bool) {
3176         // Test that if we reload in the middle of an HTLC claim commitment signed dance we'll still
3177         // receive the PaymentSent event even if the ChannelManager had no idea about the payment when
3178         // it was last persisted.
3179         let chanmon_cfgs = create_chanmon_cfgs(2);
3180         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
3181         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
3182         let (persister_a, persister_b, persister_c);
3183         let (chain_monitor_a, chain_monitor_b, chain_monitor_c);
3184         let (nodes_0_deserialized, nodes_0_deserialized_b, nodes_0_deserialized_c);
3185         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
3186
3187         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
3188
3189         let mut nodes_0_serialized = Vec::new();
3190         if !persist_manager_with_payment {
3191                 nodes_0_serialized = nodes[0].node.encode();
3192         }
3193
3194         let (our_payment_preimage, our_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 1_000_000);
3195
3196         if persist_manager_with_payment {
3197                 nodes_0_serialized = nodes[0].node.encode();
3198         }
3199
3200         nodes[1].node.claim_funds(our_payment_preimage);
3201         check_added_monitors!(nodes[1], 1);
3202         expect_payment_claimed!(nodes[1], our_payment_hash, 1_000_000);
3203
3204         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3205         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3206         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
3207         check_added_monitors!(nodes[0], 1);
3208
3209         // The ChannelMonitor should always be the latest version, as we're required to persist it
3210         // during the commitment signed handling.
3211         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3212         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister_a, chain_monitor_a, nodes_0_deserialized);
3213
3214         let events = nodes[0].node.get_and_clear_pending_events();
3215         assert_eq!(events.len(), 2);
3216         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[0] {} else { panic!(); }
3217         if let Event::PaymentSent { payment_preimage, .. } = events[1] { assert_eq!(payment_preimage, our_payment_preimage); } else { panic!(); }
3218         // Note that we don't get a PaymentPathSuccessful here as we leave the HTLC pending to avoid
3219         // the double-claim that would otherwise appear at the end of this test.
3220         nodes[0].node.timer_tick_occurred();
3221         let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3222         assert_eq!(as_broadcasted_txn.len(), 1);
3223
3224         // Ensure that, even after some time, if we restart we still include *something* in the current
3225         // `ChannelManager` which prevents a `PaymentFailed` when we restart even if pending resolved
3226         // payments have since been timed out thanks to `IDEMPOTENCY_TIMEOUT_TICKS`.
3227         // A naive implementation of the fix here would wipe the pending payments set, causing a
3228         // failure event when we restart.
3229         for _ in 0..(IDEMPOTENCY_TIMEOUT_TICKS * 2) { nodes[0].node.timer_tick_occurred(); }
3230
3231         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3232         reload_node!(nodes[0], test_default_channel_config(), &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister_b, chain_monitor_b, nodes_0_deserialized_b);
3233         let events = nodes[0].node.get_and_clear_pending_events();
3234         assert!(events.is_empty());
3235
3236         // Ensure that we don't generate any further events even after the channel-closing commitment
3237         // transaction is confirmed on-chain.
3238         confirm_transaction(&nodes[0], &as_broadcasted_txn[0]);
3239         for _ in 0..(IDEMPOTENCY_TIMEOUT_TICKS * 2) { nodes[0].node.timer_tick_occurred(); }
3240
3241         let events = nodes[0].node.get_and_clear_pending_events();
3242         assert!(events.is_empty());
3243
3244         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3245         reload_node!(nodes[0], test_default_channel_config(), &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister_c, chain_monitor_c, nodes_0_deserialized_c);
3246         let events = nodes[0].node.get_and_clear_pending_events();
3247         assert!(events.is_empty());
3248         check_added_monitors(&nodes[0], 1);
3249 }
3250
3251 #[test]
3252 fn no_missing_sent_on_midpoint_reload() {
3253         do_no_missing_sent_on_midpoint_reload(false);
3254         do_no_missing_sent_on_midpoint_reload(true);
3255 }
3256
3257 fn do_claim_from_closed_chan(fail_payment: bool) {
3258         // Previously, LDK would refuse to claim a payment if a channel on which the payment was
3259         // received had been closed between when the HTLC was received and when we went to claim it.
3260         // This makes sense in the payment case - why pay an on-chain fee to claim the HTLC when
3261         // presumably the sender may retry later. Long ago it also reduced total code in the claim
3262         // pipeline.
3263         //
3264         // However, this doesn't make sense if you're trying to do an atomic swap or some other
3265         // protocol that requires atomicity with some other action - if your money got claimed
3266         // elsewhere you need to be able to claim the HTLC in lightning no matter what. Further, this
3267         // is an over-optimization - there should be a very, very low likelihood that a channel closes
3268         // between when we receive the last HTLC for a payment and the user goes to claim the payment.
3269         // Since we now have code to handle this anyway we should allow it.
3270
3271         // Build 4 nodes and send an MPP payment across two paths. By building a route manually set the
3272         // CLTVs on the paths to different value resulting in a different claim deadline.
3273         let chanmon_cfgs = create_chanmon_cfgs(4);
3274         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3275         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3276         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3277
3278         create_announced_chan_between_nodes(&nodes, 0, 1);
3279         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
3280         let chan_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
3281         create_announced_chan_between_nodes(&nodes, 2, 3);
3282
3283         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3]);
3284         let mut route_params = RouteParameters {
3285                 payment_params: PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3286                         .with_bolt11_features(nodes[1].node.invoice_features()).unwrap(),
3287                 final_value_msat: 10_000_000,
3288         };
3289         let mut route = nodes[0].router.find_route(&nodes[0].node.get_our_node_id(), &route_params,
3290                 None, nodes[0].node.compute_inflight_htlcs()).unwrap();
3291         // Make sure the route is ordered as the B->D path before C->D
3292         route.paths.sort_by(|a, _| if a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
3293                 std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater });
3294
3295         // Note that we add an extra 1 in the send pipeline to compensate for any blocks found while
3296         // the HTLC is being relayed.
3297         route.paths[0].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 8;
3298         route.paths[1].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 12;
3299         let final_cltv = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 8 + 1;
3300
3301         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3302         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3303                 PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(1)).unwrap();
3304         check_added_monitors(&nodes[0], 2);
3305         let mut send_msgs = nodes[0].node.get_and_clear_pending_msg_events();
3306         send_msgs.sort_by(|a, _| {
3307                 let a_node_id =
3308                         if let MessageSendEvent::UpdateHTLCs { node_id, .. } = a { node_id } else { panic!() };
3309                 let node_b_id = nodes[1].node.get_our_node_id();
3310                 if *a_node_id == node_b_id { std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater }
3311         });
3312
3313         assert_eq!(send_msgs.len(), 2);
3314         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 10_000_000,
3315                 payment_hash, Some(payment_secret), send_msgs.remove(0), false, None);
3316         let receive_event = pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 10_000_000,
3317                 payment_hash, Some(payment_secret), send_msgs.remove(0), true, None);
3318
3319         match receive_event.unwrap() {
3320                 Event::PaymentClaimable { claim_deadline, .. } => {
3321                         assert_eq!(claim_deadline.unwrap(), final_cltv - HTLC_FAIL_BACK_BUFFER);
3322                 },
3323                 _ => panic!(),
3324         }
3325
3326         // Ensure that the claim_deadline is correct, with the payment failing at exactly the given
3327         // height.
3328         connect_blocks(&nodes[3], final_cltv - HTLC_FAIL_BACK_BUFFER - nodes[3].best_block_info().1
3329                 - if fail_payment { 0 } else { 2 });
3330         if fail_payment {
3331                 // We fail the HTLC on the A->B->D path first as it expires 4 blocks earlier. We go ahead
3332                 // and expire both immediately, though, by connecting another 4 blocks.
3333                 let reason = HTLCDestination::FailedPayment { payment_hash };
3334                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason.clone()]);
3335                 connect_blocks(&nodes[3], 4);
3336                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason]);
3337                 pass_failed_payment_back(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
3338         } else {
3339                 nodes[1].node.force_close_broadcasting_latest_txn(&chan_bd, &nodes[3].node.get_our_node_id()).unwrap();
3340                 check_closed_event(&nodes[1], 1, ClosureReason::HolderForceClosed, false);
3341                 check_closed_broadcast(&nodes[1], 1, true);
3342                 let bs_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3343                 assert_eq!(bs_tx.len(), 1);
3344
3345                 mine_transaction(&nodes[3], &bs_tx[0]);
3346                 check_added_monitors(&nodes[3], 1);
3347                 check_closed_broadcast(&nodes[3], 1, true);
3348                 check_closed_event(&nodes[3], 1, ClosureReason::CommitmentTxConfirmed, false);
3349
3350                 nodes[3].node.claim_funds(payment_preimage);
3351                 check_added_monitors(&nodes[3], 2);
3352                 expect_payment_claimed!(nodes[3], payment_hash, 10_000_000);
3353
3354                 let ds_tx = nodes[3].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3355                 assert_eq!(ds_tx.len(), 1);
3356                 check_spends!(&ds_tx[0], &bs_tx[0]);
3357
3358                 mine_transactions(&nodes[1], &[&bs_tx[0], &ds_tx[0]]);
3359                 check_added_monitors(&nodes[1], 1);
3360                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[3], Some(1000), false, true);
3361
3362                 let bs_claims = nodes[1].node.get_and_clear_pending_msg_events();
3363                 check_added_monitors(&nodes[1], 1);
3364                 assert_eq!(bs_claims.len(), 1);
3365                 if let MessageSendEvent::UpdateHTLCs { updates, .. } = &bs_claims[0] {
3366                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3367                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
3368                 } else { panic!(); }
3369
3370                 expect_payment_sent!(nodes[0], payment_preimage);
3371
3372                 let ds_claim_msgs = nodes[3].node.get_and_clear_pending_msg_events();
3373                 assert_eq!(ds_claim_msgs.len(), 1);
3374                 let cs_claim_msgs = if let MessageSendEvent::UpdateHTLCs { updates, .. } = &ds_claim_msgs[0] {
3375                         nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3376                         let cs_claim_msgs = nodes[2].node.get_and_clear_pending_msg_events();
3377                         check_added_monitors(&nodes[2], 1);
3378                         commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
3379                         expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
3380                         cs_claim_msgs
3381                 } else { panic!(); };
3382
3383                 assert_eq!(cs_claim_msgs.len(), 1);
3384                 if let MessageSendEvent::UpdateHTLCs { updates, .. } = &cs_claim_msgs[0] {
3385                         nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3386                         commitment_signed_dance!(nodes[0], nodes[2], updates.commitment_signed, false, true);
3387                 } else { panic!(); }
3388
3389                 expect_payment_path_successful!(nodes[0]);
3390         }
3391 }
3392
3393 #[test]
3394 fn claim_from_closed_chan() {
3395         do_claim_from_closed_chan(true);
3396         do_claim_from_closed_chan(false);
3397 }
3398
3399 fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) {
3400         // Check that a payment metadata received on one HTLC that doesn't match the one received on
3401         // another results in the HTLC being rejected.
3402         //
3403         // We first set up a diamond shaped network, allowing us to split a payment into two HTLCs, the
3404         // first of which we'll deliver and the second of which we'll fail and then re-send with
3405         // modified payment metadata, which will in turn result in it being failed by the recipient.
3406         let chanmon_cfgs = create_chanmon_cfgs(4);
3407         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3408         let mut config = test_default_channel_config();
3409         config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 50;
3410         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, Some(config), Some(config), Some(config)]);
3411
3412         let persister;
3413         let new_chain_monitor;
3414         let nodes_0_deserialized;
3415
3416         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3417
3418         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
3419         let chan_id_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
3420         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
3421         let chan_id_cd = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
3422
3423         // Pay more than half of each channel's max, requiring MPP
3424         let amt_msat = 750_000_000;
3425         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3], Some(amt_msat));
3426         let payment_id = PaymentId(payment_hash.0);
3427         let payment_metadata = vec![44, 49, 52, 142];
3428
3429         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3430                 .with_bolt11_features(nodes[1].node.invoice_features()).unwrap();
3431         let mut route_params = RouteParameters {
3432                 payment_params,
3433                 final_value_msat: amt_msat,
3434         };
3435
3436         // Send the MPP payment, delivering the updated commitment state to nodes[1].
3437         nodes[0].node.send_payment(payment_hash, RecipientOnionFields {
3438                         payment_secret: Some(payment_secret), payment_metadata: Some(payment_metadata),
3439                 }, payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
3440         check_added_monitors!(nodes[0], 2);
3441
3442         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
3443         assert_eq!(send_events.len(), 2);
3444         let first_send = SendEvent::from_event(send_events.pop().unwrap());
3445         let second_send = SendEvent::from_event(send_events.pop().unwrap());
3446
3447         let (b_recv_ev, c_recv_ev) = if first_send.node_id == nodes[1].node.get_our_node_id() {
3448                 (&first_send, &second_send)
3449         } else {
3450                 (&second_send, &first_send)
3451         };
3452         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &b_recv_ev.msgs[0]);
3453         commitment_signed_dance!(nodes[1], nodes[0], b_recv_ev.commitment_msg, false, true);
3454
3455         expect_pending_htlcs_forwardable!(nodes[1]);
3456         check_added_monitors(&nodes[1], 1);
3457         let b_forward_ev = SendEvent::from_node(&nodes[1]);
3458         nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &b_forward_ev.msgs[0]);
3459         commitment_signed_dance!(nodes[3], nodes[1], b_forward_ev.commitment_msg, false, true);
3460
3461         expect_pending_htlcs_forwardable!(nodes[3]);
3462
3463         // Before delivering the second MPP HTLC to nodes[2], disconnect nodes[2] and nodes[3], which
3464         // will result in nodes[2] failing the HTLC back.
3465         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
3466         nodes[3].node.peer_disconnected(&nodes[2].node.get_our_node_id());
3467
3468         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &c_recv_ev.msgs[0]);
3469         commitment_signed_dance!(nodes[2], nodes[0], c_recv_ev.commitment_msg, false, true);
3470
3471         let cs_fail = get_htlc_update_msgs(&nodes[2], &nodes[0].node.get_our_node_id());
3472         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail.update_fail_htlcs[0]);
3473         commitment_signed_dance!(nodes[0], nodes[2], cs_fail.commitment_signed, false, true);
3474
3475         let payment_fail_retryable_evs = nodes[0].node.get_and_clear_pending_events();
3476         assert_eq!(payment_fail_retryable_evs.len(), 2);
3477         if let Event::PaymentPathFailed { .. } = payment_fail_retryable_evs[0] {} else { panic!(); }
3478         if let Event::PendingHTLCsForwardable { .. } = payment_fail_retryable_evs[1] {} else { panic!(); }
3479
3480         // Before we allow the HTLC to be retried, optionally change the payment_metadata we have
3481         // stored for our payment.
3482         if do_modify {
3483                 nodes[0].node.test_set_payment_metadata(payment_id, Some(Vec::new()));
3484         }
3485
3486         // Optionally reload nodes[3] to check that the payment_metadata is properly serialized with
3487         // the payment state.
3488         if do_reload {
3489                 let mon_bd = get_monitor!(nodes[3], chan_id_bd).encode();
3490                 let mon_cd = get_monitor!(nodes[3], chan_id_cd).encode();
3491                 reload_node!(nodes[3], config, &nodes[3].node.encode(), &[&mon_bd, &mon_cd],
3492                         persister, new_chain_monitor, nodes_0_deserialized);
3493                 nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
3494                 reconnect_nodes(ReconnectArgs::new(&nodes[1], &nodes[3]));
3495         }
3496         let mut reconnect_args = ReconnectArgs::new(&nodes[2], &nodes[3]);
3497         reconnect_args.send_channel_ready = (true, true);
3498         reconnect_nodes(reconnect_args);
3499
3500         // Create a new channel between C and D as A will refuse to retry on the existing one because
3501         // it just failed.
3502         let chan_id_cd_2 = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
3503
3504         // Now retry the failed HTLC.
3505         nodes[0].node.process_pending_htlc_forwards();
3506         check_added_monitors(&nodes[0], 1);
3507         let as_resend = SendEvent::from_node(&nodes[0]);
3508         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_resend.msgs[0]);
3509         commitment_signed_dance!(nodes[2], nodes[0], as_resend.commitment_msg, false, true);
3510
3511         expect_pending_htlcs_forwardable!(nodes[2]);
3512         check_added_monitors(&nodes[2], 1);
3513         let cs_forward = SendEvent::from_node(&nodes[2]);
3514         nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &cs_forward.msgs[0]);
3515         commitment_signed_dance!(nodes[3], nodes[2], cs_forward.commitment_msg, false, true);
3516
3517         // Finally, check that nodes[3] does the correct thing - either accepting the payment or, if
3518         // the payment metadata was modified, failing only the one modified HTLC and retaining the
3519         // other.
3520         if do_modify {
3521                 expect_pending_htlcs_forwardable_ignore!(nodes[3]);
3522                 nodes[3].node.process_pending_htlc_forwards();
3523                 expect_pending_htlcs_forwardable_conditions(nodes[3].node.get_and_clear_pending_events(),
3524                         &[HTLCDestination::FailedPayment {payment_hash}]);
3525                 nodes[3].node.process_pending_htlc_forwards();
3526
3527                 check_added_monitors(&nodes[3], 1);
3528                 let ds_fail = get_htlc_update_msgs(&nodes[3], &nodes[2].node.get_our_node_id());
3529
3530                 nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &ds_fail.update_fail_htlcs[0]);
3531                 commitment_signed_dance!(nodes[2], nodes[3], ds_fail.commitment_signed, false, true);
3532                 expect_pending_htlcs_forwardable_conditions(nodes[2].node.get_and_clear_pending_events(),
3533                         &[HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_id_cd_2 }]);
3534         } else {
3535                 expect_pending_htlcs_forwardable!(nodes[3]);
3536                 expect_payment_claimable!(nodes[3], payment_hash, payment_secret, amt_msat);
3537                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
3538         }
3539 }
3540
3541 #[test]
3542 fn test_payment_metadata_consistency() {
3543         do_test_payment_metadata_consistency(true, true);
3544         do_test_payment_metadata_consistency(true, false);
3545         do_test_payment_metadata_consistency(false, true);
3546         do_test_payment_metadata_consistency(false, false);
3547 }