_ Git - rust-lightning/blob - lightning/src/ln/payment_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Tests that test the payment retry logic in ChannelManager, including various edge-cases around
  11 //! serialization ordering between ChannelManager/ChannelMonitors and ensuring we can still retry
  12 //! payments thereafter.
  13
  14 use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Listen, Watch};
  15 use crate::chain::channelmonitor::{ANTI_REORG_DELAY, HTLC_FAIL_BACK_BUFFER, LATENCY_GRACE_PERIOD_BLOCKS};
  16 use crate::sign::EntropySource;
  17 use crate::chain::transaction::OutPoint;
  18 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentFailureReason};
  19 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
  20 use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, ChannelManager, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS, RecentPaymentDetails, RecipientOnionFields, HTLCForwardInfo, PendingHTLCRouting, PendingAddHTLCInfo};
  21 use crate::ln::features::InvoiceFeatures;
  22 use crate::ln::{msgs, PaymentSecret, PaymentPreimage};
  23 use crate::ln::msgs::ChannelMessageHandler;
  24 use crate::ln::outbound_payment::Retry;
  25 use crate::routing::gossip::{EffectiveCapacity, RoutingFees};
  26 use crate::routing::router::{get_route, Path, PaymentParameters, Route, Router, RouteHint, RouteHintHop, RouteHop, RouteParameters, find_route};
  27 use crate::routing::scoring::ChannelUsage;
  28 use crate::util::test_utils;
  29 use crate::util::errors::APIError;
  30 use crate::util::ser::Writeable;
  31 use crate::util::string::UntrustedString;
  32
  33 use bitcoin::network::constants::Network;
  34
  35 use crate::prelude::*;
  36
  37 use crate::ln::functional_test_utils::*;
  38 use crate::routing::gossip::NodeId;
  39 #[cfg(feature = "std")]
  40 use {
  41         crate::util::time::tests::SinceEpoch,
  42         std::time::{SystemTime, Instant, Duration}
  43 };
  44
  45 #[test]
  46 fn mpp_failure() {
  47         let chanmon_cfgs = create_chanmon_cfgs(4);
  48         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
  49         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
  50         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
  51
  52         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
  53         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
  54         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
  55         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3).0.contents.short_channel_id;
  56
  57         let (mut route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
  58         let path = route.paths[0].clone();
  59         route.paths.push(path);
  60         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
  61         route.paths[0].hops[0].short_channel_id = chan_1_id;
  62         route.paths[0].hops[1].short_channel_id = chan_3_id;
  63         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
  64         route.paths[1].hops[0].short_channel_id = chan_2_id;
  65         route.paths[1].hops[1].short_channel_id = chan_4_id;
  66         send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
  67         fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
  68 }
  69
  70 #[test]
  71 fn mpp_retry() {
  72         let chanmon_cfgs = create_chanmon_cfgs(4);
  73         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
  74         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
  75         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
  76
  77         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
  78         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
  79         let (chan_3_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
  80         let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes(&nodes, 3, 2);
  81         // Rebalance
  82         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
  83
  84         let amt_msat = 1_000_000;
  85         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], amt_msat);
  86         let path = route.paths[0].clone();
  87         route.paths.push(path);
  88         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
  89         route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
  90         route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
  91         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
  92         route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
  93         route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
  94
  95         // Initiate the MPP payment.
  96         let payment_id = PaymentId(payment_hash.0);
  97         let mut route_params = RouteParameters {
  98                 payment_params: route.payment_params.clone().unwrap(),
  99                 final_value_msat: amt_msat,
 100         };
 101
 102         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
 103         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 104                 payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
 105         check_added_monitors!(nodes[0], 2); // one monitor per path
 106         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 107         assert_eq!(events.len(), 2);
 108
 109         // Pass half of the payment along the success path.
 110         let success_path_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 111         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), success_path_msgs, false, None);
 112
 113         // Add the HTLC along the first hop.
 114         let fail_path_msgs_1 = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 115         let (update_add, commitment_signed) = match fail_path_msgs_1 {
 116                 MessageSendEvent::UpdateHTLCs { node_id: _, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 117                         assert_eq!(update_add_htlcs.len(), 1);
 118                         assert!(update_fail_htlcs.is_empty());
 119                         assert!(update_fulfill_htlcs.is_empty());
 120                         assert!(update_fail_malformed_htlcs.is_empty());
 121                         assert!(update_fee.is_none());
 122                         (update_add_htlcs[0].clone(), commitment_signed.clone())
 123                 },
 124                 _ => panic!("Unexpected event"),
 125         };
 126         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
 127         commitment_signed_dance!(nodes[2], nodes[0], commitment_signed, false);
 128
 129         // Attempt to forward the payment and complete the 2nd path's failure.
 130         expect_pending_htlcs_forwardable!(&nodes[2]);
 131         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id }]);
 132         let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 133         assert!(htlc_updates.update_add_htlcs.is_empty());
 134         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 135         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 136         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 137         check_added_monitors!(nodes[2], 1);
 138         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 139         commitment_signed_dance!(nodes[0], nodes[2], htlc_updates.commitment_signed, false);
 140         let mut events = nodes[0].node.get_and_clear_pending_events();
 141         match events[1] {
 142                 Event::PendingHTLCsForwardable { .. } => {},
 143                 _ => panic!("Unexpected event")
 144         }
 145         events.remove(1);
 146         expect_payment_failed_conditions_event(events, payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 147
 148         // Rebalance the channel so the second half of the payment can succeed.
 149         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 150
 151         // Retry the second half of the payment and make sure it succeeds.
 152         route.paths.remove(0);
 153         route_params.final_value_msat = 1_000_000;
 154         route_params.payment_params.previously_failed_channels.push(chan_4_update.contents.short_channel_id);
 155         nodes[0].router.expect_find_route(route_params, Ok(route));
 156         nodes[0].node.process_pending_htlc_forwards();
 157         check_added_monitors!(nodes[0], 1);
 158         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 159         assert_eq!(events.len(), 1);
 160         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 161         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 162 }
 163
 164 fn do_mpp_receive_timeout(send_partial_mpp: bool) {
 165         let chanmon_cfgs = create_chanmon_cfgs(4);
 166         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 167         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 168         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 169
 170         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 171         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 172         let (chan_3_update, _, chan_3_id, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 173         let (chan_4_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 174
 175         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 100_000);
 176         let path = route.paths[0].clone();
 177         route.paths.push(path);
 178         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
 179         route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
 180         route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
 181         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
 182         route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
 183         route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
 184
 185         // Initiate the MPP payment.
 186         nodes[0].node.send_payment_with_route(&route, payment_hash,
 187                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 188         check_added_monitors!(nodes[0], 2); // one monitor per path
 189         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 190         assert_eq!(events.len(), 2);
 191
 192         // Pass half of the payment along the first path.
 193         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 194         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_1_msgs, false, None);
 195
 196         if send_partial_mpp {
 197                 // Time out the partial MPP
 198                 for _ in 0..MPP_TIMEOUT_TICKS {
 199                         nodes[3].node.timer_tick_occurred();
 200                 }
 201
 202                 // Failed HTLC from node 3 -> 1
 203                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 204                 let htlc_fail_updates_3_1 = get_htlc_update_msgs!(nodes[3], nodes[1].node.get_our_node_id());
 205                 assert_eq!(htlc_fail_updates_3_1.update_fail_htlcs.len(), 1);
 206                 nodes[1].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &htlc_fail_updates_3_1.update_fail_htlcs[0]);
 207                 check_added_monitors!(nodes[3], 1);
 208                 commitment_signed_dance!(nodes[1], nodes[3], htlc_fail_updates_3_1.commitment_signed, false);
 209
 210                 // Failed HTLC from node 1 -> 0
 211                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_3_id }]);
 212                 let htlc_fail_updates_1_0 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 213                 assert_eq!(htlc_fail_updates_1_0.update_fail_htlcs.len(), 1);
 214                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates_1_0.update_fail_htlcs[0]);
 215                 check_added_monitors!(nodes[1], 1);
 216                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates_1_0.commitment_signed, false);
 217
 218                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain().expected_htlc_error_data(23, &[][..]));
 219         } else {
 220                 // Pass half of the payment along the second path.
 221                 let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 222                 pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_2_msgs, true, None);
 223
 224                 // Even after MPP_TIMEOUT_TICKS we should not timeout the MPP if we have all the parts
 225                 for _ in 0..MPP_TIMEOUT_TICKS {
 226                         nodes[3].node.timer_tick_occurred();
 227                 }
 228
 229                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 230         }
 231 }
 232
 233 #[test]
 234 fn mpp_receive_timeout() {
 235         do_mpp_receive_timeout(true);
 236         do_mpp_receive_timeout(false);
 237 }
 238
 239 #[test]
 240 fn test_mpp_keysend() {
 241         let mut mpp_keysend_config = test_default_channel_config();
 242         mpp_keysend_config.accept_mpp_keysend = true;
 243         let chanmon_cfgs = create_chanmon_cfgs(4);
 244         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 245         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, Some(mpp_keysend_config)]);
 246         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 247
 248         create_announced_chan_between_nodes(&nodes, 0, 1);
 249         create_announced_chan_between_nodes(&nodes, 0, 2);
 250         create_announced_chan_between_nodes(&nodes, 1, 3);
 251         create_announced_chan_between_nodes(&nodes, 2, 3);
 252         let network_graph = nodes[0].network_graph.clone();
 253
 254         let payer_pubkey = nodes[0].node.get_our_node_id();
 255         let payee_pubkey = nodes[3].node.get_our_node_id();
 256         let recv_value = 15_000_000;
 257         let route_params = RouteParameters {
 258                 payment_params: PaymentParameters::for_keysend(payee_pubkey, 40, true),
 259                 final_value_msat: recv_value,
 260         };
 261         let scorer = test_utils::TestScorer::new();
 262         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
 263         let route = find_route(&payer_pubkey, &route_params, &network_graph, None, nodes[0].logger,
 264                 &scorer, &(), &random_seed_bytes).unwrap();
 265
 266         let payment_preimage = PaymentPreimage([42; 32]);
 267         let payment_secret = PaymentSecret(payment_preimage.0);
 268         let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage),
 269                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_preimage.0)).unwrap();
 270         check_added_monitors!(nodes[0], 2);
 271
 272         let expected_route: &[&[&Node]] = &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]];
 273         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 274         assert_eq!(events.len(), 2);
 275
 276         let ev = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 277         pass_along_path(&nodes[0], expected_route[0], recv_value, payment_hash.clone(),
 278                 Some(payment_secret), ev.clone(), false, Some(payment_preimage));
 279
 280         let ev = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 281         pass_along_path(&nodes[0], expected_route[1], recv_value, payment_hash.clone(),
 282                 Some(payment_secret), ev.clone(), true, Some(payment_preimage));
 283         claim_payment_along_route(&nodes[0], expected_route, false, payment_preimage);
 284 }
 285
 286 #[test]
 287 fn test_reject_mpp_keysend_htlc() {
 288         // This test enforces that we reject MPP keysend HTLCs if our config states we don't support
 289         // MPP keysend. When receiving a payment, if we don't support MPP keysend we'll reject the
 290         // payment if it's keysend and has a payment secret, never reaching our payment validation
 291         // logic. To check that we enforce rejecting MPP keysends in our payment logic, here we send
 292         // keysend payments without payment secrets, then modify them by adding payment secrets in the
 293         // final node in between receiving the HTLCs and actually processing them.
 294         let mut reject_mpp_keysend_cfg = test_default_channel_config();
 295         reject_mpp_keysend_cfg.accept_mpp_keysend = false;
 296
 297         let chanmon_cfgs = create_chanmon_cfgs(4);
 298         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 299         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, Some(reject_mpp_keysend_cfg)]);
 300         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 301         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
 302         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
 303         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
 304         let (update_a, _, chan_4_channel_id, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 305         let chan_4_id = update_a.contents.short_channel_id;
 306         let amount = 40_000;
 307         let (mut route, payment_hash, payment_preimage, _) = get_route_and_payment_hash!(nodes[0], nodes[3], amount);
 308
 309         // Pay along nodes[1]
 310         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
 311         route.paths[0].hops[0].short_channel_id = chan_1_id;
 312         route.paths[0].hops[1].short_channel_id = chan_3_id;
 313
 314         let payment_id_0 = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 315         nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), RecipientOnionFields::spontaneous_empty(), payment_id_0).unwrap();
 316         check_added_monitors!(nodes[0], 1);
 317
 318         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 319         let update_add_0 = update_0.update_add_htlcs[0].clone();
 320         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_0);
 321         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
 322         expect_pending_htlcs_forwardable!(nodes[1]);
 323
 324         check_added_monitors!(&nodes[1], 1);
 325         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[3].node.get_our_node_id());
 326         let update_add_1 = update_1.update_add_htlcs[0].clone();
 327         nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &update_add_1);
 328         commitment_signed_dance!(nodes[3], nodes[1], update_1.commitment_signed, false, true);
 329
 330         assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
 331         for (_, pending_forwards) in nodes[3].node.forward_htlcs.lock().unwrap().iter_mut() {
 332                 for f in pending_forwards.iter_mut() {
 333                         match f {
 334                                 &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) => {
 335                                         match forward_info.routing {
 336                                                 PendingHTLCRouting::ReceiveKeysend { ref mut payment_data, .. } => {
 337                                                         *payment_data = Some(msgs::FinalOnionHopData {
 338                                                                 payment_secret: PaymentSecret([42; 32]),
 339                                                                 total_msat: amount * 2,
 340                                                         });
 341                                                 },
 342                                                 _ => panic!("Expected PendingHTLCRouting::ReceiveKeysend"),
 343                                         }
 344                                 },
 345                                 _ => {},
 346                         }
 347                 }
 348         }
 349         expect_pending_htlcs_forwardable!(nodes[3]);
 350
 351         // Pay along nodes[2]
 352         route.paths[0].hops[0].pubkey = nodes[2].node.get_our_node_id();
 353         route.paths[0].hops[0].short_channel_id = chan_2_id;
 354         route.paths[0].hops[1].short_channel_id = chan_4_id;
 355
 356         let payment_id_1 = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 357         nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), RecipientOnionFields::spontaneous_empty(), payment_id_1).unwrap();
 358         check_added_monitors!(nodes[0], 1);
 359
 360         let update_2 = get_htlc_update_msgs!(nodes[0], nodes[2].node.get_our_node_id());
 361         let update_add_2 = update_2.update_add_htlcs[0].clone();
 362         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_2);
 363         commitment_signed_dance!(nodes[2], nodes[0], &update_2.commitment_signed, false, true);
 364         expect_pending_htlcs_forwardable!(nodes[2]);
 365
 366         check_added_monitors!(&nodes[2], 1);
 367         let update_3 = get_htlc_update_msgs!(nodes[2], nodes[3].node.get_our_node_id());
 368         let update_add_3 = update_3.update_add_htlcs[0].clone();
 369         nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &update_add_3);
 370         commitment_signed_dance!(nodes[3], nodes[2], update_3.commitment_signed, false, true);
 371
 372         assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
 373         for (_, pending_forwards) in nodes[3].node.forward_htlcs.lock().unwrap().iter_mut() {
 374                 for f in pending_forwards.iter_mut() {
 375                         match f {
 376                                 &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) => {
 377                                         match forward_info.routing {
 378                                                 PendingHTLCRouting::ReceiveKeysend { ref mut payment_data, .. } => {
 379                                                         *payment_data = Some(msgs::FinalOnionHopData {
 380                                                                 payment_secret: PaymentSecret([42; 32]),
 381                                                                 total_msat: amount * 2,
 382                                                         });
 383                                                 },
 384                                                 _ => panic!("Expected PendingHTLCRouting::ReceiveKeysend"),
 385                                         }
 386                                 },
 387                                 _ => {},
 388                         }
 389                 }
 390         }
 391         expect_pending_htlcs_forwardable!(nodes[3]);
 392         check_added_monitors!(nodes[3], 1);
 393
 394         // Fail back along nodes[2]
 395         let update_fail_0 = get_htlc_update_msgs!(&nodes[3], &nodes[2].node.get_our_node_id());
 396         nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &update_fail_0.update_fail_htlcs[0]);
 397         commitment_signed_dance!(nodes[2], nodes[3], update_fail_0.commitment_signed, false);
 398         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_channel_id }]);
 399         check_added_monitors!(nodes[2], 1);
 400
 401         let update_fail_1 = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 402         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_1.update_fail_htlcs[0]);
 403         commitment_signed_dance!(nodes[0], nodes[2], update_fail_1.commitment_signed, false);
 404
 405         expect_payment_failed_conditions(&nodes[0], payment_hash, true, PaymentFailedConditions::new());
 406         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 407 }
 408
 409
 410 #[test]
 411 fn no_pending_leak_on_initial_send_failure() {
 412         // In an earlier version of our payment tracking, we'd have a retry entry even when the initial
 413         // HTLC for payment failed to send due to local channel errors (e.g. peer disconnected). In this
 414         // case, the user wouldn't have a PaymentId to retry the payment with, but we'd think we have a
 415         // pending payment forever and never time it out.
 416         // Here we test exactly that - retrying a payment when a peer was disconnected on the first
 417         // try, and then check that no pending payment is being tracked.
 418         let chanmon_cfgs = create_chanmon_cfgs(2);
 419         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 420         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 421         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 422
 423         create_announced_chan_between_nodes(&nodes, 0, 1);
 424
 425         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 426
 427         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 428         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 429
 430         unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash,
 431                         RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
 432                 ), true, APIError::ChannelUnavailable { ref err },
 433                 assert_eq!(err, "Peer for first hop currently disconnected"));
 434
 435         assert!(!nodes[0].node.has_pending_payments());
 436 }
 437
 438 fn do_retry_with_no_persist(confirm_before_reload: bool) {
 439         // If we send a pending payment and `send_payment` returns success, we should always either
 440         // return a payment failure event or a payment success event, and on failure the payment should
 441         // be retryable.
 442         //
 443         // In order to do so when the ChannelManager isn't immediately persisted (which is normal - its
 444         // always persisted asynchronously), the ChannelManager has to reload some payment data from
 445         // ChannelMonitor(s) in some cases. This tests that reloading.
 446         //
 447         // `confirm_before_reload` confirms the channel-closing commitment transaction on-chain prior
 448         // to reloading the ChannelManager, increasing test coverage in ChannelMonitor HTLC tracking
 449         // which has separate codepaths for "commitment transaction already confirmed" and not.
 450         let chanmon_cfgs = create_chanmon_cfgs(3);
 451         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 452         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 453         let persister: test_utils::TestPersister;
 454         let new_chain_monitor: test_utils::TestChainMonitor;
 455         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 456         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 457
 458         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 459         let (_, _, chan_id_2, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
 460
 461         // Serialize the ChannelManager prior to sending payments
 462         let nodes_0_serialized = nodes[0].node.encode();
 463
 464         // Send two payments - one which will get to nodes[2] and will be claimed, one which we'll time
 465         // out and retry.
 466         let amt_msat = 1_000_000;
 467         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
 468         let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000);
 469         let route_params = RouteParameters {
 470                 payment_params: route.payment_params.clone().unwrap(),
 471                 final_value_msat: amt_msat,
 472         };
 473         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 474                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
 475         check_added_monitors!(nodes[0], 1);
 476
 477         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 478         assert_eq!(events.len(), 1);
 479         let payment_event = SendEvent::from_event(events.pop().unwrap());
 480         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 481
 482         // We relay the payment to nodes[1] while its disconnected from nodes[2], causing the payment
 483         // to be returned immediately to nodes[0], without having nodes[2] fail the inbound payment
 484         // which would prevent retry.
 485         nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id());
 486         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 487
 488         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 489         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true);
 490         // nodes[1] now immediately fails the HTLC as the next-hop channel is disconnected
 491         let _ = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 492
 493         reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 494
 495         let as_commitment_tx = get_local_commitment_txn!(nodes[0], chan_id)[0].clone();
 496         if confirm_before_reload {
 497                 mine_transaction(&nodes[0], &as_commitment_tx);
 498                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 499         }
 500
 501         // The ChannelMonitor should always be the latest version, as we're required to persist it
 502         // during the `commitment_signed_dance!()`.
 503         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 504         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 505
 506         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 507         // force-close the channel.
 508         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 509         assert!(nodes[0].node.list_channels().is_empty());
 510         assert!(nodes[0].node.has_pending_payments());
 511         nodes[0].node.timer_tick_occurred();
 512         if !confirm_before_reload {
 513                 let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 514                 assert_eq!(as_broadcasted_txn.len(), 1);
 515                 assert_eq!(as_broadcasted_txn[0].txid(), as_commitment_tx.txid());
 516         } else {
 517                 assert!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
 518         }
 519         check_added_monitors!(nodes[0], 1);
 520
 521         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 522         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 523                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 524         }, true).unwrap();
 525         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 526
 527         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 528         // error, as the channel has hit the chain.
 529         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 530                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 531         }, false).unwrap();
 532         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 533         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 534         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 535         assert_eq!(as_err.len(), 1);
 536         match as_err[0] {
 537                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 538                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 539                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 540                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) });
 541                         check_added_monitors!(nodes[1], 1);
 542                         assert_eq!(nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 543                 },
 544                 _ => panic!("Unexpected event"),
 545         }
 546         check_closed_broadcast!(nodes[1], false);
 547
 548         // Now claim the first payment, which should allow nodes[1] to claim the payment on-chain when
 549         // we close in a moment.
 550         nodes[2].node.claim_funds(payment_preimage_1);
 551         check_added_monitors!(nodes[2], 1);
 552         expect_payment_claimed!(nodes[2], payment_hash_1, 1_000_000);
 553
 554         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 555         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 556         check_added_monitors!(nodes[1], 1);
 557         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 558         expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], None, false, false);
 559
 560         if confirm_before_reload {
 561                 let best_block = nodes[0].blocks.lock().unwrap().last().unwrap().clone();
 562                 nodes[0].node.best_block_updated(&best_block.0.header, best_block.1);
 563         }
 564
 565         // Create a new channel on which to retry the payment before we fail the payment via the
 566         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 567         // connecting several blocks while creating the channel (implying time has passed).
 568         create_announced_chan_between_nodes(&nodes, 0, 1);
 569         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 570
 571         mine_transaction(&nodes[1], &as_commitment_tx);
 572         let bs_htlc_claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 573         assert_eq!(bs_htlc_claim_txn.len(), 1);
 574         check_spends!(bs_htlc_claim_txn[0], as_commitment_tx);
 575
 576         if !confirm_before_reload {
 577                 mine_transaction(&nodes[0], &as_commitment_tx);
 578         }
 579         mine_transaction(&nodes[0], &bs_htlc_claim_txn[0]);
 580         expect_payment_sent!(nodes[0], payment_preimage_1);
 581         connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
 582         let (first_htlc_timeout_tx, second_htlc_timeout_tx) = {
 583                 let mut txn = nodes[0].tx_broadcaster.unique_txn_broadcast();
 584                 assert_eq!(txn.len(), 2);
 585                 (txn.remove(0), txn.remove(0))
 586         };
 587         check_spends!(first_htlc_timeout_tx, as_commitment_tx);
 588         check_spends!(second_htlc_timeout_tx, as_commitment_tx);
 589         if first_htlc_timeout_tx.input[0].previous_output == bs_htlc_claim_txn[0].input[0].previous_output {
 590                 confirm_transaction(&nodes[0], &second_htlc_timeout_tx);
 591         } else {
 592                 confirm_transaction(&nodes[0], &first_htlc_timeout_tx);
 593         }
 594         nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 595         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new());
 596
 597         // Finally, retry the payment (which was reloaded from the ChannelMonitor when nodes[0] was
 598         // reloaded) via a route over the new channel, which work without issue and eventually be
 599         // received and claimed at the recipient just like any other payment.
 600         let (mut new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 601
 602         // Update the fee on the middle hop to ensure PaymentSent events have the correct (retried) fee
 603         // and not the original fee. We also update node[1]'s relevant config as
 604         // do_claim_payment_along_route expects us to never overpay.
 605         {
 606                 let per_peer_state = nodes[1].node.per_peer_state.read().unwrap();
 607                 let mut peer_state = per_peer_state.get(&nodes[2].node.get_our_node_id())
 608                         .unwrap().lock().unwrap();
 609                 let mut channel = peer_state.channel_by_id.get_mut(&chan_id_2).unwrap();
 610                 let mut new_config = channel.context.config();
 611                 new_config.forwarding_fee_base_msat += 100_000;
 612                 channel.context.update_config(&new_config);
 613                 new_route.paths[0].hops[0].fee_msat += 100_000;
 614         }
 615
 616         // Force expiration of the channel's previous config.
 617         for _ in 0..EXPIRE_PREV_CONFIG_TICKS {
 618                 nodes[1].node.timer_tick_occurred();
 619         }
 620
 621         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash, // Shouldn't be allowed to retry a fulfilled payment
 622                 RecipientOnionFields::secret_only(payment_secret), payment_id_1).is_err());
 623         nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 624                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 625         check_added_monitors!(nodes[0], 1);
 626         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 627         assert_eq!(events.len(), 1);
 628         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 629         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
 630         expect_payment_sent!(nodes[0], payment_preimage, Some(new_route.paths[0].hops[0].fee_msat));
 631 }
 632
 633 #[test]
 634 fn retry_with_no_persist() {
 635         do_retry_with_no_persist(true);
 636         do_retry_with_no_persist(false);
 637 }
 638
 639 fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 640         // Test that an off-chain completed payment is not retryable on restart. This was previously
 641         // broken for dust payments, but we test for both dust and non-dust payments.
 642         //
 643         // `use_dust` switches to using a dust HTLC, which results in the HTLC not having an on-chain
 644         // output at all.
 645         let chanmon_cfgs = create_chanmon_cfgs(3);
 646         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 647
 648         let mut manually_accept_config = test_default_channel_config();
 649         manually_accept_config.manually_accept_inbound_channels = true;
 650
 651         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(manually_accept_config), None]);
 652
 653         let first_persister: test_utils::TestPersister;
 654         let first_new_chain_monitor: test_utils::TestChainMonitor;
 655         let first_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 656         let second_persister: test_utils::TestPersister;
 657         let second_new_chain_monitor: test_utils::TestChainMonitor;
 658         let second_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 659         let third_persister: test_utils::TestPersister;
 660         let third_new_chain_monitor: test_utils::TestChainMonitor;
 661         let third_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 662
 663         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 664
 665         // Because we set nodes[1] to manually accept channels, just open a 0-conf channel.
 666         let (funding_tx, chan_id) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 667         confirm_transaction(&nodes[0], &funding_tx);
 668         confirm_transaction(&nodes[1], &funding_tx);
 669         // Ignore the announcement_signatures messages
 670         nodes[0].node.get_and_clear_pending_msg_events();
 671         nodes[1].node.get_and_clear_pending_msg_events();
 672         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 673
 674         // Serialize the ChannelManager prior to sending payments
 675         let mut nodes_0_serialized = nodes[0].node.encode();
 676
 677         let route = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 }).0;
 678         let (payment_preimage, payment_hash, payment_secret, payment_id) = send_along_route(&nodes[0], route, &[&nodes[1], &nodes[2]], if use_dust { 1_000 } else { 1_000_000 });
 679
 680         // The ChannelMonitor should always be the latest version, as we're required to persist it
 681         // during the `commitment_signed_dance!()`.
 682         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 683
 684         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized], first_persister, first_new_chain_monitor, first_nodes_0_deserialized);
 685         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 686
 687         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 688         // force-close the channel.
 689         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 690         nodes[0].node.timer_tick_occurred();
 691         assert!(nodes[0].node.list_channels().is_empty());
 692         assert!(nodes[0].node.has_pending_payments());
 693         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 694         check_added_monitors!(nodes[0], 1);
 695
 696         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 697                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 698         }, true).unwrap();
 699         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 700
 701         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 702         // error, as the channel has hit the chain.
 703         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 704                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 705         }, false).unwrap();
 706         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 707         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 708         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 709         assert_eq!(as_err.len(), 1);
 710         let bs_commitment_tx;
 711         match as_err[0] {
 712                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 713                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 714                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 715                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) });
 716                         check_added_monitors!(nodes[1], 1);
 717                         bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 718                 },
 719                 _ => panic!("Unexpected event"),
 720         }
 721         check_closed_broadcast!(nodes[1], false);
 722
 723         // Now fail back the payment from nodes[2] to nodes[1]. This doesn't really matter as the
 724         // previous hop channel is already on-chain, but it makes nodes[2] willing to see additional
 725         // incoming HTLCs with the same payment hash later.
 726         nodes[2].node.fail_htlc_backwards(&payment_hash);
 727         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
 728         check_added_monitors!(nodes[2], 1);
 729
 730         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 731         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fail_htlcs[0]);
 732         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 733         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1],
 734                 [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 735
 736         // Connect the HTLC-Timeout transaction, timing out the HTLC on both nodes (but not confirming
 737         // the HTLC-Timeout transaction beyond 1 conf). For dust HTLCs, the HTLC is considered resolved
 738         // after the commitment transaction, so always connect the commitment transaction.
 739         mine_transaction(&nodes[0], &bs_commitment_tx[0]);
 740         mine_transaction(&nodes[1], &bs_commitment_tx[0]);
 741         if !use_dust {
 742                 connect_blocks(&nodes[0], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
 743                 connect_blocks(&nodes[1], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
 744                 let as_htlc_timeout = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 745                 check_spends!(as_htlc_timeout[0], bs_commitment_tx[0]);
 746                 assert_eq!(as_htlc_timeout.len(), 1);
 747
 748                 mine_transaction(&nodes[0], &as_htlc_timeout[0]);
 749                 // nodes[0] may rebroadcast (or RBF-bump) its HTLC-Timeout, so wipe the announced set.
 750                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 751                 mine_transaction(&nodes[1], &as_htlc_timeout[0]);
 752         }
 753
 754         // Create a new channel on which to retry the payment before we fail the payment via the
 755         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 756         // connecting several blocks while creating the channel (implying time has passed).
 757         // We do this with a zero-conf channel to avoid connecting blocks as a side-effect.
 758         let (_, chan_id_3) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 759         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 760
 761         // If we attempt to retry prior to the HTLC-Timeout (or commitment transaction, for dust HTLCs)
 762         // confirming, we will fail as it's considered still-pending...
 763         let (new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 });
 764         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 765                 Err(PaymentSendFailure::DuplicatePayment) => {},
 766                 _ => panic!("Unexpected error")
 767         }
 768         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 769
 770         // After ANTI_REORG_DELAY confirmations, the HTLC should be failed and we can try the payment
 771         // again. We serialize the node first as we'll then test retrying the HTLC after a restart
 772         // (which should also still work).
 773         connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
 774         connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 775         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new());
 776
 777         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 778         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 779         nodes_0_serialized = nodes[0].node.encode();
 780
 781         // After the payment failed, we're free to send it again.
 782         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 783                 RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
 784         assert!(!nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 785
 786         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], second_persister, second_new_chain_monitor, second_nodes_0_deserialized);
 787         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 788
 789         nodes[0].node.test_process_background_events();
 790         check_added_monitors(&nodes[0], 1);
 791
 792         reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 793
 794         // Now resend the payment, delivering the HTLC and actually claiming it this time. This ensures
 795         // the payment is not (spuriously) listed as still pending.
 796         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 797                 RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
 798         check_added_monitors!(nodes[0], 1);
 799         pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], if use_dust { 1_000 } else { 1_000_000 }, payment_hash, payment_secret);
 800         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
 801
 802         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 803                 Err(PaymentSendFailure::DuplicatePayment) => {},
 804                 _ => panic!("Unexpected error")
 805         }
 806         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 807
 808         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 809         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 810         nodes_0_serialized = nodes[0].node.encode();
 811
 812         // Check that after reload we can send the payment again (though we shouldn't, since it was
 813         // claimed previously).
 814         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], third_persister, third_new_chain_monitor, third_nodes_0_deserialized);
 815         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 816
 817         nodes[0].node.test_process_background_events();
 818         check_added_monitors(&nodes[0], 1);
 819
 820         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 821
 822         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 823                 Err(PaymentSendFailure::DuplicatePayment) => {},
 824                 _ => panic!("Unexpected error")
 825         }
 826         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 827 }
 828
 829 #[test]
 830 fn test_completed_payment_not_retryable_on_reload() {
 831         do_test_completed_payment_not_retryable_on_reload(true);
 832         do_test_completed_payment_not_retryable_on_reload(false);
 833 }
 834
 835
 836 fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, confirm_commitment_tx: bool, payment_timeout: bool) {
 837         // When a Channel is closed, any outbound HTLCs which were relayed through it are simply
 838         // dropped when the Channel is. From there, the ChannelManager relies on the ChannelMonitor
 839         // having a copy of the relevant fail-/claim-back data and processes the HTLC fail/claim when
 840         // the ChannelMonitor tells it to.
 841         //
 842         // If, due to an on-chain event, an HTLC is failed/claimed, we should avoid providing the
 843         // ChannelManager the HTLC event until after the monitor is re-persisted. This should prevent a
 844         // duplicate HTLC fail/claim (e.g. via a PaymentPathFailed event).
 845         let chanmon_cfgs = create_chanmon_cfgs(2);
 846         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 847         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 848         let persister: test_utils::TestPersister;
 849         let new_chain_monitor: test_utils::TestChainMonitor;
 850         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 851         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 852
 853         let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1);
 854
 855         // Route a payment, but force-close the channel before the HTLC fulfill message arrives at
 856         // nodes[0].
 857         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 10_000_000);
 858         nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap();
 859         check_closed_broadcast!(nodes[0], true);
 860         check_added_monitors!(nodes[0], 1);
 861         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 862
 863         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 864         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 865
 866         // Connect blocks until the CLTV timeout is up so that we get an HTLC-Timeout transaction
 867         connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
 868         let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 869         assert_eq!(node_txn.len(), 3);
 870         assert_eq!(node_txn[0].txid(), node_txn[1].txid());
 871         check_spends!(node_txn[1], funding_tx);
 872         check_spends!(node_txn[2], node_txn[1]);
 873         let timeout_txn = vec![node_txn[2].clone()];
 874
 875         nodes[1].node.claim_funds(payment_preimage);
 876         check_added_monitors!(nodes[1], 1);
 877         expect_payment_claimed!(nodes[1], payment_hash, 10_000_000);
 878
 879         connect_block(&nodes[1], &create_dummy_block(nodes[1].best_block_hash(), 42, vec![node_txn[1].clone()]));
 880         check_closed_broadcast!(nodes[1], true);
 881         check_added_monitors!(nodes[1], 1);
 882         check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 883         let claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 884         assert_eq!(claim_txn.len(), 1);
 885         check_spends!(claim_txn[0], node_txn[1]);
 886
 887         connect_block(&nodes[0], &create_dummy_block(nodes[0].best_block_hash(), 42, vec![node_txn[1].clone()]));
 888
 889         if confirm_commitment_tx {
 890                 connect_blocks(&nodes[0], BREAKDOWN_TIMEOUT as u32 - 1);
 891         }
 892
 893         let claim_block = create_dummy_block(nodes[0].best_block_hash(), 42, if payment_timeout { timeout_txn } else { vec![claim_txn[0].clone()] });
 894
 895         if payment_timeout {
 896                 assert!(confirm_commitment_tx); // Otherwise we're spending below our CSV!
 897                 connect_block(&nodes[0], &claim_block);
 898                 connect_blocks(&nodes[0], ANTI_REORG_DELAY - 2);
 899         }
 900
 901         // Now connect the HTLC claim transaction with the ChainMonitor-generated ChannelMonitor update
 902         // returning InProgress. This should cause the claim event to never make its way to the
 903         // ChannelManager.
 904         chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap().clear();
 905         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
 906
 907         if payment_timeout {
 908                 connect_blocks(&nodes[0], 1);
 909         } else {
 910                 connect_block(&nodes[0], &claim_block);
 911         }
 912
 913         let funding_txo = OutPoint { txid: funding_tx.txid(), index: 0 };
 914         let mon_updates: Vec<_> = chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap()
 915                 .get_mut(&funding_txo).unwrap().drain().collect();
 916         // If we are using chain::Confirm instead of chain::Listen, we will get the same update twice.
 917         // If we're testing connection idempotency we may get substantially more.
 918         assert!(mon_updates.len() >= 1);
 919         assert!(nodes[0].chain_monitor.release_pending_monitor_events().is_empty());
 920         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 921
 922         // If we persist the ChannelManager here, we should get the PaymentSent event after
 923         // deserialization.
 924         let mut chan_manager_serialized = Vec::new();
 925         if !persist_manager_post_event {
 926                 chan_manager_serialized = nodes[0].node.encode();
 927         }
 928
 929         // Now persist the ChannelMonitor and inform the ChainMonitor that we're done, generating the
 930         // payment sent event.
 931         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
 932         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 933         for update in mon_updates {
 934                 nodes[0].chain_monitor.chain_monitor.channel_monitor_updated(funding_txo, update).unwrap();
 935         }
 936         if payment_timeout {
 937                 expect_payment_failed!(nodes[0], payment_hash, false);
 938         } else {
 939                 expect_payment_sent!(nodes[0], payment_preimage);
 940         }
 941
 942         // If we persist the ChannelManager after we get the PaymentSent event, we shouldn't get it
 943         // twice.
 944         if persist_manager_post_event {
 945                 chan_manager_serialized = nodes[0].node.encode();
 946         }
 947
 948         // Now reload nodes[0]...
 949         reload_node!(nodes[0], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 950
 951         if persist_manager_post_event {
 952                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 953         } else if payment_timeout {
 954                 expect_payment_failed!(nodes[0], payment_hash, false);
 955         } else {
 956                 expect_payment_sent!(nodes[0], payment_preimage);
 957         }
 958
 959         // Note that if we re-connect the block which exposed nodes[0] to the payment preimage (but
 960         // which the current ChannelMonitor has not seen), the ChannelManager's de-duplication of
 961         // payment events should kick in, leaving us with no pending events here.
 962         let height = nodes[0].blocks.lock().unwrap().len() as u32 - 1;
 963         nodes[0].chain_monitor.chain_monitor.block_connected(&claim_block, height);
 964         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 965         check_added_monitors(&nodes[0], 1);
 966 }
 967
 968 #[test]
 969 fn test_dup_htlc_onchain_fails_on_reload() {
 970         do_test_dup_htlc_onchain_fails_on_reload(true, true, true);
 971         do_test_dup_htlc_onchain_fails_on_reload(true, true, false);
 972         do_test_dup_htlc_onchain_fails_on_reload(true, false, false);
 973         do_test_dup_htlc_onchain_fails_on_reload(false, true, true);
 974         do_test_dup_htlc_onchain_fails_on_reload(false, true, false);
 975         do_test_dup_htlc_onchain_fails_on_reload(false, false, false);
 976 }
 977
 978 #[test]
 979 fn test_fulfill_restart_failure() {
 980         // When we receive an update_fulfill_htlc message, we immediately consider the HTLC fully
 981         // fulfilled. At this point, the peer can reconnect and decide to either fulfill the HTLC
 982         // again, or fail it, giving us free money.
 983         //
 984         // Of course probably they won't fail it and give us free money, but because we have code to
 985         // handle it, we should test the logic for it anyway. We do that here.
 986         let chanmon_cfgs = create_chanmon_cfgs(2);
 987         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 988         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 989         let persister: test_utils::TestPersister;
 990         let new_chain_monitor: test_utils::TestChainMonitor;
 991         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 992         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 993
 994         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 995         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 100_000);
 996
 997         // The simplest way to get a failure after a fulfill is to reload nodes[1] from a state
 998         // pre-fulfill, which we do by serializing it here.
 999         let chan_manager_serialized = nodes[1].node.encode();
1000         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id).encode();
1001
1002         nodes[1].node.claim_funds(payment_preimage);
1003         check_added_monitors!(nodes[1], 1);
1004         expect_payment_claimed!(nodes[1], payment_hash, 100_000);
1005
1006         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1007         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
1008         expect_payment_sent_without_paths!(nodes[0], payment_preimage);
1009
1010         // Now reload nodes[1]...
1011         reload_node!(nodes[1], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1012
1013         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1014         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1015
1016         nodes[1].node.fail_htlc_backwards(&payment_hash);
1017         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]);
1018         check_added_monitors!(nodes[1], 1);
1019         let htlc_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1020         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates.update_fail_htlcs[0]);
1021         commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates.commitment_signed, false);
1022         // nodes[0] shouldn't generate any events here, while it just got a payment failure completion
1023         // it had already considered the payment fulfilled, and now they just got free money.
1024         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1025 }
1026
1027 #[test]
1028 fn get_ldk_payment_preimage() {
1029         // Ensure that `ChannelManager::get_payment_preimage` can successfully be used to claim a payment.
1030         let chanmon_cfgs = create_chanmon_cfgs(2);
1031         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1032         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1033         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1034         create_announced_chan_between_nodes(&nodes, 0, 1);
1035
1036         let amt_msat = 60_000;
1037         let expiry_secs = 60 * 60;
1038         let (payment_hash, payment_secret) = nodes[1].node.create_inbound_payment(Some(amt_msat), expiry_secs, None).unwrap();
1039
1040         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
1041                 .with_bolt11_features(nodes[1].node.invoice_features()).unwrap();
1042         let scorer = test_utils::TestScorer::new();
1043         let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet);
1044         let random_seed_bytes = keys_manager.get_secure_random_bytes();
1045         let route = get_route(
1046                 &nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(),
1047                 Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()),
1048                 amt_msat, nodes[0].logger, &scorer, &(), &random_seed_bytes).unwrap();
1049         nodes[0].node.send_payment_with_route(&route, payment_hash,
1050                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
1051         check_added_monitors!(nodes[0], 1);
1052
1053         // Make sure to use `get_payment_preimage`
1054         let payment_preimage = nodes[1].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1055         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1056         assert_eq!(events.len(), 1);
1057         pass_along_path(&nodes[0], &[&nodes[1]], amt_msat, payment_hash, Some(payment_secret), events.pop().unwrap(), true, Some(payment_preimage));
1058         claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, payment_preimage);
1059 }
1060
1061 #[test]
1062 fn sent_probe_is_probe_of_sending_node() {
1063         let chanmon_cfgs = create_chanmon_cfgs(3);
1064         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1065         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1066         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1067
1068         create_announced_chan_between_nodes(&nodes, 0, 1);
1069         create_announced_chan_between_nodes(&nodes, 1, 2);
1070
1071         // First check we refuse to build a single-hop probe
1072         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[1], 100_000);
1073         assert!(nodes[0].node.send_probe(route.paths[0].clone()).is_err());
1074
1075         // Then build an actual two-hop probing path
1076         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
1077
1078         match nodes[0].node.send_probe(route.paths[0].clone()) {
1079                 Ok((payment_hash, payment_id)) => {
1080                         assert!(nodes[0].node.payment_is_probe(&payment_hash, &payment_id));
1081                         assert!(!nodes[1].node.payment_is_probe(&payment_hash, &payment_id));
1082                         assert!(!nodes[2].node.payment_is_probe(&payment_hash, &payment_id));
1083                 },
1084                 _ => panic!(),
1085         }
1086
1087         get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1088         check_added_monitors!(nodes[0], 1);
1089 }
1090
1091 #[test]
1092 fn successful_probe_yields_event() {
1093         let chanmon_cfgs = create_chanmon_cfgs(3);
1094         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1095         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1096         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1097
1098         create_announced_chan_between_nodes(&nodes, 0, 1);
1099         create_announced_chan_between_nodes(&nodes, 1, 2);
1100
1101         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
1102
1103         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1104
1105         // node[0] -- update_add_htlcs -> node[1]
1106         check_added_monitors!(nodes[0], 1);
1107         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1108         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1109         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1110         check_added_monitors!(nodes[1], 0);
1111         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1112         expect_pending_htlcs_forwardable!(nodes[1]);
1113
1114         // node[1] -- update_add_htlcs -> node[2]
1115         check_added_monitors!(nodes[1], 1);
1116         let updates = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1117         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1118         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &probe_event.msgs[0]);
1119         check_added_monitors!(nodes[2], 0);
1120         commitment_signed_dance!(nodes[2], nodes[1], probe_event.commitment_msg, true, true);
1121
1122         // node[1] <- update_fail_htlcs -- node[2]
1123         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1124         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1125         check_added_monitors!(nodes[1], 0);
1126         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, true);
1127
1128         // node[0] <- update_fail_htlcs -- node[1]
1129         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1130         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1131         check_added_monitors!(nodes[0], 0);
1132         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1133
1134         let mut events = nodes[0].node.get_and_clear_pending_events();
1135         assert_eq!(events.len(), 1);
1136         match events.drain(..).next().unwrap() {
1137                 crate::events::Event::ProbeSuccessful { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1138                         assert_eq!(payment_id, ev_pid);
1139                         assert_eq!(payment_hash, ev_ph);
1140                 },
1141                 _ => panic!(),
1142         };
1143         assert!(!nodes[0].node.has_pending_payments());
1144 }
1145
1146 #[test]
1147 fn failed_probe_yields_event() {
1148         let chanmon_cfgs = create_chanmon_cfgs(3);
1149         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1150         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1151         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1152
1153         create_announced_chan_between_nodes(&nodes, 0, 1);
1154         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 100000, 90000000);
1155
1156         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1157
1158         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 9_998_000);
1159
1160         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1161
1162         // node[0] -- update_add_htlcs -> node[1]
1163         check_added_monitors!(nodes[0], 1);
1164         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1165         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1166         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1167         check_added_monitors!(nodes[1], 0);
1168         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1169         expect_pending_htlcs_forwardable!(nodes[1]);
1170
1171         // node[0] <- update_fail_htlcs -- node[1]
1172         check_added_monitors!(nodes[1], 1);
1173         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1174         // Skip the PendingHTLCsForwardable event
1175         let _events = nodes[1].node.get_and_clear_pending_events();
1176         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1177         check_added_monitors!(nodes[0], 0);
1178         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1179
1180         let mut events = nodes[0].node.get_and_clear_pending_events();
1181         assert_eq!(events.len(), 1);
1182         match events.drain(..).next().unwrap() {
1183                 crate::events::Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1184                         assert_eq!(payment_id, ev_pid);
1185                         assert_eq!(payment_hash, ev_ph);
1186                 },
1187                 _ => panic!(),
1188         };
1189         assert!(!nodes[0].node.has_pending_payments());
1190 }
1191
1192 #[test]
1193 fn onchain_failed_probe_yields_event() {
1194         // Tests that an attempt to probe over a channel that is eventaully closed results in a failure
1195         // event.
1196         let chanmon_cfgs = create_chanmon_cfgs(3);
1197         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1198         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1199         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1200
1201         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1202         create_announced_chan_between_nodes(&nodes, 1, 2);
1203
1204         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1205
1206         // Send a dust HTLC, which will be treated as if it timed out once the channel hits the chain.
1207         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 1_000);
1208         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1209
1210         // node[0] -- update_add_htlcs -> node[1]
1211         check_added_monitors!(nodes[0], 1);
1212         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1213         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1214         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1215         check_added_monitors!(nodes[1], 0);
1216         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1217         expect_pending_htlcs_forwardable!(nodes[1]);
1218
1219         check_added_monitors!(nodes[1], 1);
1220         let _ = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1221
1222         // Don't bother forwarding the HTLC onwards and just confirm the force-close transaction on
1223         // Node A, which after 6 confirmations should result in a probe failure event.
1224         let bs_txn = get_local_commitment_txn!(nodes[1], chan_id);
1225         confirm_transaction(&nodes[0], &bs_txn[0]);
1226         check_closed_broadcast!(&nodes[0], true);
1227         check_added_monitors!(nodes[0], 1);
1228
1229         let mut events = nodes[0].node.get_and_clear_pending_events();
1230         assert_eq!(events.len(), 2);
1231         let mut found_probe_failed = false;
1232         for event in events.drain(..) {
1233                 match event {
1234                         Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1235                                 assert_eq!(payment_id, ev_pid);
1236                                 assert_eq!(payment_hash, ev_ph);
1237                                 found_probe_failed = true;
1238                         },
1239                         Event::ChannelClosed { .. } => {},
1240                         _ => panic!(),
1241                 }
1242         }
1243         assert!(found_probe_failed);
1244         assert!(!nodes[0].node.has_pending_payments());
1245 }
1246
1247 #[test]
1248 fn claimed_send_payment_idempotent() {
1249         // Tests that `send_payment` (and friends) are (reasonably) idempotent.
1250         let chanmon_cfgs = create_chanmon_cfgs(2);
1251         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1252         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1253         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1254
1255         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1256
1257         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1258         let (first_payment_preimage, _, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1259
1260         macro_rules! check_send_rejected {
1261                 () => {
1262                         // If we try to resend a new payment with a different payment_hash but with the same
1263                         // payment_id, it should be rejected.
1264                         let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1265                                 RecipientOnionFields::secret_only(second_payment_secret), payment_id);
1266                         match send_result {
1267                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1268                                 _ => panic!("Unexpected send result: {:?}", send_result),
1269                         }
1270
1271                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1272                         // also be rejected.
1273                         let send_result = nodes[0].node.send_spontaneous_payment(
1274                                 &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
1275                         match send_result {
1276                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1277                                 _ => panic!("Unexpected send result: {:?}", send_result),
1278                         }
1279                 }
1280         }
1281
1282         check_send_rejected!();
1283
1284         // Claim the payment backwards, but note that the PaymentSent event is still pending and has
1285         // not been seen by the user. At this point, from the user perspective nothing has changed, so
1286         // we must remain just as idempotent as we were before.
1287         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, first_payment_preimage);
1288
1289         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1290                 nodes[0].node.timer_tick_occurred();
1291         }
1292
1293         check_send_rejected!();
1294
1295         // Once the user sees and handles the `PaymentSent` event, we expect them to no longer call
1296         // `send_payment`, and our idempotency guarantees are off - they should have atomically marked
1297         // the payment complete. However, they could have called `send_payment` while the event was
1298         // being processed, leading to a race in our idempotency guarantees. Thus, even immediately
1299         // after the event is handled a duplicate payment should sitll be rejected.
1300         expect_payment_sent!(&nodes[0], first_payment_preimage, Some(0));
1301         check_send_rejected!();
1302
1303         // If relatively little time has passed, a duplicate payment should still fail.
1304         nodes[0].node.timer_tick_occurred();
1305         check_send_rejected!();
1306
1307         // However, after some time has passed (at least more than the one timer tick above), a
1308         // duplicate payment should go through, as ChannelManager should no longer have any remaining
1309         // references to the old payment data.
1310         for _ in 0..IDEMPOTENCY_TIMEOUT_TICKS {
1311                 nodes[0].node.timer_tick_occurred();
1312         }
1313
1314         nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1315                 RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
1316         check_added_monitors!(nodes[0], 1);
1317         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1318         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1319 }
1320
1321 #[test]
1322 fn abandoned_send_payment_idempotent() {
1323         // Tests that `send_payment` (and friends) allow duplicate PaymentIds immediately after
1324         // abandon_payment.
1325         let chanmon_cfgs = create_chanmon_cfgs(2);
1326         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1327         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1328         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1329
1330         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1331
1332         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1333         let (_, first_payment_hash, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1334
1335         macro_rules! check_send_rejected {
1336                 () => {
1337                         // If we try to resend a new payment with a different payment_hash but with the same
1338                         // payment_id, it should be rejected.
1339                         let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1340                                 RecipientOnionFields::secret_only(second_payment_secret), payment_id);
1341                         match send_result {
1342                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1343                                 _ => panic!("Unexpected send result: {:?}", send_result),
1344                         }
1345
1346                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1347                         // also be rejected.
1348                         let send_result = nodes[0].node.send_spontaneous_payment(
1349                                 &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
1350                         match send_result {
1351                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1352                                 _ => panic!("Unexpected send result: {:?}", send_result),
1353                         }
1354                 }
1355         }
1356
1357         check_send_rejected!();
1358
1359         nodes[1].node.fail_htlc_backwards(&first_payment_hash);
1360         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash: first_payment_hash }]);
1361
1362         // Until we abandon the payment upon path failure, no matter how many timer ticks pass, we still cannot reuse the
1363         // PaymentId.
1364         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1365                 nodes[0].node.timer_tick_occurred();
1366         }
1367         check_send_rejected!();
1368
1369         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash, PaymentFailureReason::RecipientRejected);
1370
1371         // However, we can reuse the PaymentId immediately after we `abandon_payment` upon passing the
1372         // failed payment back.
1373         nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1374                 RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
1375         check_added_monitors!(nodes[0], 1);
1376         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1377         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1378 }
1379
1380 #[derive(PartialEq)]
1381 enum InterceptTest {
1382         Forward,
1383         Fail,
1384         Timeout,
1385 }
1386
1387 #[test]
1388 fn test_trivial_inflight_htlc_tracking(){
1389         // In this test, we test three scenarios:
1390         // (1) Sending + claiming a payment successfully should return `None` when querying InFlightHtlcs
1391         // (2) Sending a payment without claiming it should return the payment's value (500000) when querying InFlightHtlcs
1392         // (3) After we claim the payment sent in (2), InFlightHtlcs should return `None` for the query.
1393         let chanmon_cfgs = create_chanmon_cfgs(3);
1394         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1395         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1396         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1397
1398         let (_, _, chan_1_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
1399         let (_, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
1400
1401         // Send and claim the payment. Inflight HTLCs should be empty.
1402         let payment_hash = send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 500000).1;
1403         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1404         {
1405                 let mut node_0_per_peer_lock;
1406                 let mut node_0_peer_state_lock;
1407                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1408
1409                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1410                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1411                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1412                         channel_1.context.get_short_channel_id().unwrap()
1413                 );
1414                 assert_eq!(chan_1_used_liquidity, None);
1415         }
1416         {
1417                 let mut node_1_per_peer_lock;
1418                 let mut node_1_peer_state_lock;
1419                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1420
1421                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1422                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1423                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1424                         channel_2.context.get_short_channel_id().unwrap()
1425                 );
1426
1427                 assert_eq!(chan_2_used_liquidity, None);
1428         }
1429         let pending_payments = nodes[0].node.list_recent_payments();
1430         assert_eq!(pending_payments.len(), 1);
1431         assert_eq!(pending_payments[0], RecentPaymentDetails::Fulfilled { payment_hash: Some(payment_hash) });
1432
1433         // Remove fulfilled payment
1434         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1435                 nodes[0].node.timer_tick_occurred();
1436         }
1437
1438         // Send the payment, but do not claim it. Our inflight HTLCs should contain the pending payment.
1439         let (payment_preimage, payment_hash,  _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 500000);
1440         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1441         {
1442                 let mut node_0_per_peer_lock;
1443                 let mut node_0_peer_state_lock;
1444                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1445
1446                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1447                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1448                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1449                         channel_1.context.get_short_channel_id().unwrap()
1450                 );
1451                 // First hop accounts for expected 1000 msat fee
1452                 assert_eq!(chan_1_used_liquidity, Some(501000));
1453         }
1454         {
1455                 let mut node_1_per_peer_lock;
1456                 let mut node_1_peer_state_lock;
1457                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1458
1459                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1460                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1461                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1462                         channel_2.context.get_short_channel_id().unwrap()
1463                 );
1464
1465                 assert_eq!(chan_2_used_liquidity, Some(500000));
1466         }
1467         let pending_payments = nodes[0].node.list_recent_payments();
1468         assert_eq!(pending_payments.len(), 1);
1469         assert_eq!(pending_payments[0], RecentPaymentDetails::Pending { payment_hash, total_msat: 500000 });
1470
1471         // Now, let's claim the payment. This should result in the used liquidity to return `None`.
1472         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
1473
1474         // Remove fulfilled payment
1475         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1476                 nodes[0].node.timer_tick_occurred();
1477         }
1478
1479         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1480         {
1481                 let mut node_0_per_peer_lock;
1482                 let mut node_0_peer_state_lock;
1483                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1484
1485                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1486                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1487                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1488                         channel_1.context.get_short_channel_id().unwrap()
1489                 );
1490                 assert_eq!(chan_1_used_liquidity, None);
1491         }
1492         {
1493                 let mut node_1_per_peer_lock;
1494                 let mut node_1_peer_state_lock;
1495                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1496
1497                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1498                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1499                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1500                         channel_2.context.get_short_channel_id().unwrap()
1501                 );
1502                 assert_eq!(chan_2_used_liquidity, None);
1503         }
1504
1505         let pending_payments = nodes[0].node.list_recent_payments();
1506         assert_eq!(pending_payments.len(), 0);
1507 }
1508
1509 #[test]
1510 fn test_holding_cell_inflight_htlcs() {
1511         let chanmon_cfgs = create_chanmon_cfgs(2);
1512         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1513         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1514         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1515         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1516
1517         let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
1518         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1519
1520         // Queue up two payments - one will be delivered right away, one immediately goes into the
1521         // holding cell as nodes[0] is AwaitingRAA.
1522         {
1523                 nodes[0].node.send_payment_with_route(&route, payment_hash_1,
1524                         RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
1525                 check_added_monitors!(nodes[0], 1);
1526                 nodes[0].node.send_payment_with_route(&route, payment_hash_2,
1527                         RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
1528                 check_added_monitors!(nodes[0], 0);
1529         }
1530
1531         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1532
1533         {
1534                 let mut node_0_per_peer_lock;
1535                 let mut node_0_peer_state_lock;
1536                 let channel =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, channel_id);
1537
1538                 let used_liquidity = inflight_htlcs.used_liquidity_msat(
1539                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1540                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1541                         channel.context.get_short_channel_id().unwrap()
1542                 );
1543
1544                 assert_eq!(used_liquidity, Some(2000000));
1545         }
1546
1547         // Clear pending events so test doesn't throw a "Had excess message on node..." error
1548         nodes[0].node.get_and_clear_pending_msg_events();
1549 }
1550
1551 #[test]
1552 fn intercepted_payment() {
1553         // Test that detecting an intercept scid on payment forward will signal LDK to generate an
1554         // intercept event, which the LSP can then use to either (a) open a JIT channel to forward the
1555         // payment or (b) fail the payment.
1556         do_test_intercepted_payment(InterceptTest::Forward);
1557         do_test_intercepted_payment(InterceptTest::Fail);
1558         // Make sure that intercepted payments will be automatically failed back if too many blocks pass.
1559         do_test_intercepted_payment(InterceptTest::Timeout);
1560 }
1561
1562 fn do_test_intercepted_payment(test: InterceptTest) {
1563         let chanmon_cfgs = create_chanmon_cfgs(3);
1564         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1565
1566         let mut zero_conf_chan_config = test_default_channel_config();
1567         zero_conf_chan_config.manually_accept_inbound_channels = true;
1568         let mut intercept_forwards_config = test_default_channel_config();
1569         intercept_forwards_config.accept_intercept_htlcs = true;
1570         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(zero_conf_chan_config)]);
1571
1572         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1573         let scorer = test_utils::TestScorer::new();
1574         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
1575
1576         let _ = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1577
1578         let amt_msat = 100_000;
1579         let intercept_scid = nodes[1].node.get_intercept_scid();
1580         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1581                 .with_route_hints(vec![
1582                         RouteHint(vec![RouteHintHop {
1583                                 src_node_id: nodes[1].node.get_our_node_id(),
1584                                 short_channel_id: intercept_scid,
1585                                 fees: RoutingFees {
1586                                         base_msat: 1000,
1587                                         proportional_millionths: 0,
1588                                 },
1589                                 cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
1590                                 htlc_minimum_msat: None,
1591                                 htlc_maximum_msat: None,
1592                         }])
1593                 ]).unwrap()
1594                 .with_bolt11_features(nodes[2].node.invoice_features()).unwrap();
1595         let route_params = RouteParameters {
1596                 payment_params,
1597                 final_value_msat: amt_msat,
1598         };
1599         let route = get_route(
1600                 &nodes[0].node.get_our_node_id(), &route_params.payment_params,
1601                 &nodes[0].network_graph.read_only(), None, route_params.final_value_msat,
1602                 nodes[0].logger, &scorer, &(), &random_seed_bytes,
1603         ).unwrap();
1604
1605         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
1606         nodes[0].node.send_payment_with_route(&route, payment_hash,
1607                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
1608         let payment_event = {
1609                 {
1610                         let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
1611                         assert_eq!(added_monitors.len(), 1);
1612                         added_monitors.clear();
1613                 }
1614                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1615                 assert_eq!(events.len(), 1);
1616                 SendEvent::from_event(events.remove(0))
1617         };
1618         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1619         commitment_signed_dance!(nodes[1], nodes[0], &payment_event.commitment_msg, false, true);
1620
1621         // Check that we generate the PaymentIntercepted event when an intercept forward is detected.
1622         let events = nodes[1].node.get_and_clear_pending_events();
1623         assert_eq!(events.len(), 1);
1624         let (intercept_id, expected_outbound_amount_msat) = match events[0] {
1625                 crate::events::Event::HTLCIntercepted {
1626                         intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, inbound_amount_msat, requested_next_hop_scid: short_channel_id
1627                 } => {
1628                         assert_eq!(pmt_hash, payment_hash);
1629                         assert_eq!(inbound_amount_msat, route.get_total_amount() + route.get_total_fees());
1630                         assert_eq!(short_channel_id, intercept_scid);
1631                         (intercept_id, expected_outbound_amount_msat)
1632                 },
1633                 _ => panic!()
1634         };
1635
1636         // Check for unknown channel id error.
1637         let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &[42; 32], nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1638         assert_eq!(unknown_chan_id_err , APIError::ChannelUnavailable  { err: format!("Channel with id {} not found for the passed counterparty node_id {}", log_bytes!([42; 32]), nodes[2].node.get_our_node_id()) });
1639
1640         if test == InterceptTest::Fail {
1641                 // Ensure we can fail the intercepted payment back.
1642                 nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap();
1643                 expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::UnknownNextHop { requested_forward_scid: intercept_scid }]);
1644                 nodes[1].node.process_pending_htlc_forwards();
1645                 let update_fail = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1646                 check_added_monitors!(&nodes[1], 1);
1647                 assert!(update_fail.update_fail_htlcs.len() == 1);
1648                 let fail_msg = update_fail.update_fail_htlcs[0].clone();
1649                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1650                 commitment_signed_dance!(nodes[0], nodes[1], update_fail.commitment_signed, false);
1651
1652                 // Ensure the payment fails with the expected error.
1653                 let fail_conditions = PaymentFailedConditions::new()
1654                         .blamed_scid(intercept_scid)
1655                         .blamed_chan_closed(true)
1656                         .expected_htlc_error_data(0x4000 | 10, &[]);
1657                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
1658         } else if test == InterceptTest::Forward {
1659                 // Check that we'll fail as expected when sending to a channel that isn't in `ChannelReady` yet.
1660                 let temp_chan_id = nodes[1].node.create_channel(nodes[2].node.get_our_node_id(), 100_000, 0, 42, None).unwrap();
1661                 let unusable_chan_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &temp_chan_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1662                 assert_eq!(unusable_chan_err , APIError::ChannelUnavailable { err: format!("Channel with id {} not fully established", log_bytes!(temp_chan_id)) });
1663                 assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
1664
1665                 // Open the just-in-time channel so the payment can then be forwarded.
1666                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1667
1668                 // Finally, forward the intercepted payment through and claim it.
1669                 nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap();
1670                 expect_pending_htlcs_forwardable!(nodes[1]);
1671
1672                 let payment_event = {
1673                         {
1674                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
1675                                 assert_eq!(added_monitors.len(), 1);
1676                                 added_monitors.clear();
1677                         }
1678                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
1679                         assert_eq!(events.len(), 1);
1680                         SendEvent::from_event(events.remove(0))
1681                 };
1682                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1683                 commitment_signed_dance!(nodes[2], nodes[1], &payment_event.commitment_msg, false, true);
1684                 expect_pending_htlcs_forwardable!(nodes[2]);
1685
1686                 let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1687                 expect_payment_claimable!(&nodes[2], payment_hash, payment_secret, amt_msat, Some(payment_preimage), nodes[2].node.get_our_node_id());
1688                 do_claim_payment_along_route(&nodes[0], &vec!(&vec!(&nodes[1], &nodes[2])[..]), false, payment_preimage);
1689                 let events = nodes[0].node.get_and_clear_pending_events();
1690                 assert_eq!(events.len(), 2);
1691                 match events[0] {
1692                         Event::PaymentSent { payment_preimage: ref ev_preimage, payment_hash: ref ev_hash, ref fee_paid_msat, .. } => {
1693                                 assert_eq!(payment_preimage, *ev_preimage);
1694                                 assert_eq!(payment_hash, *ev_hash);
1695                                 assert_eq!(fee_paid_msat, &Some(1000));
1696                         },
1697                         _ => panic!("Unexpected event")
1698                 }
1699                 match events[1] {
1700                         Event::PaymentPathSuccessful { payment_hash: hash, .. } => {
1701                                 assert_eq!(hash, Some(payment_hash));
1702                         },
1703                         _ => panic!("Unexpected event")
1704                 }
1705         } else if test == InterceptTest::Timeout {
1706                 let mut block = create_dummy_block(nodes[0].best_block_hash(), 42, Vec::new());
1707                 connect_block(&nodes[0], &block);
1708                 connect_block(&nodes[1], &block);
1709                 for _ in 0..TEST_FINAL_CLTV {
1710                         block.header.prev_blockhash = block.block_hash();
1711                         connect_block(&nodes[0], &block);
1712                         connect_block(&nodes[1], &block);
1713                 }
1714                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::InvalidForward { requested_forward_scid: intercept_scid }]);
1715                 check_added_monitors!(nodes[1], 1);
1716                 let htlc_timeout_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1717                 assert!(htlc_timeout_updates.update_add_htlcs.is_empty());
1718                 assert_eq!(htlc_timeout_updates.update_fail_htlcs.len(), 1);
1719                 assert!(htlc_timeout_updates.update_fail_malformed_htlcs.is_empty());
1720                 assert!(htlc_timeout_updates.update_fee.is_none());
1721
1722                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_timeout_updates.update_fail_htlcs[0]);
1723                 commitment_signed_dance!(nodes[0], nodes[1], htlc_timeout_updates.commitment_signed, false);
1724                 expect_payment_failed!(nodes[0], payment_hash, false, 0x2000 | 2, []);
1725
1726                 // Check for unknown intercept id error.
1727                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1728                 let unknown_intercept_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1729                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1730                 let unknown_intercept_id_err = nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap_err();
1731                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1732         }
1733 }
1734
1735 #[derive(PartialEq)]
1736 enum AutoRetry {
1737         Success,
1738         Spontaneous,
1739         FailAttempts,
1740         FailTimeout,
1741         FailOnRestart,
1742         FailOnRetry,
1743 }
1744
1745 #[test]
1746 fn automatic_retries() {
1747         do_automatic_retries(AutoRetry::Success);
1748         do_automatic_retries(AutoRetry::Spontaneous);
1749         do_automatic_retries(AutoRetry::FailAttempts);
1750         do_automatic_retries(AutoRetry::FailTimeout);
1751         do_automatic_retries(AutoRetry::FailOnRestart);
1752         do_automatic_retries(AutoRetry::FailOnRetry);
1753 }
1754 fn do_automatic_retries(test: AutoRetry) {
1755         // Test basic automatic payment retries in ChannelManager. See individual `test` variant comments
1756         // below.
1757         let chanmon_cfgs = create_chanmon_cfgs(3);
1758         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1759         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1760
1761         let persister;
1762         let new_chain_monitor;
1763         let node_0_deserialized;
1764
1765         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1766         let channel_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1767         let channel_id_2 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1768
1769         // Marshall data to send the payment
1770         #[cfg(feature = "std")]
1771         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1772         #[cfg(not(feature = "std"))]
1773         let payment_expiry_secs = 60 * 60;
1774         let amt_msat = 1000;
1775         let mut invoice_features = InvoiceFeatures::empty();
1776         invoice_features.set_variable_length_onion_required();
1777         invoice_features.set_payment_secret_required();
1778         invoice_features.set_basic_mpp_optional();
1779         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1780                 .with_expiry_time(payment_expiry_secs as u64)
1781                 .with_bolt11_features(invoice_features).unwrap();
1782         let route_params = RouteParameters {
1783                 payment_params,
1784                 final_value_msat: amt_msat,
1785         };
1786         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
1787
1788         macro_rules! pass_failed_attempt_with_retry_along_path {
1789                 ($failing_channel_id: expr, $expect_pending_htlcs_forwardable: expr) => {
1790                         // Send a payment attempt that fails due to lack of liquidity on the second hop
1791                         check_added_monitors!(nodes[0], 1);
1792                         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1793                         let mut update_add = update_0.update_add_htlcs[0].clone();
1794                         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
1795                         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
1796                         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
1797                         nodes[1].node.process_pending_htlc_forwards();
1798                         expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1],
1799                                 vec![HTLCDestination::NextHopChannel {
1800                                         node_id: Some(nodes[2].node.get_our_node_id()),
1801                                         channel_id: $failing_channel_id,
1802                                 }]);
1803                         nodes[1].node.process_pending_htlc_forwards();
1804                         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1805                         check_added_monitors!(&nodes[1], 1);
1806                         assert!(update_1.update_fail_htlcs.len() == 1);
1807                         let fail_msg = update_1.update_fail_htlcs[0].clone();
1808                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1809                         commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
1810
1811                         // Ensure the attempt fails and a new PendingHTLCsForwardable event is generated for the retry
1812                         let mut events = nodes[0].node.get_and_clear_pending_events();
1813                         assert_eq!(events.len(), 2);
1814                         match events[0] {
1815                                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
1816                                         assert_eq!(payment_hash, ev_payment_hash);
1817                                         assert_eq!(payment_failed_permanently, false);
1818                                 },
1819                                 _ => panic!("Unexpected event"),
1820                         }
1821                         if $expect_pending_htlcs_forwardable {
1822                                 match events[1] {
1823                                         Event::PendingHTLCsForwardable { .. } => {},
1824                                         _ => panic!("Unexpected event"),
1825                                 }
1826                         } else {
1827                                 match events[1] {
1828                                         Event::PaymentFailed { payment_hash: ev_payment_hash, .. } => {
1829                                                 assert_eq!(payment_hash, ev_payment_hash);
1830                                         },
1831                                         _ => panic!("Unexpected event"),
1832                                 }
1833                         }
1834                 }
1835         }
1836
1837         if test == AutoRetry::Success {
1838                 // Test that we can succeed on the first retry.
1839                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
1840                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1841                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1842
1843                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
1844                 // attempt, since the initial second hop channel will be excluded from pathfinding
1845                 create_announced_chan_between_nodes(&nodes, 1, 2);
1846
1847                 // We retry payments in `process_pending_htlc_forwards`
1848                 nodes[0].node.process_pending_htlc_forwards();
1849                 check_added_monitors!(nodes[0], 1);
1850                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1851                 assert_eq!(msg_events.len(), 1);
1852                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, Some(payment_secret), msg_events.pop().unwrap(), true, None);
1853                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
1854         } else if test == AutoRetry::Spontaneous {
1855                 nodes[0].node.send_spontaneous_payment_with_retry(Some(payment_preimage),
1856                         RecipientOnionFields::spontaneous_empty(), PaymentId(payment_hash.0), route_params,
1857                         Retry::Attempts(1)).unwrap();
1858                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1859
1860                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
1861                 // attempt, since the initial second hop channel will be excluded from pathfinding
1862                 create_announced_chan_between_nodes(&nodes, 1, 2);
1863
1864                 // We retry payments in `process_pending_htlc_forwards`
1865                 nodes[0].node.process_pending_htlc_forwards();
1866                 check_added_monitors!(nodes[0], 1);
1867                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1868                 assert_eq!(msg_events.len(), 1);
1869                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, None, msg_events.pop().unwrap(), true, Some(payment_preimage));
1870                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
1871         } else if test == AutoRetry::FailAttempts {
1872                 // Ensure ChannelManager will not retry a payment if it has run out of payment attempts.
1873                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
1874                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1875                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1876
1877                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
1878                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
1879                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1880
1881                 // We retry payments in `process_pending_htlc_forwards`
1882                 nodes[0].node.process_pending_htlc_forwards();
1883                 pass_failed_attempt_with_retry_along_path!(channel_id_3, false);
1884
1885                 // Ensure we won't retry a second time.
1886                 nodes[0].node.process_pending_htlc_forwards();
1887                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1888                 assert_eq!(msg_events.len(), 0);
1889         } else if test == AutoRetry::FailTimeout {
1890                 #[cfg(not(feature = "no-std"))] {
1891                         // Ensure ChannelManager will not retry a payment if it times out due to Retry::Timeout.
1892                         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
1893                                 PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
1894                         pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1895
1896                         // Advance the time so the second attempt fails due to timeout.
1897                         SinceEpoch::advance(Duration::from_secs(61));
1898
1899                         // Make sure we don't retry again.
1900                         nodes[0].node.process_pending_htlc_forwards();
1901                         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1902                         assert_eq!(msg_events.len(), 0);
1903
1904                         let mut events = nodes[0].node.get_and_clear_pending_events();
1905                         assert_eq!(events.len(), 1);
1906                         match events[0] {
1907                                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
1908                                         assert_eq!(payment_hash, *ev_payment_hash);
1909                                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1910                                         assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
1911                                 },
1912                                 _ => panic!("Unexpected event"),
1913                         }
1914                 }
1915         } else if test == AutoRetry::FailOnRestart {
1916                 // Ensure ChannelManager will not retry a payment after restart, even if there were retry
1917                 // attempts remaining prior to restart.
1918                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
1919                         PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
1920                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1921
1922                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
1923                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
1924                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1925
1926                 // Ensure the first retry attempt fails, with 1 retry attempt remaining
1927                 nodes[0].node.process_pending_htlc_forwards();
1928                 pass_failed_attempt_with_retry_along_path!(channel_id_3, true);
1929
1930                 // Restart the node and ensure that ChannelManager does not use its remaining retry attempt
1931                 let node_encoded = nodes[0].node.encode();
1932                 let chan_1_monitor_serialized = get_monitor!(nodes[0], channel_id_1).encode();
1933                 reload_node!(nodes[0], node_encoded, &[&chan_1_monitor_serialized], persister, new_chain_monitor, node_0_deserialized);
1934
1935                 let mut events = nodes[0].node.get_and_clear_pending_events();
1936                 expect_pending_htlcs_forwardable_from_events!(nodes[0], events, true);
1937                 // Make sure we don't retry again.
1938                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1939                 assert_eq!(msg_events.len(), 0);
1940
1941                 let mut events = nodes[0].node.get_and_clear_pending_events();
1942                 assert_eq!(events.len(), 1);
1943                 match events[0] {
1944                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
1945                                 assert_eq!(payment_hash, *ev_payment_hash);
1946                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1947                                 assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
1948                         },
1949                         _ => panic!("Unexpected event"),
1950                 }
1951         } else if test == AutoRetry::FailOnRetry {
1952                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
1953                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1954                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1955
1956                 // We retry payments in `process_pending_htlc_forwards`. Since our channel closed, we should
1957                 // fail to find a route.
1958                 nodes[0].node.process_pending_htlc_forwards();
1959                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1960                 assert_eq!(msg_events.len(), 0);
1961
1962                 let mut events = nodes[0].node.get_and_clear_pending_events();
1963                 assert_eq!(events.len(), 1);
1964                 match events[0] {
1965                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
1966                                 assert_eq!(payment_hash, *ev_payment_hash);
1967                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1968                                 assert_eq!(PaymentFailureReason::RouteNotFound, ev_reason.unwrap());
1969                         },
1970                         _ => panic!("Unexpected event"),
1971                 }
1972         }
1973 }
1974
1975 #[test]
1976 fn auto_retry_partial_failure() {
1977         // Test that we'll retry appropriately on send partial failure and retry partial failure.
1978         let chanmon_cfgs = create_chanmon_cfgs(2);
1979         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1980         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1981         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1982
1983         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1984         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1985         let chan_3_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1986
1987         // Marshall data to send the payment
1988         let amt_msat = 20_000;
1989         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
1990         #[cfg(feature = "std")]
1991         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1992         #[cfg(not(feature = "std"))]
1993         let payment_expiry_secs = 60 * 60;
1994         let mut invoice_features = InvoiceFeatures::empty();
1995         invoice_features.set_variable_length_onion_required();
1996         invoice_features.set_payment_secret_required();
1997         invoice_features.set_basic_mpp_optional();
1998         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
1999                 .with_expiry_time(payment_expiry_secs as u64)
2000                 .with_bolt11_features(invoice_features).unwrap();
2001         let route_params = RouteParameters {
2002                 payment_params,
2003                 final_value_msat: amt_msat,
2004         };
2005
2006         // Ensure the first monitor update (for the initial send path1 over chan_1) succeeds, but the
2007         // second (for the initial send path2 over chan_2) fails.
2008         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
2009         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2010         // Ensure third monitor update (for the retry1's path1 over chan_1) succeeds, but the fourth (for
2011         // the retry1's path2 over chan_3) fails, and monitor updates succeed after that.
2012         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
2013         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2014         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
2015
2016         // Configure the initial send, retry1 and retry2's paths.
2017         let send_route = Route {
2018                 paths: vec![
2019                         Path { hops: vec![RouteHop {
2020                                 pubkey: nodes[1].node.get_our_node_id(),
2021                                 node_features: nodes[1].node.node_features(),
2022                                 short_channel_id: chan_1_id,
2023                                 channel_features: nodes[1].node.channel_features(),
2024                                 fee_msat: amt_msat / 2,
2025                                 cltv_expiry_delta: 100,
2026                         }], blinded_tail: None },
2027                         Path { hops: vec![RouteHop {
2028                                 pubkey: nodes[1].node.get_our_node_id(),
2029                                 node_features: nodes[1].node.node_features(),
2030                                 short_channel_id: chan_2_id,
2031                                 channel_features: nodes[1].node.channel_features(),
2032                                 fee_msat: amt_msat / 2,
2033                                 cltv_expiry_delta: 100,
2034                         }], blinded_tail: None },
2035                 ],
2036                 payment_params: Some(route_params.payment_params.clone()),
2037         };
2038         let retry_1_route = Route {
2039                 paths: vec![
2040                         Path { hops: vec![RouteHop {
2041                                 pubkey: nodes[1].node.get_our_node_id(),
2042                                 node_features: nodes[1].node.node_features(),
2043                                 short_channel_id: chan_1_id,
2044                                 channel_features: nodes[1].node.channel_features(),
2045                                 fee_msat: amt_msat / 4,
2046                                 cltv_expiry_delta: 100,
2047                         }], blinded_tail: None },
2048                         Path { hops: vec![RouteHop {
2049                                 pubkey: nodes[1].node.get_our_node_id(),
2050                                 node_features: nodes[1].node.node_features(),
2051                                 short_channel_id: chan_3_id,
2052                                 channel_features: nodes[1].node.channel_features(),
2053                                 fee_msat: amt_msat / 4,
2054                                 cltv_expiry_delta: 100,
2055                         }], blinded_tail: None },
2056                 ],
2057                 payment_params: Some(route_params.payment_params.clone()),
2058         };
2059         let retry_2_route = Route {
2060                 paths: vec![
2061                         Path { hops: vec![RouteHop {
2062                                 pubkey: nodes[1].node.get_our_node_id(),
2063                                 node_features: nodes[1].node.node_features(),
2064                                 short_channel_id: chan_1_id,
2065                                 channel_features: nodes[1].node.channel_features(),
2066                                 fee_msat: amt_msat / 4,
2067                                 cltv_expiry_delta: 100,
2068                         }], blinded_tail: None },
2069                 ],
2070                 payment_params: Some(route_params.payment_params.clone()),
2071         };
2072         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
2073         let mut payment_params = route_params.payment_params.clone();
2074         payment_params.previously_failed_channels.push(chan_2_id);
2075         nodes[0].router.expect_find_route(RouteParameters {
2076                         payment_params, final_value_msat: amt_msat / 2,
2077                 }, Ok(retry_1_route));
2078         let mut payment_params = route_params.payment_params.clone();
2079         payment_params.previously_failed_channels.push(chan_3_id);
2080         nodes[0].router.expect_find_route(RouteParameters {
2081                         payment_params, final_value_msat: amt_msat / 4,
2082                 }, Ok(retry_2_route));
2083
2084         // Send a payment that will partially fail on send, then partially fail on retry, then succeed.
2085         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2086                 PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap();
2087         let closed_chan_events = nodes[0].node.get_and_clear_pending_events();
2088         assert_eq!(closed_chan_events.len(), 4);
2089         match closed_chan_events[0] {
2090                 Event::ChannelClosed { .. } => {},
2091                 _ => panic!("Unexpected event"),
2092         }
2093         match closed_chan_events[1] {
2094                 Event::PaymentPathFailed { .. } => {},
2095                 _ => panic!("Unexpected event"),
2096         }
2097         match closed_chan_events[2] {
2098                 Event::ChannelClosed { .. } => {},
2099                 _ => panic!("Unexpected event"),
2100         }
2101         match closed_chan_events[3] {
2102                 Event::PaymentPathFailed { .. } => {},
2103                 _ => panic!("Unexpected event"),
2104         }
2105
2106         // Pass the first part of the payment along the path.
2107         check_added_monitors!(nodes[0], 5); // three outbound channel updates succeeded, two permanently failed
2108         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2109
2110         // First message is the first update_add, remaining messages are broadcasting channel updates and
2111         // errors for the permfailed channels
2112         assert_eq!(msg_events.len(), 5);
2113         let mut payment_event = SendEvent::from_event(msg_events.remove(0));
2114
2115         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2116         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
2117         check_added_monitors!(nodes[1], 1);
2118         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2119
2120         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2121         check_added_monitors!(nodes[0], 1);
2122         let as_second_htlc_updates = SendEvent::from_node(&nodes[0]);
2123
2124         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2125         check_added_monitors!(nodes[0], 1);
2126         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2127
2128         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2129         check_added_monitors!(nodes[1], 1);
2130
2131         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[0]);
2132         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[1]);
2133         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.commitment_msg);
2134         check_added_monitors!(nodes[1], 1);
2135         let (bs_second_raa, bs_second_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2136
2137         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2138         check_added_monitors!(nodes[0], 1);
2139
2140         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_cs);
2141         check_added_monitors!(nodes[0], 1);
2142         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2143
2144         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2145         check_added_monitors!(nodes[1], 1);
2146
2147         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
2148         nodes[1].node.process_pending_htlc_forwards();
2149         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, amt_msat);
2150         nodes[1].node.claim_funds(payment_preimage);
2151         expect_payment_claimed!(nodes[1], payment_hash, amt_msat);
2152         let bs_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2153         assert_eq!(bs_claim_update.update_fulfill_htlcs.len(), 1);
2154
2155         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_claim_update.update_fulfill_htlcs[0]);
2156         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_claim_update.commitment_signed);
2157         check_added_monitors!(nodes[0], 1);
2158         let (as_third_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2159
2160         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2161         check_added_monitors!(nodes[1], 4);
2162         let bs_second_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2163
2164         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2165         check_added_monitors!(nodes[1], 1);
2166         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2167
2168         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2169         check_added_monitors!(nodes[0], 1);
2170
2171         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[0]);
2172         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[1]);
2173         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.commitment_signed);
2174         check_added_monitors!(nodes[0], 1);
2175         let (as_fourth_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2176
2177         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_fourth_raa);
2178         check_added_monitors!(nodes[1], 1);
2179
2180         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2181         check_added_monitors!(nodes[1], 1);
2182         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2183
2184         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2185         check_added_monitors!(nodes[0], 1);
2186         expect_payment_sent!(nodes[0], payment_preimage);
2187 }
2188
2189 #[test]
2190 fn auto_retry_zero_attempts_send_error() {
2191         let chanmon_cfgs = create_chanmon_cfgs(2);
2192         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2193         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2194         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2195
2196         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2197         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2198
2199         // Marshall data to send the payment
2200         let amt_msat = 20_000;
2201         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2202         #[cfg(feature = "std")]
2203         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2204         #[cfg(not(feature = "std"))]
2205         let payment_expiry_secs = 60 * 60;
2206         let mut invoice_features = InvoiceFeatures::empty();
2207         invoice_features.set_variable_length_onion_required();
2208         invoice_features.set_payment_secret_required();
2209         invoice_features.set_basic_mpp_optional();
2210         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2211                 .with_expiry_time(payment_expiry_secs as u64)
2212                 .with_bolt11_features(invoice_features).unwrap();
2213         let route_params = RouteParameters {
2214                 payment_params,
2215                 final_value_msat: amt_msat,
2216         };
2217
2218         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2219         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2220                 PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
2221         assert_eq!(nodes[0].node.get_and_clear_pending_msg_events().len(), 2); // channel close messages
2222         let events = nodes[0].node.get_and_clear_pending_events();
2223         assert_eq!(events.len(), 3);
2224         if let Event::ChannelClosed { .. } = events[0] { } else { panic!(); }
2225         if let Event::PaymentPathFailed { .. } = events[1] { } else { panic!(); }
2226         if let Event::PaymentFailed { .. } = events[2] { } else { panic!(); }
2227         check_added_monitors!(nodes[0], 2);
2228 }
2229
2230 #[test]
2231 fn fails_paying_after_rejected_by_payee() {
2232         let chanmon_cfgs = create_chanmon_cfgs(2);
2233         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2234         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2235         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2236
2237         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2238
2239         // Marshall data to send the payment
2240         let amt_msat = 20_000;
2241         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2242         #[cfg(feature = "std")]
2243         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2244         #[cfg(not(feature = "std"))]
2245         let payment_expiry_secs = 60 * 60;
2246         let mut invoice_features = InvoiceFeatures::empty();
2247         invoice_features.set_variable_length_onion_required();
2248         invoice_features.set_payment_secret_required();
2249         invoice_features.set_basic_mpp_optional();
2250         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2251                 .with_expiry_time(payment_expiry_secs as u64)
2252                 .with_bolt11_features(invoice_features).unwrap();
2253         let route_params = RouteParameters {
2254                 payment_params,
2255                 final_value_msat: amt_msat,
2256         };
2257
2258         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2259                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2260         check_added_monitors!(nodes[0], 1);
2261         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
2262         assert_eq!(events.len(), 1);
2263         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
2264         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2265         check_added_monitors!(nodes[1], 0);
2266         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
2267         expect_pending_htlcs_forwardable!(nodes[1]);
2268         expect_payment_claimable!(&nodes[1], payment_hash, payment_secret, amt_msat);
2269
2270         nodes[1].node.fail_htlc_backwards(&payment_hash);
2271         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash }]);
2272         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
2273 }
2274
2275 #[test]
2276 fn retry_multi_path_single_failed_payment() {
2277         // Tests that we can/will retry after a single path of an MPP payment failed immediately
2278         let chanmon_cfgs = create_chanmon_cfgs(2);
2279         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2280         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2281         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2282
2283         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2284         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2285
2286         let amt_msat = 100_010_000;
2287
2288         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2289         #[cfg(feature = "std")]
2290         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2291         #[cfg(not(feature = "std"))]
2292         let payment_expiry_secs = 60 * 60;
2293         let mut invoice_features = InvoiceFeatures::empty();
2294         invoice_features.set_variable_length_onion_required();
2295         invoice_features.set_payment_secret_required();
2296         invoice_features.set_basic_mpp_optional();
2297         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2298                 .with_expiry_time(payment_expiry_secs as u64)
2299                 .with_bolt11_features(invoice_features).unwrap();
2300         let route_params = RouteParameters {
2301                 payment_params: payment_params.clone(),
2302                 final_value_msat: amt_msat,
2303         };
2304
2305         let chans = nodes[0].node.list_usable_channels();
2306         let mut route = Route {
2307                 paths: vec![
2308                         Path { hops: vec![RouteHop {
2309                                 pubkey: nodes[1].node.get_our_node_id(),
2310                                 node_features: nodes[1].node.node_features(),
2311                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2312                                 channel_features: nodes[1].node.channel_features(),
2313                                 fee_msat: 10_000,
2314                                 cltv_expiry_delta: 100,
2315                         }], blinded_tail: None },
2316                         Path { hops: vec![RouteHop {
2317                                 pubkey: nodes[1].node.get_our_node_id(),
2318                                 node_features: nodes[1].node.node_features(),
2319                                 short_channel_id: chans[1].short_channel_id.unwrap(),
2320                                 channel_features: nodes[1].node.channel_features(),
2321                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2322                                 cltv_expiry_delta: 100,
2323                         }], blinded_tail: None },
2324                 ],
2325                 payment_params: Some(payment_params),
2326         };
2327         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2328         // On retry, split the payment across both channels.
2329         route.paths[0].hops[0].fee_msat = 50_000_001;
2330         route.paths[1].hops[0].fee_msat = 50_000_000;
2331         let mut pay_params = route.payment_params.clone().unwrap();
2332         pay_params.previously_failed_channels.push(chans[1].short_channel_id.unwrap());
2333         nodes[0].router.expect_find_route(RouteParameters {
2334                         payment_params: pay_params,
2335                         // Note that the second request here requests the amount we originally failed to send,
2336                         // not the amount remaining on the full payment, which should be changed.
2337                         final_value_msat: 100_000_001,
2338                 }, Ok(route.clone()));
2339
2340         {
2341                 let scorer = chanmon_cfgs[0].scorer.lock().unwrap();
2342                 // The initial send attempt, 2 paths
2343                 scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 10_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2344                 scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 100_000_001, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2345                 // The retry, 2 paths. Ensure that the in-flight HTLC amount is factored in.
2346                 scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_001, inflight_htlc_msat: 10_000, effective_capacity: EffectiveCapacity::Unknown });
2347                 scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2348         }
2349
2350         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2351                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2352         let events = nodes[0].node.get_and_clear_pending_events();
2353         assert_eq!(events.len(), 1);
2354         match events[0] {
2355                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false,
2356                         failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }},
2357                         short_channel_id: Some(expected_scid), .. } =>
2358                 {
2359                         assert_eq!(payment_hash, ev_payment_hash);
2360                         assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
2361                 },
2362                 _ => panic!("Unexpected event"),
2363         }
2364         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2365         assert_eq!(htlc_msgs.len(), 2);
2366         check_added_monitors!(nodes[0], 2);
2367 }
2368
2369 #[test]
2370 fn immediate_retry_on_failure() {
2371         // Tests that we can/will retry immediately after a failure
2372         let chanmon_cfgs = create_chanmon_cfgs(2);
2373         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2374         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2375         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2376
2377         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2378         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2379
2380         let amt_msat = 100_000_001;
2381         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2382         #[cfg(feature = "std")]
2383         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2384         #[cfg(not(feature = "std"))]
2385         let payment_expiry_secs = 60 * 60;
2386         let mut invoice_features = InvoiceFeatures::empty();
2387         invoice_features.set_variable_length_onion_required();
2388         invoice_features.set_payment_secret_required();
2389         invoice_features.set_basic_mpp_optional();
2390         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2391                 .with_expiry_time(payment_expiry_secs as u64)
2392                 .with_bolt11_features(invoice_features).unwrap();
2393         let route_params = RouteParameters {
2394                 payment_params,
2395                 final_value_msat: amt_msat,
2396         };
2397
2398         let chans = nodes[0].node.list_usable_channels();
2399         let mut route = Route {
2400                 paths: vec![
2401                         Path { hops: vec![RouteHop {
2402                                 pubkey: nodes[1].node.get_our_node_id(),
2403                                 node_features: nodes[1].node.node_features(),
2404                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2405                                 channel_features: nodes[1].node.channel_features(),
2406                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2407                                 cltv_expiry_delta: 100,
2408                         }], blinded_tail: None },
2409                 ],
2410                 payment_params: Some(PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)),
2411         };
2412         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2413         // On retry, split the payment across both channels.
2414         route.paths.push(route.paths[0].clone());
2415         route.paths[0].hops[0].short_channel_id = chans[1].short_channel_id.unwrap();
2416         route.paths[0].hops[0].fee_msat = 50_000_000;
2417         route.paths[1].hops[0].fee_msat = 50_000_001;
2418         let mut pay_params = route_params.payment_params.clone();
2419         pay_params.previously_failed_channels.push(chans[0].short_channel_id.unwrap());
2420         nodes[0].router.expect_find_route(RouteParameters {
2421                         payment_params: pay_params, final_value_msat: amt_msat,
2422                 }, Ok(route.clone()));
2423
2424         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2425                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2426         let events = nodes[0].node.get_and_clear_pending_events();
2427         assert_eq!(events.len(), 1);
2428         match events[0] {
2429                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false,
2430                         failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }},
2431                         short_channel_id: Some(expected_scid), .. } =>
2432                 {
2433                         assert_eq!(payment_hash, ev_payment_hash);
2434                         assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
2435                 },
2436                 _ => panic!("Unexpected event"),
2437         }
2438         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2439         assert_eq!(htlc_msgs.len(), 2);
2440         check_added_monitors!(nodes[0], 2);
2441 }
2442
2443 #[test]
2444 fn no_extra_retries_on_back_to_back_fail() {
2445         // In a previous release, we had a race where we may exceed the payment retry count if we
2446         // get two failures in a row with the second indicating that all paths had failed (this field,
2447         // `all_paths_failed`, has since been removed).
2448         // Generally, when we give up trying to retry a payment, we don't know for sure what the
2449         // current state of the ChannelManager event queue is. Specifically, we cannot be sure that
2450         // there are not multiple additional `PaymentPathFailed` or even `PaymentSent` events
2451         // pending which we will see later. Thus, when we previously removed the retry tracking map
2452         // entry after a `all_paths_failed` `PaymentPathFailed` event, we may have dropped the
2453         // retry entry even though more events for the same payment were still pending. This led to
2454         // us retrying a payment again even though we'd already given up on it.
2455         //
2456         // We now have a separate event - `PaymentFailed` which indicates no HTLCs remain and which
2457         // is used to remove the payment retry counter entries instead. This tests for the specific
2458         // excess-retry case while also testing `PaymentFailed` generation.
2459
2460         let chanmon_cfgs = create_chanmon_cfgs(3);
2461         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2462         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2463         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2464
2465         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2466         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2467
2468         let amt_msat = 200_000_000;
2469         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2470         #[cfg(feature = "std")]
2471         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2472         #[cfg(not(feature = "std"))]
2473         let payment_expiry_secs = 60 * 60;
2474         let mut invoice_features = InvoiceFeatures::empty();
2475         invoice_features.set_variable_length_onion_required();
2476         invoice_features.set_payment_secret_required();
2477         invoice_features.set_basic_mpp_optional();
2478         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2479                 .with_expiry_time(payment_expiry_secs as u64)
2480                 .with_bolt11_features(invoice_features).unwrap();
2481         let route_params = RouteParameters {
2482                 payment_params,
2483                 final_value_msat: amt_msat,
2484         };
2485
2486         let mut route = Route {
2487                 paths: vec![
2488                         Path { hops: vec![RouteHop {
2489                                 pubkey: nodes[1].node.get_our_node_id(),
2490                                 node_features: nodes[1].node.node_features(),
2491                                 short_channel_id: chan_1_scid,
2492                                 channel_features: nodes[1].node.channel_features(),
2493                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2494                                 cltv_expiry_delta: 100,
2495                         }, RouteHop {
2496                                 pubkey: nodes[2].node.get_our_node_id(),
2497                                 node_features: nodes[2].node.node_features(),
2498                                 short_channel_id: chan_2_scid,
2499                                 channel_features: nodes[2].node.channel_features(),
2500                                 fee_msat: 100_000_000,
2501                                 cltv_expiry_delta: 100,
2502                         }], blinded_tail: None },
2503                         Path { hops: vec![RouteHop {
2504                                 pubkey: nodes[1].node.get_our_node_id(),
2505                                 node_features: nodes[1].node.node_features(),
2506                                 short_channel_id: chan_1_scid,
2507                                 channel_features: nodes[1].node.channel_features(),
2508                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2509                                 cltv_expiry_delta: 100,
2510                         }, RouteHop {
2511                                 pubkey: nodes[2].node.get_our_node_id(),
2512                                 node_features: nodes[2].node.node_features(),
2513                                 short_channel_id: chan_2_scid,
2514                                 channel_features: nodes[2].node.channel_features(),
2515                                 fee_msat: 100_000_000,
2516                                 cltv_expiry_delta: 100,
2517                         }], blinded_tail: None }
2518                 ],
2519                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2520         };
2521         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2522         let mut second_payment_params = route_params.payment_params.clone();
2523         second_payment_params.previously_failed_channels = vec![chan_2_scid, chan_2_scid];
2524         // On retry, we'll only return one path
2525         route.paths.remove(1);
2526         route.paths[0].hops[1].fee_msat = amt_msat;
2527         nodes[0].router.expect_find_route(RouteParameters {
2528                         payment_params: second_payment_params,
2529                         final_value_msat: amt_msat,
2530                 }, Ok(route.clone()));
2531
2532         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2533                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2534         let htlc_updates = SendEvent::from_node(&nodes[0]);
2535         check_added_monitors!(nodes[0], 1);
2536         assert_eq!(htlc_updates.msgs.len(), 1);
2537
2538         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2539         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2540         check_added_monitors!(nodes[1], 1);
2541         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2542
2543         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2544         check_added_monitors!(nodes[0], 1);
2545         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2546
2547         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2548         check_added_monitors!(nodes[0], 1);
2549         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2550
2551         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2552         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2553         check_added_monitors!(nodes[1], 1);
2554         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2555
2556         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2557         check_added_monitors!(nodes[1], 1);
2558         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2559
2560         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2561         check_added_monitors!(nodes[0], 1);
2562
2563         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2564         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2565         check_added_monitors!(nodes[0], 1);
2566         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2567
2568         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2569         check_added_monitors!(nodes[1], 1);
2570         let bs_second_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2571
2572         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2573         check_added_monitors!(nodes[1], 1);
2574         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2575
2576         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.update_fail_htlcs[0]);
2577         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.commitment_signed);
2578         check_added_monitors!(nodes[0], 1);
2579
2580         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2581         check_added_monitors!(nodes[0], 1);
2582         let (as_third_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2583
2584         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2585         check_added_monitors!(nodes[1], 1);
2586         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2587         check_added_monitors!(nodes[1], 1);
2588         let bs_fourth_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2589
2590         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_fourth_raa);
2591         check_added_monitors!(nodes[0], 1);
2592
2593         // At this point A has sent two HTLCs which both failed due to lack of fee. It now has two
2594         // pending `PaymentPathFailed` events, one with `all_paths_failed` unset, and the second
2595         // with it set.
2596         //
2597         // Previously, we retried payments in an event consumer, which would retry each
2598         // `PaymentPathFailed` individually. In that setup, we had retried the payment in response to
2599         // the first `PaymentPathFailed`, then seen the second `PaymentPathFailed` with
2600         // `all_paths_failed` set and assumed the payment was completely failed. We ultimately fixed it
2601         // by adding the `PaymentFailed` event.
2602         //
2603         // Because we now retry payments as a batch, we simply return a single-path route in the
2604         // second, batched, request, have that fail, ensure the payment was abandoned.
2605         let mut events = nodes[0].node.get_and_clear_pending_events();
2606         assert_eq!(events.len(), 3);
2607         match events[0] {
2608                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2609                         assert_eq!(payment_hash, ev_payment_hash);
2610                         assert_eq!(payment_failed_permanently, false);
2611                 },
2612                 _ => panic!("Unexpected event"),
2613         }
2614         match events[1] {
2615                 Event::PendingHTLCsForwardable { .. } => {},
2616                 _ => panic!("Unexpected event"),
2617         }
2618         match events[2] {
2619                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2620                         assert_eq!(payment_hash, ev_payment_hash);
2621                         assert_eq!(payment_failed_permanently, false);
2622                 },
2623                 _ => panic!("Unexpected event"),
2624         }
2625
2626         nodes[0].node.process_pending_htlc_forwards();
2627         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2628         check_added_monitors!(nodes[0], 1);
2629
2630         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2631         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2632         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2633         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2634         commitment_signed_dance!(nodes[0], nodes[1], &bs_fail_update.commitment_signed, false, true);
2635
2636         let mut events = nodes[0].node.get_and_clear_pending_events();
2637         assert_eq!(events.len(), 2);
2638         match events[0] {
2639                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2640                         assert_eq!(payment_hash, ev_payment_hash);
2641                         assert_eq!(payment_failed_permanently, false);
2642                 },
2643                 _ => panic!("Unexpected event"),
2644         }
2645         match events[1] {
2646                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2647                         assert_eq!(payment_hash, *ev_payment_hash);
2648                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2649                         assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
2650                 },
2651                 _ => panic!("Unexpected event"),
2652         }
2653 }
2654
2655 #[test]
2656 fn test_simple_partial_retry() {
2657         // In the first version of the in-`ChannelManager` payment retries, retries were sent for the
2658         // full amount of the payment, rather than only the missing amount. Here we simply test for
2659         // this by sending a payment with two parts, failing one, and retrying the second. Note that
2660         // `TestRouter` will check that the `RouteParameters` (which contain the amount) matches the
2661         // request.
2662         let chanmon_cfgs = create_chanmon_cfgs(3);
2663         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2664         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2665         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2666
2667         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2668         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2669
2670         let amt_msat = 200_000_000;
2671         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
2672         #[cfg(feature = "std")]
2673         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2674         #[cfg(not(feature = "std"))]
2675         let payment_expiry_secs = 60 * 60;
2676         let mut invoice_features = InvoiceFeatures::empty();
2677         invoice_features.set_variable_length_onion_required();
2678         invoice_features.set_payment_secret_required();
2679         invoice_features.set_basic_mpp_optional();
2680         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2681                 .with_expiry_time(payment_expiry_secs as u64)
2682                 .with_bolt11_features(invoice_features).unwrap();
2683         let route_params = RouteParameters {
2684                 payment_params,
2685                 final_value_msat: amt_msat,
2686         };
2687
2688         let mut route = Route {
2689                 paths: vec![
2690                         Path { hops: vec![RouteHop {
2691                                 pubkey: nodes[1].node.get_our_node_id(),
2692                                 node_features: nodes[1].node.node_features(),
2693                                 short_channel_id: chan_1_scid,
2694                                 channel_features: nodes[1].node.channel_features(),
2695                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2696                                 cltv_expiry_delta: 100,
2697                         }, RouteHop {
2698                                 pubkey: nodes[2].node.get_our_node_id(),
2699                                 node_features: nodes[2].node.node_features(),
2700                                 short_channel_id: chan_2_scid,
2701                                 channel_features: nodes[2].node.channel_features(),
2702                                 fee_msat: 100_000_000,
2703                                 cltv_expiry_delta: 100,
2704                         }], blinded_tail: None },
2705                         Path { hops: vec![RouteHop {
2706                                 pubkey: nodes[1].node.get_our_node_id(),
2707                                 node_features: nodes[1].node.node_features(),
2708                                 short_channel_id: chan_1_scid,
2709                                 channel_features: nodes[1].node.channel_features(),
2710                                 fee_msat: 100_000,
2711                                 cltv_expiry_delta: 100,
2712                         }, RouteHop {
2713                                 pubkey: nodes[2].node.get_our_node_id(),
2714                                 node_features: nodes[2].node.node_features(),
2715                                 short_channel_id: chan_2_scid,
2716                                 channel_features: nodes[2].node.channel_features(),
2717                                 fee_msat: 100_000_000,
2718                                 cltv_expiry_delta: 100,
2719                         }], blinded_tail: None }
2720                 ],
2721                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2722         };
2723         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2724         let mut second_payment_params = route_params.payment_params.clone();
2725         second_payment_params.previously_failed_channels = vec![chan_2_scid];
2726         // On retry, we'll only be asked for one path (or 100k sats)
2727         route.paths.remove(0);
2728         nodes[0].router.expect_find_route(RouteParameters {
2729                         payment_params: second_payment_params,
2730                         final_value_msat: amt_msat / 2,
2731                 }, Ok(route.clone()));
2732
2733         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2734                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2735         let htlc_updates = SendEvent::from_node(&nodes[0]);
2736         check_added_monitors!(nodes[0], 1);
2737         assert_eq!(htlc_updates.msgs.len(), 1);
2738
2739         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2740         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2741         check_added_monitors!(nodes[1], 1);
2742         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2743
2744         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2745         check_added_monitors!(nodes[0], 1);
2746         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2747
2748         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2749         check_added_monitors!(nodes[0], 1);
2750         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2751
2752         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2753         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2754         check_added_monitors!(nodes[1], 1);
2755         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2756
2757         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2758         check_added_monitors!(nodes[1], 1);
2759         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2760
2761         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2762         check_added_monitors!(nodes[0], 1);
2763
2764         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2765         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2766         check_added_monitors!(nodes[0], 1);
2767         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2768
2769         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2770         check_added_monitors!(nodes[1], 1);
2771
2772         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2773         check_added_monitors!(nodes[1], 1);
2774
2775         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2776
2777         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2778         check_added_monitors!(nodes[0], 1);
2779
2780         let mut events = nodes[0].node.get_and_clear_pending_events();
2781         assert_eq!(events.len(), 2);
2782         match events[0] {
2783                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2784                         assert_eq!(payment_hash, ev_payment_hash);
2785                         assert_eq!(payment_failed_permanently, false);
2786                 },
2787                 _ => panic!("Unexpected event"),
2788         }
2789         match events[1] {
2790                 Event::PendingHTLCsForwardable { .. } => {},
2791                 _ => panic!("Unexpected event"),
2792         }
2793
2794         nodes[0].node.process_pending_htlc_forwards();
2795         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2796         check_added_monitors!(nodes[0], 1);
2797
2798         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2799         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2800
2801         expect_pending_htlcs_forwardable!(nodes[1]);
2802         check_added_monitors!(nodes[1], 1);
2803
2804         let bs_forward_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
2805         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[0]);
2806         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[1]);
2807         commitment_signed_dance!(nodes[2], nodes[1], &bs_forward_update.commitment_signed, false);
2808
2809         expect_pending_htlcs_forwardable!(nodes[2]);
2810         expect_payment_claimable!(nodes[2], payment_hash, payment_secret, amt_msat);
2811 }
2812
2813 #[test]
2814 #[cfg(feature = "std")]
2815 fn test_threaded_payment_retries() {
2816         // In the first version of the in-`ChannelManager` payment retries, retries weren't limited to
2817         // a single thread and would happily let multiple threads run retries at the same time. Because
2818         // retries are done by first calculating the amount we need to retry, then dropping the
2819         // relevant lock, then actually sending, we would happily let multiple threads retry the same
2820         // amount at the same time, overpaying our original HTLC!
2821         let chanmon_cfgs = create_chanmon_cfgs(4);
2822         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
2823         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
2824         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
2825
2826         // There is one mitigating guardrail when retrying payments - we can never over-pay by more
2827         // than 10% of the original value. Thus, we want all our retries to be below that. In order to
2828         // keep things simple, we route one HTLC for 0.1% of the payment over channel 1 and the rest
2829         // out over channel 3+4. This will let us ignore 99% of the payment value and deal with only
2830         // our channel.
2831         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2832         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 10_000_000, 0);
2833         let chan_3_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 10_000_000, 0).0.contents.short_channel_id;
2834         let chan_4_scid = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 10_000_000, 0).0.contents.short_channel_id;
2835
2836         let amt_msat = 100_000_000;
2837         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
2838         #[cfg(feature = "std")]
2839         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2840         #[cfg(not(feature = "std"))]
2841         let payment_expiry_secs = 60 * 60;
2842         let mut invoice_features = InvoiceFeatures::empty();
2843         invoice_features.set_variable_length_onion_required();
2844         invoice_features.set_payment_secret_required();
2845         invoice_features.set_basic_mpp_optional();
2846         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2847                 .with_expiry_time(payment_expiry_secs as u64)
2848                 .with_bolt11_features(invoice_features).unwrap();
2849         let mut route_params = RouteParameters {
2850                 payment_params,
2851                 final_value_msat: amt_msat,
2852         };
2853
2854         let mut route = Route {
2855                 paths: vec![
2856                         Path { hops: vec![RouteHop {
2857                                 pubkey: nodes[1].node.get_our_node_id(),
2858                                 node_features: nodes[1].node.node_features(),
2859                                 short_channel_id: chan_1_scid,
2860                                 channel_features: nodes[1].node.channel_features(),
2861                                 fee_msat: 0,
2862                                 cltv_expiry_delta: 100,
2863                         }, RouteHop {
2864                                 pubkey: nodes[3].node.get_our_node_id(),
2865                                 node_features: nodes[2].node.node_features(),
2866                                 short_channel_id: 42, // Set a random SCID which nodes[1] will fail as unknown
2867                                 channel_features: nodes[2].node.channel_features(),
2868                                 fee_msat: amt_msat / 1000,
2869                                 cltv_expiry_delta: 100,
2870                         }], blinded_tail: None },
2871                         Path { hops: vec![RouteHop {
2872                                 pubkey: nodes[2].node.get_our_node_id(),
2873                                 node_features: nodes[2].node.node_features(),
2874                                 short_channel_id: chan_3_scid,
2875                                 channel_features: nodes[2].node.channel_features(),
2876                                 fee_msat: 100_000,
2877                                 cltv_expiry_delta: 100,
2878                         }, RouteHop {
2879                                 pubkey: nodes[3].node.get_our_node_id(),
2880                                 node_features: nodes[3].node.node_features(),
2881                                 short_channel_id: chan_4_scid,
2882                                 channel_features: nodes[3].node.channel_features(),
2883                                 fee_msat: amt_msat - amt_msat / 1000,
2884                                 cltv_expiry_delta: 100,
2885                         }], blinded_tail: None }
2886                 ],
2887                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2888         };
2889         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2890
2891         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2892                 PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(0xdeadbeef)).unwrap();
2893         check_added_monitors!(nodes[0], 2);
2894         let mut send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2895         assert_eq!(send_msg_events.len(), 2);
2896         send_msg_events.retain(|msg|
2897                 if let MessageSendEvent::UpdateHTLCs { node_id, .. } = msg {
2898                         // Drop the commitment update for nodes[2], we can just let that one sit pending
2899                         // forever.
2900                         *node_id == nodes[1].node.get_our_node_id()
2901                 } else { panic!(); }
2902         );
2903
2904         // from here on out, the retry `RouteParameters` amount will be amt/1000
2905         route_params.final_value_msat /= 1000;
2906         route.paths.pop();
2907
2908         let end_time = Instant::now() + Duration::from_secs(1);
2909         macro_rules! thread_body { () => { {
2910                 // We really want std::thread::scope, but its not stable until 1.63. Until then, we get unsafe.
2911                 let node_ref = NodePtr::from_node(&nodes[0]);
2912                 move || {
2913                         let node_a = unsafe { &*node_ref.0 };
2914                         while Instant::now() < end_time {
2915                                 node_a.node.get_and_clear_pending_events(); // wipe the PendingHTLCsForwardable
2916                                 // Ignore if we have any pending events, just always pretend we just got a
2917                                 // PendingHTLCsForwardable
2918                                 node_a.node.process_pending_htlc_forwards();
2919                         }
2920                 }
2921         } } }
2922         let mut threads = Vec::new();
2923         for _ in 0..16 { threads.push(std::thread::spawn(thread_body!())); }
2924
2925         // Back in the main thread, poll pending messages and make sure that we never have more than
2926         // one HTLC pending at a time. Note that the commitment_signed_dance will fail horribly if
2927         // there are HTLC messages shoved in while its running. This allows us to test that we never
2928         // generate an additional update_add_htlc until we've fully failed the first.
2929         let mut previously_failed_channels = Vec::new();
2930         loop {
2931                 assert_eq!(send_msg_events.len(), 1);
2932                 let send_event = SendEvent::from_event(send_msg_events.pop().unwrap());
2933                 assert_eq!(send_event.msgs.len(), 1);
2934
2935                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
2936                 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
2937
2938                 // Note that we only push one route into `expect_find_route` at a time, because that's all
2939                 // the retries (should) need. If the bug is reintroduced "real" routes may be selected, but
2940                 // we should still ultimately fail for the same reason - because we're trying to send too
2941                 // many HTLCs at once.
2942                 let mut new_route_params = route_params.clone();
2943                 previously_failed_channels.push(route.paths[0].hops[1].short_channel_id);
2944                 new_route_params.payment_params.previously_failed_channels = previously_failed_channels.clone();
2945                 route.paths[0].hops[1].short_channel_id += 1;
2946                 nodes[0].router.expect_find_route(new_route_params, Ok(route.clone()));
2947
2948                 let bs_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2949                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_updates.update_fail_htlcs[0]);
2950                 // The "normal" commitment_signed_dance delivers the final RAA and then calls
2951                 // `check_added_monitors` to ensure only the one RAA-generated monitor update was created.
2952                 // This races with our other threads which may generate an add-HTLCs commitment update via
2953                 // `process_pending_htlc_forwards`. Instead, we defer the monitor update check until after
2954                 // *we've* called `process_pending_htlc_forwards` when its guaranteed to have two updates.
2955                 let last_raa = commitment_signed_dance!(nodes[0], nodes[1], bs_fail_updates.commitment_signed, false, true, false, true);
2956                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &last_raa);
2957
2958                 let cur_time = Instant::now();
2959                 if cur_time > end_time {
2960                         for thread in threads.drain(..) { thread.join().unwrap(); }
2961                 }
2962
2963                 // Make sure we have some events to handle when we go around...
2964                 nodes[0].node.get_and_clear_pending_events(); // wipe the PendingHTLCsForwardable
2965                 nodes[0].node.process_pending_htlc_forwards();
2966                 send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2967                 check_added_monitors!(nodes[0], 2);
2968
2969                 if cur_time > end_time {
2970                         break;
2971                 }
2972         }
2973 }
2974
2975 fn do_no_missing_sent_on_midpoint_reload(persist_manager_with_payment: bool) {
2976         // Test that if we reload in the middle of an HTLC claim commitment signed dance we'll still
2977         // receive the PaymentSent event even if the ChannelManager had no idea about the payment when
2978         // it was last persisted.
2979         let chanmon_cfgs = create_chanmon_cfgs(2);
2980         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2981         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2982         let (persister_a, persister_b, persister_c);
2983         let (chain_monitor_a, chain_monitor_b, chain_monitor_c);
2984         let (nodes_0_deserialized, nodes_0_deserialized_b, nodes_0_deserialized_c);
2985         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2986
2987         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
2988
2989         let mut nodes_0_serialized = Vec::new();
2990         if !persist_manager_with_payment {
2991                 nodes_0_serialized = nodes[0].node.encode();
2992         }
2993
2994         let (our_payment_preimage, our_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 1_000_000);
2995
2996         if persist_manager_with_payment {
2997                 nodes_0_serialized = nodes[0].node.encode();
2998         }
2999
3000         nodes[1].node.claim_funds(our_payment_preimage);
3001         check_added_monitors!(nodes[1], 1);
3002         expect_payment_claimed!(nodes[1], our_payment_hash, 1_000_000);
3003
3004         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3005         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3006         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
3007         check_added_monitors!(nodes[0], 1);
3008
3009         // The ChannelMonitor should always be the latest version, as we're required to persist it
3010         // during the commitment signed handling.
3011         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3012         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister_a, chain_monitor_a, nodes_0_deserialized);
3013
3014         let events = nodes[0].node.get_and_clear_pending_events();
3015         assert_eq!(events.len(), 2);
3016         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[0] {} else { panic!(); }
3017         if let Event::PaymentSent { payment_preimage, .. } = events[1] { assert_eq!(payment_preimage, our_payment_preimage); } else { panic!(); }
3018         // Note that we don't get a PaymentPathSuccessful here as we leave the HTLC pending to avoid
3019         // the double-claim that would otherwise appear at the end of this test.
3020         nodes[0].node.timer_tick_occurred();
3021         let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3022         assert_eq!(as_broadcasted_txn.len(), 1);
3023
3024         // Ensure that, even after some time, if we restart we still include *something* in the current
3025         // `ChannelManager` which prevents a `PaymentFailed` when we restart even if pending resolved
3026         // payments have since been timed out thanks to `IDEMPOTENCY_TIMEOUT_TICKS`.
3027         // A naive implementation of the fix here would wipe the pending payments set, causing a
3028         // failure event when we restart.
3029         for _ in 0..(IDEMPOTENCY_TIMEOUT_TICKS * 2) { nodes[0].node.timer_tick_occurred(); }
3030
3031         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3032         reload_node!(nodes[0], test_default_channel_config(), &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister_b, chain_monitor_b, nodes_0_deserialized_b);
3033         let events = nodes[0].node.get_and_clear_pending_events();
3034         assert!(events.is_empty());
3035
3036         // Ensure that we don't generate any further events even after the channel-closing commitment
3037         // transaction is confirmed on-chain.
3038         confirm_transaction(&nodes[0], &as_broadcasted_txn[0]);
3039         for _ in 0..(IDEMPOTENCY_TIMEOUT_TICKS * 2) { nodes[0].node.timer_tick_occurred(); }
3040
3041         let events = nodes[0].node.get_and_clear_pending_events();
3042         assert!(events.is_empty());
3043
3044         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3045         reload_node!(nodes[0], test_default_channel_config(), &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister_c, chain_monitor_c, nodes_0_deserialized_c);
3046         let events = nodes[0].node.get_and_clear_pending_events();
3047         assert!(events.is_empty());
3048         check_added_monitors(&nodes[0], 1);
3049 }
3050
3051 #[test]
3052 fn no_missing_sent_on_midpoint_reload() {
3053         do_no_missing_sent_on_midpoint_reload(false);
3054         do_no_missing_sent_on_midpoint_reload(true);
3055 }
3056
3057 fn do_claim_from_closed_chan(fail_payment: bool) {
3058         // Previously, LDK would refuse to claim a payment if a channel on which the payment was
3059         // received had been closed between when the HTLC was received and when we went to claim it.
3060         // This makes sense in the payment case - why pay an on-chain fee to claim the HTLC when
3061         // presumably the sender may retry later. Long ago it also reduced total code in the claim
3062         // pipeline.
3063         //
3064         // However, this doesn't make sense if you're trying to do an atomic swap or some other
3065         // protocol that requires atomicity with some other action - if your money got claimed
3066         // elsewhere you need to be able to claim the HTLC in lightning no matter what. Further, this
3067         // is an over-optimization - there should be a very, very low likelihood that a channel closes
3068         // between when we receive the last HTLC for a payment and the user goes to claim the payment.
3069         // Since we now have code to handle this anyway we should allow it.
3070
3071         // Build 4 nodes and send an MPP payment across two paths. By building a route manually set the
3072         // CLTVs on the paths to different value resulting in a different claim deadline.
3073         let chanmon_cfgs = create_chanmon_cfgs(4);
3074         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3075         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3076         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3077
3078         create_announced_chan_between_nodes(&nodes, 0, 1);
3079         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
3080         let chan_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
3081         create_announced_chan_between_nodes(&nodes, 2, 3);
3082
3083         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3]);
3084         let mut route_params = RouteParameters {
3085                 payment_params: PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3086                         .with_bolt11_features(nodes[1].node.invoice_features()).unwrap(),
3087                 final_value_msat: 10_000_000,
3088         };
3089         let mut route = nodes[0].router.find_route(&nodes[0].node.get_our_node_id(), &route_params,
3090                 None, &nodes[0].node.compute_inflight_htlcs()).unwrap();
3091         // Make sure the route is ordered as the B->D path before C->D
3092         route.paths.sort_by(|a, _| if a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
3093                 std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater });
3094
3095         // Note that we add an extra 1 in the send pipeline to compensate for any blocks found while
3096         // the HTLC is being relayed.
3097         route.paths[0].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 8;
3098         route.paths[1].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 12;
3099         let final_cltv = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 8 + 1;
3100
3101         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3102         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3103                 PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(1)).unwrap();
3104         check_added_monitors(&nodes[0], 2);
3105         let mut send_msgs = nodes[0].node.get_and_clear_pending_msg_events();
3106         send_msgs.sort_by(|a, _| {
3107                 let a_node_id =
3108                         if let MessageSendEvent::UpdateHTLCs { node_id, .. } = a { node_id } else { panic!() };
3109                 let node_b_id = nodes[1].node.get_our_node_id();
3110                 if *a_node_id == node_b_id { std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater }
3111         });
3112
3113         assert_eq!(send_msgs.len(), 2);
3114         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 10_000_000,
3115                 payment_hash, Some(payment_secret), send_msgs.remove(0), false, None);
3116         let receive_event = pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 10_000_000,
3117                 payment_hash, Some(payment_secret), send_msgs.remove(0), true, None);
3118
3119         match receive_event.unwrap() {
3120                 Event::PaymentClaimable { claim_deadline, .. } => {
3121                         assert_eq!(claim_deadline.unwrap(), final_cltv - HTLC_FAIL_BACK_BUFFER);
3122                 },
3123                 _ => panic!(),
3124         }
3125
3126         // Ensure that the claim_deadline is correct, with the payment failing at exactly the given
3127         // height.
3128         connect_blocks(&nodes[3], final_cltv - HTLC_FAIL_BACK_BUFFER - nodes[3].best_block_info().1
3129                 - if fail_payment { 0 } else { 2 });
3130         if fail_payment {
3131                 // We fail the HTLC on the A->B->D path first as it expires 4 blocks earlier. We go ahead
3132                 // and expire both immediately, though, by connecting another 4 blocks.
3133                 let reason = HTLCDestination::FailedPayment { payment_hash };
3134                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason.clone()]);
3135                 connect_blocks(&nodes[3], 4);
3136                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason]);
3137                 pass_failed_payment_back(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
3138         } else {
3139                 nodes[1].node.force_close_broadcasting_latest_txn(&chan_bd, &nodes[3].node.get_our_node_id()).unwrap();
3140                 check_closed_event(&nodes[1], 1, ClosureReason::HolderForceClosed, false);
3141                 check_closed_broadcast(&nodes[1], 1, true);
3142                 let bs_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3143                 assert_eq!(bs_tx.len(), 1);
3144
3145                 mine_transaction(&nodes[3], &bs_tx[0]);
3146                 check_added_monitors(&nodes[3], 1);
3147                 check_closed_broadcast(&nodes[3], 1, true);
3148                 check_closed_event(&nodes[3], 1, ClosureReason::CommitmentTxConfirmed, false);
3149
3150                 nodes[3].node.claim_funds(payment_preimage);
3151                 check_added_monitors(&nodes[3], 2);
3152                 expect_payment_claimed!(nodes[3], payment_hash, 10_000_000);
3153
3154                 let ds_tx = nodes[3].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3155                 assert_eq!(ds_tx.len(), 1);
3156                 check_spends!(&ds_tx[0], &bs_tx[0]);
3157
3158                 mine_transactions(&nodes[1], &[&bs_tx[0], &ds_tx[0]]);
3159                 check_added_monitors(&nodes[1], 1);
3160                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[3], Some(1000), false, true);
3161
3162                 let bs_claims = nodes[1].node.get_and_clear_pending_msg_events();
3163                 check_added_monitors(&nodes[1], 1);
3164                 assert_eq!(bs_claims.len(), 1);
3165                 if let MessageSendEvent::UpdateHTLCs { updates, .. } = &bs_claims[0] {
3166                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3167                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
3168                 } else { panic!(); }
3169
3170                 expect_payment_sent!(nodes[0], payment_preimage);
3171
3172                 let ds_claim_msgs = nodes[3].node.get_and_clear_pending_msg_events();
3173                 assert_eq!(ds_claim_msgs.len(), 1);
3174                 let cs_claim_msgs = if let MessageSendEvent::UpdateHTLCs { updates, .. } = &ds_claim_msgs[0] {
3175                         nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3176                         let cs_claim_msgs = nodes[2].node.get_and_clear_pending_msg_events();
3177                         check_added_monitors(&nodes[2], 1);
3178                         commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
3179                         expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
3180                         cs_claim_msgs
3181                 } else { panic!(); };
3182
3183                 assert_eq!(cs_claim_msgs.len(), 1);
3184                 if let MessageSendEvent::UpdateHTLCs { updates, .. } = &cs_claim_msgs[0] {
3185                         nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3186                         commitment_signed_dance!(nodes[0], nodes[2], updates.commitment_signed, false, true);
3187                 } else { panic!(); }
3188
3189                 expect_payment_path_successful!(nodes[0]);
3190         }
3191 }
3192
3193 #[test]
3194 fn claim_from_closed_chan() {
3195         do_claim_from_closed_chan(true);
3196         do_claim_from_closed_chan(false);
3197 }
3198
3199 fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) {
3200         // Check that a payment metadata received on one HTLC that doesn't match the one received on
3201         // another results in the HTLC being rejected.
3202         //
3203         // We first set up a diamond shaped network, allowing us to split a payment into two HTLCs, the
3204         // first of which we'll deliver and the second of which we'll fail and then re-send with
3205         // modified payment metadata, which will in turn result in it being failed by the recipient.
3206         let chanmon_cfgs = create_chanmon_cfgs(4);
3207         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3208         let mut config = test_default_channel_config();
3209         config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 50;
3210         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, Some(config), Some(config), Some(config)]);
3211
3212         let persister;
3213         let new_chain_monitor;
3214         let nodes_0_deserialized;
3215
3216         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3217
3218         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
3219         let chan_id_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
3220         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
3221         let chan_id_cd = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
3222
3223         // Pay more than half of each channel's max, requiring MPP
3224         let amt_msat = 750_000_000;
3225         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3], Some(amt_msat));
3226         let payment_id = PaymentId(payment_hash.0);
3227         let payment_metadata = vec![44, 49, 52, 142];
3228
3229         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3230                 .with_bolt11_features(nodes[1].node.invoice_features()).unwrap();
3231         let mut route_params = RouteParameters {
3232                 payment_params,
3233                 final_value_msat: amt_msat,
3234         };
3235
3236         // Send the MPP payment, delivering the updated commitment state to nodes[1].
3237         nodes[0].node.send_payment(payment_hash, RecipientOnionFields {
3238                         payment_secret: Some(payment_secret), payment_metadata: Some(payment_metadata),
3239                 }, payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
3240         check_added_monitors!(nodes[0], 2);
3241
3242         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
3243         assert_eq!(send_events.len(), 2);
3244         let first_send = SendEvent::from_event(send_events.pop().unwrap());
3245         let second_send = SendEvent::from_event(send_events.pop().unwrap());
3246
3247         let (b_recv_ev, c_recv_ev) = if first_send.node_id == nodes[1].node.get_our_node_id() {
3248                 (&first_send, &second_send)
3249         } else {
3250                 (&second_send, &first_send)
3251         };
3252         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &b_recv_ev.msgs[0]);
3253         commitment_signed_dance!(nodes[1], nodes[0], b_recv_ev.commitment_msg, false, true);
3254
3255         expect_pending_htlcs_forwardable!(nodes[1]);
3256         check_added_monitors(&nodes[1], 1);
3257         let b_forward_ev = SendEvent::from_node(&nodes[1]);
3258         nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &b_forward_ev.msgs[0]);
3259         commitment_signed_dance!(nodes[3], nodes[1], b_forward_ev.commitment_msg, false, true);
3260
3261         expect_pending_htlcs_forwardable!(nodes[3]);
3262
3263         // Before delivering the second MPP HTLC to nodes[2], disconnect nodes[2] and nodes[3], which
3264         // will result in nodes[2] failing the HTLC back.
3265         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
3266         nodes[3].node.peer_disconnected(&nodes[2].node.get_our_node_id());
3267
3268         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &c_recv_ev.msgs[0]);
3269         commitment_signed_dance!(nodes[2], nodes[0], c_recv_ev.commitment_msg, false, true);
3270
3271         let cs_fail = get_htlc_update_msgs(&nodes[2], &nodes[0].node.get_our_node_id());
3272         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail.update_fail_htlcs[0]);
3273         commitment_signed_dance!(nodes[0], nodes[2], cs_fail.commitment_signed, false, true);
3274
3275         let payment_fail_retryable_evs = nodes[0].node.get_and_clear_pending_events();
3276         assert_eq!(payment_fail_retryable_evs.len(), 2);
3277         if let Event::PaymentPathFailed { .. } = payment_fail_retryable_evs[0] {} else { panic!(); }
3278         if let Event::PendingHTLCsForwardable { .. } = payment_fail_retryable_evs[1] {} else { panic!(); }
3279
3280         // Before we allow the HTLC to be retried, optionally change the payment_metadata we have
3281         // stored for our payment.
3282         if do_modify {
3283                 nodes[0].node.test_set_payment_metadata(payment_id, Some(Vec::new()));
3284         }
3285
3286         // Optionally reload nodes[3] to check that the payment_metadata is properly serialized with
3287         // the payment state.
3288         if do_reload {
3289                 let mon_bd = get_monitor!(nodes[3], chan_id_bd).encode();
3290                 let mon_cd = get_monitor!(nodes[3], chan_id_cd).encode();
3291                 reload_node!(nodes[3], config, &nodes[3].node.encode(), &[&mon_bd, &mon_cd],
3292                         persister, new_chain_monitor, nodes_0_deserialized);
3293                 nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
3294                 reconnect_nodes(&nodes[1], &nodes[3], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
3295         }
3296         reconnect_nodes(&nodes[2], &nodes[3], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
3297
3298         // Create a new channel between C and D as A will refuse to retry on the existing one because
3299         // it just failed.
3300         let chan_id_cd_2 = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
3301
3302         // Now retry the failed HTLC.
3303         nodes[0].node.process_pending_htlc_forwards();
3304         check_added_monitors(&nodes[0], 1);
3305         let as_resend = SendEvent::from_node(&nodes[0]);
3306         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_resend.msgs[0]);
3307         commitment_signed_dance!(nodes[2], nodes[0], as_resend.commitment_msg, false, true);
3308
3309         expect_pending_htlcs_forwardable!(nodes[2]);
3310         check_added_monitors(&nodes[2], 1);
3311         let cs_forward = SendEvent::from_node(&nodes[2]);
3312         nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &cs_forward.msgs[0]);
3313         commitment_signed_dance!(nodes[3], nodes[2], cs_forward.commitment_msg, false, true);
3314
3315         // Finally, check that nodes[3] does the correct thing - either accepting the payment or, if
3316         // the payment metadata was modified, failing only the one modified HTLC and retaining the
3317         // other.
3318         if do_modify {
3319                 expect_pending_htlcs_forwardable_ignore!(nodes[3]);
3320                 nodes[3].node.process_pending_htlc_forwards();
3321                 expect_pending_htlcs_forwardable_conditions(nodes[3].node.get_and_clear_pending_events(),
3322                         &[HTLCDestination::FailedPayment {payment_hash}]);
3323                 nodes[3].node.process_pending_htlc_forwards();
3324
3325                 check_added_monitors(&nodes[3], 1);
3326                 let ds_fail = get_htlc_update_msgs(&nodes[3], &nodes[2].node.get_our_node_id());
3327
3328                 nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &ds_fail.update_fail_htlcs[0]);
3329                 commitment_signed_dance!(nodes[2], nodes[3], ds_fail.commitment_signed, false, true);
3330                 expect_pending_htlcs_forwardable_conditions(nodes[2].node.get_and_clear_pending_events(),
3331                         &[HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_id_cd_2 }]);
3332         } else {
3333                 expect_pending_htlcs_forwardable!(nodes[3]);
3334                 expect_payment_claimable!(nodes[3], payment_hash, payment_secret, amt_msat);
3335                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
3336         }
3337 }
3338
3339 #[test]
3340 fn test_payment_metadata_consistency() {
3341         do_test_payment_metadata_consistency(true, true);
3342         do_test_payment_metadata_consistency(true, false);
3343         do_test_payment_metadata_consistency(false, true);
3344         do_test_payment_metadata_consistency(false, false);
3345 }