_ Git - rust-lightning/blob - lightning/src/ln/payment_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Tests that test the payment retry logic in ChannelManager, including various edge-cases around
  11 //! serialization ordering between ChannelManager/ChannelMonitors and ensuring we can still retry
  12 //! payments thereafter.
  13
  14 use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Listen, Watch};
  15 use crate::chain::channelmonitor::{ANTI_REORG_DELAY, HTLC_FAIL_BACK_BUFFER, LATENCY_GRACE_PERIOD_BLOCKS};
  16 use crate::sign::EntropySource;
  17 use crate::chain::transaction::OutPoint;
  18 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentFailureReason};
  19 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
  20 use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, ChannelManager, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS, RecentPaymentDetails, RecipientOnionFields, HTLCForwardInfo, PendingHTLCRouting, PendingAddHTLCInfo};
  21 use crate::ln::features::InvoiceFeatures;
  22 use crate::ln::{msgs, PaymentSecret, PaymentPreimage};
  23 use crate::ln::msgs::ChannelMessageHandler;
  24 use crate::ln::outbound_payment::Retry;
  25 use crate::routing::gossip::{EffectiveCapacity, RoutingFees};
  26 use crate::routing::router::{get_route, Path, PaymentParameters, Route, Router, RouteHint, RouteHintHop, RouteHop, RouteParameters, find_route};
  27 use crate::routing::scoring::ChannelUsage;
  28 use crate::util::test_utils;
  29 use crate::util::errors::APIError;
  30 use crate::util::ser::Writeable;
  31 use crate::util::string::UntrustedString;
  32
  33 use bitcoin::network::constants::Network;
  34
  35 use crate::prelude::*;
  36
  37 use crate::ln::functional_test_utils::*;
  38 use crate::routing::gossip::NodeId;
  39 #[cfg(feature = "std")]
  40 use {
  41         crate::util::time::tests::SinceEpoch,
  42         std::time::{SystemTime, Instant, Duration}
  43 };
  44
  45 #[test]
  46 fn mpp_failure() {
  47         let chanmon_cfgs = create_chanmon_cfgs(4);
  48         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
  49         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
  50         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
  51
  52         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
  53         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
  54         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
  55         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3).0.contents.short_channel_id;
  56
  57         let (mut route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
  58         let path = route.paths[0].clone();
  59         route.paths.push(path);
  60         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
  61         route.paths[0].hops[0].short_channel_id = chan_1_id;
  62         route.paths[0].hops[1].short_channel_id = chan_3_id;
  63         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
  64         route.paths[1].hops[0].short_channel_id = chan_2_id;
  65         route.paths[1].hops[1].short_channel_id = chan_4_id;
  66         send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
  67         fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
  68 }
  69
  70 #[test]
  71 fn mpp_retry() {
  72         let chanmon_cfgs = create_chanmon_cfgs(4);
  73         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
  74         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
  75         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
  76
  77         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
  78         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
  79         let (chan_3_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
  80         let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes(&nodes, 3, 2);
  81         // Rebalance
  82         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
  83
  84         let amt_msat = 1_000_000;
  85         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], amt_msat);
  86         let path = route.paths[0].clone();
  87         route.paths.push(path);
  88         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
  89         route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
  90         route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
  91         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
  92         route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
  93         route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
  94
  95         // Initiate the MPP payment.
  96         let payment_id = PaymentId(payment_hash.0);
  97         let mut route_params = RouteParameters {
  98                 payment_params: route.payment_params.clone().unwrap(),
  99                 final_value_msat: amt_msat,
 100         };
 101
 102         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
 103         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 104                 payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
 105         check_added_monitors!(nodes[0], 2); // one monitor per path
 106         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 107         assert_eq!(events.len(), 2);
 108
 109         // Pass half of the payment along the success path.
 110         let success_path_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 111         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), success_path_msgs, false, None);
 112
 113         // Add the HTLC along the first hop.
 114         let fail_path_msgs_1 = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 115         let (update_add, commitment_signed) = match fail_path_msgs_1 {
 116                 MessageSendEvent::UpdateHTLCs { node_id: _, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 117                         assert_eq!(update_add_htlcs.len(), 1);
 118                         assert!(update_fail_htlcs.is_empty());
 119                         assert!(update_fulfill_htlcs.is_empty());
 120                         assert!(update_fail_malformed_htlcs.is_empty());
 121                         assert!(update_fee.is_none());
 122                         (update_add_htlcs[0].clone(), commitment_signed.clone())
 123                 },
 124                 _ => panic!("Unexpected event"),
 125         };
 126         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
 127         commitment_signed_dance!(nodes[2], nodes[0], commitment_signed, false);
 128
 129         // Attempt to forward the payment and complete the 2nd path's failure.
 130         expect_pending_htlcs_forwardable!(&nodes[2]);
 131         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id }]);
 132         let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 133         assert!(htlc_updates.update_add_htlcs.is_empty());
 134         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 135         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 136         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 137         check_added_monitors!(nodes[2], 1);
 138         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 139         commitment_signed_dance!(nodes[0], nodes[2], htlc_updates.commitment_signed, false);
 140         let mut events = nodes[0].node.get_and_clear_pending_events();
 141         match events[1] {
 142                 Event::PendingHTLCsForwardable { .. } => {},
 143                 _ => panic!("Unexpected event")
 144         }
 145         events.remove(1);
 146         expect_payment_failed_conditions_event(events, payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 147
 148         // Rebalance the channel so the second half of the payment can succeed.
 149         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 150
 151         // Retry the second half of the payment and make sure it succeeds.
 152         route.paths.remove(0);
 153         route_params.final_value_msat = 1_000_000;
 154         route_params.payment_params.previously_failed_channels.push(chan_4_update.contents.short_channel_id);
 155         nodes[0].router.expect_find_route(route_params, Ok(route));
 156         nodes[0].node.process_pending_htlc_forwards();
 157         check_added_monitors!(nodes[0], 1);
 158         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 159         assert_eq!(events.len(), 1);
 160         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 161         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 162 }
 163
 164 fn do_mpp_receive_timeout(send_partial_mpp: bool) {
 165         let chanmon_cfgs = create_chanmon_cfgs(4);
 166         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 167         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 168         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 169
 170         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 171         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 172         let (chan_3_update, _, chan_3_id, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 173         let (chan_4_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 174
 175         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 100_000);
 176         let path = route.paths[0].clone();
 177         route.paths.push(path);
 178         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
 179         route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
 180         route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
 181         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
 182         route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
 183         route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
 184
 185         // Initiate the MPP payment.
 186         nodes[0].node.send_payment_with_route(&route, payment_hash,
 187                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 188         check_added_monitors!(nodes[0], 2); // one monitor per path
 189         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 190         assert_eq!(events.len(), 2);
 191
 192         // Pass half of the payment along the first path.
 193         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 194         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_1_msgs, false, None);
 195
 196         if send_partial_mpp {
 197                 // Time out the partial MPP
 198                 for _ in 0..MPP_TIMEOUT_TICKS {
 199                         nodes[3].node.timer_tick_occurred();
 200                 }
 201
 202                 // Failed HTLC from node 3 -> 1
 203                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 204                 let htlc_fail_updates_3_1 = get_htlc_update_msgs!(nodes[3], nodes[1].node.get_our_node_id());
 205                 assert_eq!(htlc_fail_updates_3_1.update_fail_htlcs.len(), 1);
 206                 nodes[1].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &htlc_fail_updates_3_1.update_fail_htlcs[0]);
 207                 check_added_monitors!(nodes[3], 1);
 208                 commitment_signed_dance!(nodes[1], nodes[3], htlc_fail_updates_3_1.commitment_signed, false);
 209
 210                 // Failed HTLC from node 1 -> 0
 211                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_3_id }]);
 212                 let htlc_fail_updates_1_0 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 213                 assert_eq!(htlc_fail_updates_1_0.update_fail_htlcs.len(), 1);
 214                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates_1_0.update_fail_htlcs[0]);
 215                 check_added_monitors!(nodes[1], 1);
 216                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates_1_0.commitment_signed, false);
 217
 218                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain().expected_htlc_error_data(23, &[][..]));
 219         } else {
 220                 // Pass half of the payment along the second path.
 221                 let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 222                 pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_2_msgs, true, None);
 223
 224                 // Even after MPP_TIMEOUT_TICKS we should not timeout the MPP if we have all the parts
 225                 for _ in 0..MPP_TIMEOUT_TICKS {
 226                         nodes[3].node.timer_tick_occurred();
 227                 }
 228
 229                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 230         }
 231 }
 232
 233 #[test]
 234 fn mpp_receive_timeout() {
 235         do_mpp_receive_timeout(true);
 236         do_mpp_receive_timeout(false);
 237 }
 238
 239 #[test]
 240 fn test_mpp_keysend() {
 241         let mut mpp_keysend_config = test_default_channel_config();
 242         mpp_keysend_config.accept_mpp_keysend = true;
 243         let chanmon_cfgs = create_chanmon_cfgs(4);
 244         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 245         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, Some(mpp_keysend_config)]);
 246         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 247
 248         create_announced_chan_between_nodes(&nodes, 0, 1);
 249         create_announced_chan_between_nodes(&nodes, 0, 2);
 250         create_announced_chan_between_nodes(&nodes, 1, 3);
 251         create_announced_chan_between_nodes(&nodes, 2, 3);
 252         let network_graph = nodes[0].network_graph.clone();
 253
 254         let payer_pubkey = nodes[0].node.get_our_node_id();
 255         let payee_pubkey = nodes[3].node.get_our_node_id();
 256         let recv_value = 15_000_000;
 257         let route_params = RouteParameters {
 258                 payment_params: PaymentParameters::for_keysend(payee_pubkey, 40, true),
 259                 final_value_msat: recv_value,
 260         };
 261         let scorer = test_utils::TestScorer::new();
 262         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
 263         let route = find_route(&payer_pubkey, &route_params, &network_graph, None, nodes[0].logger,
 264                 &scorer, &(), &random_seed_bytes).unwrap();
 265
 266         let payment_preimage = PaymentPreimage([42; 32]);
 267         let payment_secret = PaymentSecret(payment_preimage.0);
 268         let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage),
 269                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_preimage.0)).unwrap();
 270         check_added_monitors!(nodes[0], 2);
 271
 272         let expected_route: &[&[&Node]] = &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]];
 273         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 274         assert_eq!(events.len(), 2);
 275
 276         let ev = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 277         pass_along_path(&nodes[0], expected_route[0], recv_value, payment_hash.clone(),
 278                 Some(payment_secret), ev.clone(), false, Some(payment_preimage));
 279
 280         let ev = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 281         pass_along_path(&nodes[0], expected_route[1], recv_value, payment_hash.clone(),
 282                 Some(payment_secret), ev.clone(), true, Some(payment_preimage));
 283         claim_payment_along_route(&nodes[0], expected_route, false, payment_preimage);
 284 }
 285
 286 #[test]
 287 fn test_reject_mpp_keysend_htlc() {
 288         // This test enforces that we reject MPP keysend HTLCs if our config states we don't support
 289         // MPP keysend. When receiving a payment, if we don't support MPP keysend we'll reject the
 290         // payment if it's keysend and has a payment secret, never reaching our payment validation
 291         // logic. To check that we enforce rejecting MPP keysends in our payment logic, here we send
 292         // keysend payments without payment secrets, then modify them by adding payment secrets in the
 293         // final node in between receiving the HTLCs and actually processing them.
 294         let mut reject_mpp_keysend_cfg = test_default_channel_config();
 295         reject_mpp_keysend_cfg.accept_mpp_keysend = false;
 296
 297         let chanmon_cfgs = create_chanmon_cfgs(4);
 298         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 299         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, Some(reject_mpp_keysend_cfg)]);
 300         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 301         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
 302         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
 303         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
 304         let (update_a, _, chan_4_channel_id, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 305         let chan_4_id = update_a.contents.short_channel_id;
 306         let amount = 40_000;
 307         let (mut route, payment_hash, payment_preimage, _) = get_route_and_payment_hash!(nodes[0], nodes[3], amount);
 308
 309         // Pay along nodes[1]
 310         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
 311         route.paths[0].hops[0].short_channel_id = chan_1_id;
 312         route.paths[0].hops[1].short_channel_id = chan_3_id;
 313
 314         let payment_id_0 = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 315         nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), RecipientOnionFields::spontaneous_empty(), payment_id_0).unwrap();
 316         check_added_monitors!(nodes[0], 1);
 317
 318         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 319         let update_add_0 = update_0.update_add_htlcs[0].clone();
 320         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_0);
 321         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
 322         expect_pending_htlcs_forwardable!(nodes[1]);
 323
 324         check_added_monitors!(&nodes[1], 1);
 325         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[3].node.get_our_node_id());
 326         let update_add_1 = update_1.update_add_htlcs[0].clone();
 327         nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &update_add_1);
 328         commitment_signed_dance!(nodes[3], nodes[1], update_1.commitment_signed, false, true);
 329
 330         assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
 331         for (_, pending_forwards) in nodes[3].node.forward_htlcs.lock().unwrap().iter_mut() {
 332                 for f in pending_forwards.iter_mut() {
 333                         match f {
 334                                 &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) => {
 335                                         match forward_info.routing {
 336                                                 PendingHTLCRouting::ReceiveKeysend { ref mut payment_data, .. } => {
 337                                                         *payment_data = Some(msgs::FinalOnionHopData {
 338                                                                 payment_secret: PaymentSecret([42; 32]),
 339                                                                 total_msat: amount * 2,
 340                                                         });
 341                                                 },
 342                                                 _ => panic!("Expected PendingHTLCRouting::ReceiveKeysend"),
 343                                         }
 344                                 },
 345                                 _ => {},
 346                         }
 347                 }
 348         }
 349         expect_pending_htlcs_forwardable!(nodes[3]);
 350
 351         // Pay along nodes[2]
 352         route.paths[0].hops[0].pubkey = nodes[2].node.get_our_node_id();
 353         route.paths[0].hops[0].short_channel_id = chan_2_id;
 354         route.paths[0].hops[1].short_channel_id = chan_4_id;
 355
 356         let payment_id_1 = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 357         nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), RecipientOnionFields::spontaneous_empty(), payment_id_1).unwrap();
 358         check_added_monitors!(nodes[0], 1);
 359
 360         let update_2 = get_htlc_update_msgs!(nodes[0], nodes[2].node.get_our_node_id());
 361         let update_add_2 = update_2.update_add_htlcs[0].clone();
 362         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_2);
 363         commitment_signed_dance!(nodes[2], nodes[0], &update_2.commitment_signed, false, true);
 364         expect_pending_htlcs_forwardable!(nodes[2]);
 365
 366         check_added_monitors!(&nodes[2], 1);
 367         let update_3 = get_htlc_update_msgs!(nodes[2], nodes[3].node.get_our_node_id());
 368         let update_add_3 = update_3.update_add_htlcs[0].clone();
 369         nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &update_add_3);
 370         commitment_signed_dance!(nodes[3], nodes[2], update_3.commitment_signed, false, true);
 371
 372         assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
 373         for (_, pending_forwards) in nodes[3].node.forward_htlcs.lock().unwrap().iter_mut() {
 374                 for f in pending_forwards.iter_mut() {
 375                         match f {
 376                                 &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) => {
 377                                         match forward_info.routing {
 378                                                 PendingHTLCRouting::ReceiveKeysend { ref mut payment_data, .. } => {
 379                                                         *payment_data = Some(msgs::FinalOnionHopData {
 380                                                                 payment_secret: PaymentSecret([42; 32]),
 381                                                                 total_msat: amount * 2,
 382                                                         });
 383                                                 },
 384                                                 _ => panic!("Expected PendingHTLCRouting::ReceiveKeysend"),
 385                                         }
 386                                 },
 387                                 _ => {},
 388                         }
 389                 }
 390         }
 391         expect_pending_htlcs_forwardable!(nodes[3]);
 392         check_added_monitors!(nodes[3], 1);
 393
 394         // Fail back along nodes[2]
 395         let update_fail_0 = get_htlc_update_msgs!(&nodes[3], &nodes[2].node.get_our_node_id());
 396         nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &update_fail_0.update_fail_htlcs[0]);
 397         commitment_signed_dance!(nodes[2], nodes[3], update_fail_0.commitment_signed, false);
 398         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_channel_id }]);
 399         check_added_monitors!(nodes[2], 1);
 400
 401         let update_fail_1 = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 402         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_1.update_fail_htlcs[0]);
 403         commitment_signed_dance!(nodes[0], nodes[2], update_fail_1.commitment_signed, false);
 404
 405         expect_payment_failed_conditions(&nodes[0], payment_hash, true, PaymentFailedConditions::new());
 406         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 407 }
 408
 409
 410 #[test]
 411 fn no_pending_leak_on_initial_send_failure() {
 412         // In an earlier version of our payment tracking, we'd have a retry entry even when the initial
 413         // HTLC for payment failed to send due to local channel errors (e.g. peer disconnected). In this
 414         // case, the user wouldn't have a PaymentId to retry the payment with, but we'd think we have a
 415         // pending payment forever and never time it out.
 416         // Here we test exactly that - retrying a payment when a peer was disconnected on the first
 417         // try, and then check that no pending payment is being tracked.
 418         let chanmon_cfgs = create_chanmon_cfgs(2);
 419         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 420         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 421         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 422
 423         create_announced_chan_between_nodes(&nodes, 0, 1);
 424
 425         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 426
 427         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 428         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 429
 430         unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash,
 431                         RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
 432                 ), true, APIError::ChannelUnavailable { ref err },
 433                 assert_eq!(err, "Peer for first hop currently disconnected"));
 434
 435         assert!(!nodes[0].node.has_pending_payments());
 436 }
 437
 438 fn do_retry_with_no_persist(confirm_before_reload: bool) {
 439         // If we send a pending payment and `send_payment` returns success, we should always either
 440         // return a payment failure event or a payment success event, and on failure the payment should
 441         // be retryable.
 442         //
 443         // In order to do so when the ChannelManager isn't immediately persisted (which is normal - its
 444         // always persisted asynchronously), the ChannelManager has to reload some payment data from
 445         // ChannelMonitor(s) in some cases. This tests that reloading.
 446         //
 447         // `confirm_before_reload` confirms the channel-closing commitment transaction on-chain prior
 448         // to reloading the ChannelManager, increasing test coverage in ChannelMonitor HTLC tracking
 449         // which has separate codepaths for "commitment transaction already confirmed" and not.
 450         let chanmon_cfgs = create_chanmon_cfgs(3);
 451         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 452         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 453         let persister: test_utils::TestPersister;
 454         let new_chain_monitor: test_utils::TestChainMonitor;
 455         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 456         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 457
 458         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 459         let (_, _, chan_id_2, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
 460
 461         // Serialize the ChannelManager prior to sending payments
 462         let nodes_0_serialized = nodes[0].node.encode();
 463
 464         // Send two payments - one which will get to nodes[2] and will be claimed, one which we'll time
 465         // out and retry.
 466         let amt_msat = 1_000_000;
 467         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
 468         let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000);
 469         let route_params = RouteParameters {
 470                 payment_params: route.payment_params.clone().unwrap(),
 471                 final_value_msat: amt_msat,
 472         };
 473         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 474                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
 475         check_added_monitors!(nodes[0], 1);
 476
 477         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 478         assert_eq!(events.len(), 1);
 479         let payment_event = SendEvent::from_event(events.pop().unwrap());
 480         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 481
 482         // We relay the payment to nodes[1] while its disconnected from nodes[2], causing the payment
 483         // to be returned immediately to nodes[0], without having nodes[2] fail the inbound payment
 484         // which would prevent retry.
 485         nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id());
 486         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 487
 488         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 489         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true);
 490         // nodes[1] now immediately fails the HTLC as the next-hop channel is disconnected
 491         let _ = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 492
 493         reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 494
 495         let as_commitment_tx = get_local_commitment_txn!(nodes[0], chan_id)[0].clone();
 496         if confirm_before_reload {
 497                 mine_transaction(&nodes[0], &as_commitment_tx);
 498                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 499         }
 500
 501         // The ChannelMonitor should always be the latest version, as we're required to persist it
 502         // during the `commitment_signed_dance!()`.
 503         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 504         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 505
 506         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 507         // force-close the channel.
 508         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 509         assert!(nodes[0].node.list_channels().is_empty());
 510         assert!(nodes[0].node.has_pending_payments());
 511         nodes[0].node.timer_tick_occurred();
 512         if !confirm_before_reload {
 513                 let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 514                 assert_eq!(as_broadcasted_txn.len(), 1);
 515                 assert_eq!(as_broadcasted_txn[0].txid(), as_commitment_tx.txid());
 516         } else {
 517                 assert!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
 518         }
 519         check_added_monitors!(nodes[0], 1);
 520
 521         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 522         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 523                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 524         }, true).unwrap();
 525         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 526
 527         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 528         // error, as the channel has hit the chain.
 529         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 530                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 531         }, false).unwrap();
 532         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 533         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 534         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 535         assert_eq!(as_err.len(), 1);
 536         match as_err[0] {
 537                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 538                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 539                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 540                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) });
 541                         check_added_monitors!(nodes[1], 1);
 542                         assert_eq!(nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 543                 },
 544                 _ => panic!("Unexpected event"),
 545         }
 546         check_closed_broadcast!(nodes[1], false);
 547
 548         // Now claim the first payment, which should allow nodes[1] to claim the payment on-chain when
 549         // we close in a moment.
 550         nodes[2].node.claim_funds(payment_preimage_1);
 551         check_added_monitors!(nodes[2], 1);
 552         expect_payment_claimed!(nodes[2], payment_hash_1, 1_000_000);
 553
 554         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 555         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 556         check_added_monitors!(nodes[1], 1);
 557         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 558         expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], None, false, false);
 559
 560         if confirm_before_reload {
 561                 let best_block = nodes[0].blocks.lock().unwrap().last().unwrap().clone();
 562                 nodes[0].node.best_block_updated(&best_block.0.header, best_block.1);
 563         }
 564
 565         // Create a new channel on which to retry the payment before we fail the payment via the
 566         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 567         // connecting several blocks while creating the channel (implying time has passed).
 568         create_announced_chan_between_nodes(&nodes, 0, 1);
 569         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 570
 571         mine_transaction(&nodes[1], &as_commitment_tx);
 572         let bs_htlc_claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 573         assert_eq!(bs_htlc_claim_txn.len(), 1);
 574         check_spends!(bs_htlc_claim_txn[0], as_commitment_tx);
 575
 576         if !confirm_before_reload {
 577                 mine_transaction(&nodes[0], &as_commitment_tx);
 578         }
 579         mine_transaction(&nodes[0], &bs_htlc_claim_txn[0]);
 580         expect_payment_sent!(nodes[0], payment_preimage_1);
 581         connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
 582         let (first_htlc_timeout_tx, second_htlc_timeout_tx) = {
 583                 let mut txn = nodes[0].tx_broadcaster.unique_txn_broadcast();
 584                 assert_eq!(txn.len(), 2);
 585                 (txn.remove(0), txn.remove(0))
 586         };
 587         check_spends!(first_htlc_timeout_tx, as_commitment_tx);
 588         check_spends!(second_htlc_timeout_tx, as_commitment_tx);
 589         if first_htlc_timeout_tx.input[0].previous_output == bs_htlc_claim_txn[0].input[0].previous_output {
 590                 confirm_transaction(&nodes[0], &second_htlc_timeout_tx);
 591         } else {
 592                 confirm_transaction(&nodes[0], &first_htlc_timeout_tx);
 593         }
 594         nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 595         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new());
 596
 597         // Finally, retry the payment (which was reloaded from the ChannelMonitor when nodes[0] was
 598         // reloaded) via a route over the new channel, which work without issue and eventually be
 599         // received and claimed at the recipient just like any other payment.
 600         let (mut new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 601
 602         // Update the fee on the middle hop to ensure PaymentSent events have the correct (retried) fee
 603         // and not the original fee. We also update node[1]'s relevant config as
 604         // do_claim_payment_along_route expects us to never overpay.
 605         {
 606                 let per_peer_state = nodes[1].node.per_peer_state.read().unwrap();
 607                 let mut peer_state = per_peer_state.get(&nodes[2].node.get_our_node_id())
 608                         .unwrap().lock().unwrap();
 609                 let mut channel = peer_state.channel_by_id.get_mut(&chan_id_2).unwrap();
 610                 let mut new_config = channel.context.config();
 611                 new_config.forwarding_fee_base_msat += 100_000;
 612                 channel.context.update_config(&new_config);
 613                 new_route.paths[0].hops[0].fee_msat += 100_000;
 614         }
 615
 616         // Force expiration of the channel's previous config.
 617         for _ in 0..EXPIRE_PREV_CONFIG_TICKS {
 618                 nodes[1].node.timer_tick_occurred();
 619         }
 620
 621         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash, // Shouldn't be allowed to retry a fulfilled payment
 622                 RecipientOnionFields::secret_only(payment_secret), payment_id_1).is_err());
 623         nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 624                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 625         check_added_monitors!(nodes[0], 1);
 626         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 627         assert_eq!(events.len(), 1);
 628         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 629         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
 630         expect_payment_sent!(nodes[0], payment_preimage, Some(new_route.paths[0].hops[0].fee_msat));
 631 }
 632
 633 #[test]
 634 fn retry_with_no_persist() {
 635         do_retry_with_no_persist(true);
 636         do_retry_with_no_persist(false);
 637 }
 638
 639 fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 640         // Test that an off-chain completed payment is not retryable on restart. This was previously
 641         // broken for dust payments, but we test for both dust and non-dust payments.
 642         //
 643         // `use_dust` switches to using a dust HTLC, which results in the HTLC not having an on-chain
 644         // output at all.
 645         let chanmon_cfgs = create_chanmon_cfgs(3);
 646         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 647
 648         let mut manually_accept_config = test_default_channel_config();
 649         manually_accept_config.manually_accept_inbound_channels = true;
 650
 651         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(manually_accept_config), None]);
 652
 653         let first_persister: test_utils::TestPersister;
 654         let first_new_chain_monitor: test_utils::TestChainMonitor;
 655         let first_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 656         let second_persister: test_utils::TestPersister;
 657         let second_new_chain_monitor: test_utils::TestChainMonitor;
 658         let second_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 659         let third_persister: test_utils::TestPersister;
 660         let third_new_chain_monitor: test_utils::TestChainMonitor;
 661         let third_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 662
 663         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 664
 665         // Because we set nodes[1] to manually accept channels, just open a 0-conf channel.
 666         let (funding_tx, chan_id) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 667         confirm_transaction(&nodes[0], &funding_tx);
 668         confirm_transaction(&nodes[1], &funding_tx);
 669         // Ignore the announcement_signatures messages
 670         nodes[0].node.get_and_clear_pending_msg_events();
 671         nodes[1].node.get_and_clear_pending_msg_events();
 672         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 673
 674         // Serialize the ChannelManager prior to sending payments
 675         let mut nodes_0_serialized = nodes[0].node.encode();
 676
 677         let route = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 }).0;
 678         let (payment_preimage, payment_hash, payment_secret, payment_id) = send_along_route(&nodes[0], route, &[&nodes[1], &nodes[2]], if use_dust { 1_000 } else { 1_000_000 });
 679
 680         // The ChannelMonitor should always be the latest version, as we're required to persist it
 681         // during the `commitment_signed_dance!()`.
 682         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 683
 684         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized], first_persister, first_new_chain_monitor, first_nodes_0_deserialized);
 685         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 686
 687         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 688         // force-close the channel.
 689         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 690         nodes[0].node.timer_tick_occurred();
 691         assert!(nodes[0].node.list_channels().is_empty());
 692         assert!(nodes[0].node.has_pending_payments());
 693         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 694         check_added_monitors!(nodes[0], 1);
 695
 696         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 697                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 698         }, true).unwrap();
 699         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 700
 701         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 702         // error, as the channel has hit the chain.
 703         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 704                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 705         }, false).unwrap();
 706         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 707         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 708         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 709         assert_eq!(as_err.len(), 1);
 710         let bs_commitment_tx;
 711         match as_err[0] {
 712                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 713                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 714                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 715                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) });
 716                         check_added_monitors!(nodes[1], 1);
 717                         bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 718                 },
 719                 _ => panic!("Unexpected event"),
 720         }
 721         check_closed_broadcast!(nodes[1], false);
 722
 723         // Now fail back the payment from nodes[2] to nodes[1]. This doesn't really matter as the
 724         // previous hop channel is already on-chain, but it makes nodes[2] willing to see additional
 725         // incoming HTLCs with the same payment hash later.
 726         nodes[2].node.fail_htlc_backwards(&payment_hash);
 727         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
 728         check_added_monitors!(nodes[2], 1);
 729
 730         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 731         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fail_htlcs[0]);
 732         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 733         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1],
 734                 [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 735
 736         // Connect the HTLC-Timeout transaction, timing out the HTLC on both nodes (but not confirming
 737         // the HTLC-Timeout transaction beyond 1 conf). For dust HTLCs, the HTLC is considered resolved
 738         // after the commitment transaction, so always connect the commitment transaction.
 739         mine_transaction(&nodes[0], &bs_commitment_tx[0]);
 740         mine_transaction(&nodes[1], &bs_commitment_tx[0]);
 741         if !use_dust {
 742                 connect_blocks(&nodes[0], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
 743                 connect_blocks(&nodes[1], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
 744                 let as_htlc_timeout = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 745                 check_spends!(as_htlc_timeout[0], bs_commitment_tx[0]);
 746                 assert_eq!(as_htlc_timeout.len(), 1);
 747
 748                 mine_transaction(&nodes[0], &as_htlc_timeout[0]);
 749                 // nodes[0] may rebroadcast (or RBF-bump) its HTLC-Timeout, so wipe the announced set.
 750                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 751                 mine_transaction(&nodes[1], &as_htlc_timeout[0]);
 752         }
 753
 754         // Create a new channel on which to retry the payment before we fail the payment via the
 755         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 756         // connecting several blocks while creating the channel (implying time has passed).
 757         // We do this with a zero-conf channel to avoid connecting blocks as a side-effect.
 758         let (_, chan_id_3) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 759         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 760
 761         // If we attempt to retry prior to the HTLC-Timeout (or commitment transaction, for dust HTLCs)
 762         // confirming, we will fail as it's considered still-pending...
 763         let (new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 });
 764         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 765                 Err(PaymentSendFailure::DuplicatePayment) => {},
 766                 _ => panic!("Unexpected error")
 767         }
 768         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 769
 770         // After ANTI_REORG_DELAY confirmations, the HTLC should be failed and we can try the payment
 771         // again. We serialize the node first as we'll then test retrying the HTLC after a restart
 772         // (which should also still work).
 773         connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
 774         connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 775         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new());
 776
 777         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 778         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 779         nodes_0_serialized = nodes[0].node.encode();
 780
 781         // After the payment failed, we're free to send it again.
 782         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 783                 RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
 784         assert!(!nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 785
 786         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], second_persister, second_new_chain_monitor, second_nodes_0_deserialized);
 787         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 788
 789         nodes[0].node.test_process_background_events();
 790         check_added_monitors(&nodes[0], 1);
 791
 792         reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 793
 794         // Now resend the payment, delivering the HTLC and actually claiming it this time. This ensures
 795         // the payment is not (spuriously) listed as still pending.
 796         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 797                 RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
 798         check_added_monitors!(nodes[0], 1);
 799         pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], if use_dust { 1_000 } else { 1_000_000 }, payment_hash, payment_secret);
 800         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
 801
 802         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 803                 Err(PaymentSendFailure::DuplicatePayment) => {},
 804                 _ => panic!("Unexpected error")
 805         }
 806         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 807
 808         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 809         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 810         nodes_0_serialized = nodes[0].node.encode();
 811
 812         // Check that after reload we can send the payment again (though we shouldn't, since it was
 813         // claimed previously).
 814         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], third_persister, third_new_chain_monitor, third_nodes_0_deserialized);
 815         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 816
 817         nodes[0].node.test_process_background_events();
 818         check_added_monitors(&nodes[0], 1);
 819
 820         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 821
 822         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 823                 Err(PaymentSendFailure::DuplicatePayment) => {},
 824                 _ => panic!("Unexpected error")
 825         }
 826         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 827 }
 828
 829 #[test]
 830 fn test_completed_payment_not_retryable_on_reload() {
 831         do_test_completed_payment_not_retryable_on_reload(true);
 832         do_test_completed_payment_not_retryable_on_reload(false);
 833 }
 834
 835
 836 fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, confirm_commitment_tx: bool, payment_timeout: bool) {
 837         // When a Channel is closed, any outbound HTLCs which were relayed through it are simply
 838         // dropped when the Channel is. From there, the ChannelManager relies on the ChannelMonitor
 839         // having a copy of the relevant fail-/claim-back data and processes the HTLC fail/claim when
 840         // the ChannelMonitor tells it to.
 841         //
 842         // If, due to an on-chain event, an HTLC is failed/claimed, we should avoid providing the
 843         // ChannelManager the HTLC event until after the monitor is re-persisted. This should prevent a
 844         // duplicate HTLC fail/claim (e.g. via a PaymentPathFailed event).
 845         let chanmon_cfgs = create_chanmon_cfgs(2);
 846         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 847         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 848         let persister: test_utils::TestPersister;
 849         let new_chain_monitor: test_utils::TestChainMonitor;
 850         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 851         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 852
 853         let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1);
 854
 855         // Route a payment, but force-close the channel before the HTLC fulfill message arrives at
 856         // nodes[0].
 857         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 10_000_000);
 858         nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap();
 859         check_closed_broadcast!(nodes[0], true);
 860         check_added_monitors!(nodes[0], 1);
 861         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 862
 863         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 864         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 865
 866         // Connect blocks until the CLTV timeout is up so that we get an HTLC-Timeout transaction
 867         connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
 868         let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 869         assert_eq!(node_txn.len(), 3);
 870         assert_eq!(node_txn[0].txid(), node_txn[1].txid());
 871         check_spends!(node_txn[1], funding_tx);
 872         check_spends!(node_txn[2], node_txn[1]);
 873         let timeout_txn = vec![node_txn[2].clone()];
 874
 875         nodes[1].node.claim_funds(payment_preimage);
 876         check_added_monitors!(nodes[1], 1);
 877         expect_payment_claimed!(nodes[1], payment_hash, 10_000_000);
 878
 879         connect_block(&nodes[1], &create_dummy_block(nodes[1].best_block_hash(), 42, vec![node_txn[1].clone()]));
 880         check_closed_broadcast!(nodes[1], true);
 881         check_added_monitors!(nodes[1], 1);
 882         check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 883         let claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 884         assert_eq!(claim_txn.len(), 1);
 885         check_spends!(claim_txn[0], node_txn[1]);
 886
 887         connect_block(&nodes[0], &create_dummy_block(nodes[0].best_block_hash(), 42, vec![node_txn[1].clone()]));
 888
 889         if confirm_commitment_tx {
 890                 connect_blocks(&nodes[0], BREAKDOWN_TIMEOUT as u32 - 1);
 891         }
 892
 893         let claim_block = create_dummy_block(nodes[0].best_block_hash(), 42, if payment_timeout { timeout_txn } else { vec![claim_txn[0].clone()] });
 894
 895         if payment_timeout {
 896                 assert!(confirm_commitment_tx); // Otherwise we're spending below our CSV!
 897                 connect_block(&nodes[0], &claim_block);
 898                 connect_blocks(&nodes[0], ANTI_REORG_DELAY - 2);
 899         }
 900
 901         // Now connect the HTLC claim transaction with the ChainMonitor-generated ChannelMonitor update
 902         // returning InProgress. This should cause the claim event to never make its way to the
 903         // ChannelManager.
 904         chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap().clear();
 905         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
 906
 907         if payment_timeout {
 908                 connect_blocks(&nodes[0], 1);
 909         } else {
 910                 connect_block(&nodes[0], &claim_block);
 911         }
 912
 913         let funding_txo = OutPoint { txid: funding_tx.txid(), index: 0 };
 914         let mon_updates: Vec<_> = chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap()
 915                 .get_mut(&funding_txo).unwrap().drain().collect();
 916         // If we are using chain::Confirm instead of chain::Listen, we will get the same update twice.
 917         // If we're testing connection idempotency we may get substantially more.
 918         assert!(mon_updates.len() >= 1);
 919         assert!(nodes[0].chain_monitor.release_pending_monitor_events().is_empty());
 920         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 921
 922         // If we persist the ChannelManager here, we should get the PaymentSent event after
 923         // deserialization.
 924         let mut chan_manager_serialized = Vec::new();
 925         if !persist_manager_post_event {
 926                 chan_manager_serialized = nodes[0].node.encode();
 927         }
 928
 929         // Now persist the ChannelMonitor and inform the ChainMonitor that we're done, generating the
 930         // payment sent event.
 931         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
 932         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 933         for update in mon_updates {
 934                 nodes[0].chain_monitor.chain_monitor.channel_monitor_updated(funding_txo, update).unwrap();
 935         }
 936         if payment_timeout {
 937                 expect_payment_failed!(nodes[0], payment_hash, false);
 938         } else {
 939                 expect_payment_sent!(nodes[0], payment_preimage);
 940         }
 941
 942         // If we persist the ChannelManager after we get the PaymentSent event, we shouldn't get it
 943         // twice.
 944         if persist_manager_post_event {
 945                 chan_manager_serialized = nodes[0].node.encode();
 946         }
 947
 948         // Now reload nodes[0]...
 949         reload_node!(nodes[0], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 950
 951         if persist_manager_post_event {
 952                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 953         } else if payment_timeout {
 954                 expect_payment_failed!(nodes[0], payment_hash, false);
 955         } else {
 956                 expect_payment_sent!(nodes[0], payment_preimage);
 957         }
 958
 959         // Note that if we re-connect the block which exposed nodes[0] to the payment preimage (but
 960         // which the current ChannelMonitor has not seen), the ChannelManager's de-duplication of
 961         // payment events should kick in, leaving us with no pending events here.
 962         let height = nodes[0].blocks.lock().unwrap().len() as u32 - 1;
 963         nodes[0].chain_monitor.chain_monitor.block_connected(&claim_block, height);
 964         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 965         check_added_monitors(&nodes[0], 1);
 966 }
 967
 968 #[test]
 969 fn test_dup_htlc_onchain_fails_on_reload() {
 970         do_test_dup_htlc_onchain_fails_on_reload(true, true, true);
 971         do_test_dup_htlc_onchain_fails_on_reload(true, true, false);
 972         do_test_dup_htlc_onchain_fails_on_reload(true, false, false);
 973         do_test_dup_htlc_onchain_fails_on_reload(false, true, true);
 974         do_test_dup_htlc_onchain_fails_on_reload(false, true, false);
 975         do_test_dup_htlc_onchain_fails_on_reload(false, false, false);
 976 }
 977
 978 #[test]
 979 fn test_fulfill_restart_failure() {
 980         // When we receive an update_fulfill_htlc message, we immediately consider the HTLC fully
 981         // fulfilled. At this point, the peer can reconnect and decide to either fulfill the HTLC
 982         // again, or fail it, giving us free money.
 983         //
 984         // Of course probably they won't fail it and give us free money, but because we have code to
 985         // handle it, we should test the logic for it anyway. We do that here.
 986         let chanmon_cfgs = create_chanmon_cfgs(2);
 987         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 988         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 989         let persister: test_utils::TestPersister;
 990         let new_chain_monitor: test_utils::TestChainMonitor;
 991         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 992         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 993
 994         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 995         let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 100_000);
 996
 997         // The simplest way to get a failure after a fulfill is to reload nodes[1] from a state
 998         // pre-fulfill, which we do by serializing it here.
 999         let chan_manager_serialized = nodes[1].node.encode();
1000         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id).encode();
1001
1002         nodes[1].node.claim_funds(payment_preimage);
1003         check_added_monitors!(nodes[1], 1);
1004         expect_payment_claimed!(nodes[1], payment_hash, 100_000);
1005
1006         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1007         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
1008         expect_payment_sent_without_paths!(nodes[0], payment_preimage);
1009
1010         // Now reload nodes[1]...
1011         reload_node!(nodes[1], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1012
1013         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1014         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1015
1016         nodes[1].node.fail_htlc_backwards(&payment_hash);
1017         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]);
1018         check_added_monitors!(nodes[1], 1);
1019         let htlc_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1020         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates.update_fail_htlcs[0]);
1021         commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates.commitment_signed, false);
1022         // nodes[0] shouldn't generate any events here, while it just got a payment failure completion
1023         // it had already considered the payment fulfilled, and now they just got free money.
1024         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1025 }
1026
1027 #[test]
1028 fn get_ldk_payment_preimage() {
1029         // Ensure that `ChannelManager::get_payment_preimage` can successfully be used to claim a payment.
1030         let chanmon_cfgs = create_chanmon_cfgs(2);
1031         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1032         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1033         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1034         create_announced_chan_between_nodes(&nodes, 0, 1);
1035
1036         let amt_msat = 60_000;
1037         let expiry_secs = 60 * 60;
1038         let (payment_hash, payment_secret) = nodes[1].node.create_inbound_payment(Some(amt_msat), expiry_secs, None).unwrap();
1039
1040         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
1041                 .with_bolt11_features(nodes[1].node.invoice_features()).unwrap();
1042         let scorer = test_utils::TestScorer::new();
1043         let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet);
1044         let random_seed_bytes = keys_manager.get_secure_random_bytes();
1045         let route = get_route(
1046                 &nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(),
1047                 Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()),
1048                 amt_msat, nodes[0].logger, &scorer, &(), &random_seed_bytes).unwrap();
1049         nodes[0].node.send_payment_with_route(&route, payment_hash,
1050                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
1051         check_added_monitors!(nodes[0], 1);
1052
1053         // Make sure to use `get_payment_preimage`
1054         let payment_preimage = nodes[1].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1055         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1056         assert_eq!(events.len(), 1);
1057         pass_along_path(&nodes[0], &[&nodes[1]], amt_msat, payment_hash, Some(payment_secret), events.pop().unwrap(), true, Some(payment_preimage));
1058         claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, payment_preimage);
1059 }
1060
1061 #[test]
1062 fn sent_probe_is_probe_of_sending_node() {
1063         let chanmon_cfgs = create_chanmon_cfgs(3);
1064         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1065         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1066         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1067
1068         create_announced_chan_between_nodes(&nodes, 0, 1);
1069         create_announced_chan_between_nodes(&nodes, 1, 2);
1070
1071         // First check we refuse to build a single-hop probe
1072         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[1], 100_000);
1073         assert!(nodes[0].node.send_probe(route.paths[0].clone()).is_err());
1074
1075         // Then build an actual two-hop probing path
1076         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
1077
1078         match nodes[0].node.send_probe(route.paths[0].clone()) {
1079                 Ok((payment_hash, payment_id)) => {
1080                         assert!(nodes[0].node.payment_is_probe(&payment_hash, &payment_id));
1081                         assert!(!nodes[1].node.payment_is_probe(&payment_hash, &payment_id));
1082                         assert!(!nodes[2].node.payment_is_probe(&payment_hash, &payment_id));
1083                 },
1084                 _ => panic!(),
1085         }
1086
1087         get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1088         check_added_monitors!(nodes[0], 1);
1089 }
1090
1091 #[test]
1092 fn successful_probe_yields_event() {
1093         let chanmon_cfgs = create_chanmon_cfgs(3);
1094         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1095         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1096         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1097
1098         create_announced_chan_between_nodes(&nodes, 0, 1);
1099         create_announced_chan_between_nodes(&nodes, 1, 2);
1100
1101         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
1102
1103         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1104
1105         // node[0] -- update_add_htlcs -> node[1]
1106         check_added_monitors!(nodes[0], 1);
1107         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1108         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1109         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1110         check_added_monitors!(nodes[1], 0);
1111         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1112         expect_pending_htlcs_forwardable!(nodes[1]);
1113
1114         // node[1] -- update_add_htlcs -> node[2]
1115         check_added_monitors!(nodes[1], 1);
1116         let updates = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1117         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1118         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &probe_event.msgs[0]);
1119         check_added_monitors!(nodes[2], 0);
1120         commitment_signed_dance!(nodes[2], nodes[1], probe_event.commitment_msg, true, true);
1121
1122         // node[1] <- update_fail_htlcs -- node[2]
1123         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1124         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1125         check_added_monitors!(nodes[1], 0);
1126         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, true);
1127
1128         // node[0] <- update_fail_htlcs -- node[1]
1129         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1130         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1131         check_added_monitors!(nodes[0], 0);
1132         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1133
1134         let mut events = nodes[0].node.get_and_clear_pending_events();
1135         assert_eq!(events.len(), 1);
1136         match events.drain(..).next().unwrap() {
1137                 crate::events::Event::ProbeSuccessful { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1138                         assert_eq!(payment_id, ev_pid);
1139                         assert_eq!(payment_hash, ev_ph);
1140                 },
1141                 _ => panic!(),
1142         };
1143         assert!(!nodes[0].node.has_pending_payments());
1144 }
1145
1146 #[test]
1147 fn failed_probe_yields_event() {
1148         let chanmon_cfgs = create_chanmon_cfgs(3);
1149         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1150         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1151         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1152
1153         create_announced_chan_between_nodes(&nodes, 0, 1);
1154         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 100000, 90000000);
1155
1156         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1157
1158         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 9_998_000);
1159
1160         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1161
1162         // node[0] -- update_add_htlcs -> node[1]
1163         check_added_monitors!(nodes[0], 1);
1164         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1165         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1166         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1167         check_added_monitors!(nodes[1], 0);
1168         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1169         expect_pending_htlcs_forwardable!(nodes[1]);
1170
1171         // node[0] <- update_fail_htlcs -- node[1]
1172         check_added_monitors!(nodes[1], 1);
1173         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1174         // Skip the PendingHTLCsForwardable event
1175         let _events = nodes[1].node.get_and_clear_pending_events();
1176         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1177         check_added_monitors!(nodes[0], 0);
1178         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1179
1180         let mut events = nodes[0].node.get_and_clear_pending_events();
1181         assert_eq!(events.len(), 1);
1182         match events.drain(..).next().unwrap() {
1183                 crate::events::Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1184                         assert_eq!(payment_id, ev_pid);
1185                         assert_eq!(payment_hash, ev_ph);
1186                 },
1187                 _ => panic!(),
1188         };
1189         assert!(!nodes[0].node.has_pending_payments());
1190 }
1191
1192 #[test]
1193 fn onchain_failed_probe_yields_event() {
1194         // Tests that an attempt to probe over a channel that is eventaully closed results in a failure
1195         // event.
1196         let chanmon_cfgs = create_chanmon_cfgs(3);
1197         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1198         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1199         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1200
1201         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1202         create_announced_chan_between_nodes(&nodes, 1, 2);
1203
1204         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1205
1206         // Send a dust HTLC, which will be treated as if it timed out once the channel hits the chain.
1207         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], &payment_params, 1_000);
1208         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1209
1210         // node[0] -- update_add_htlcs -> node[1]
1211         check_added_monitors!(nodes[0], 1);
1212         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1213         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1214         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1215         check_added_monitors!(nodes[1], 0);
1216         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1217         expect_pending_htlcs_forwardable!(nodes[1]);
1218
1219         check_added_monitors!(nodes[1], 1);
1220         let _ = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1221
1222         // Don't bother forwarding the HTLC onwards and just confirm the force-close transaction on
1223         // Node A, which after 6 confirmations should result in a probe failure event.
1224         let bs_txn = get_local_commitment_txn!(nodes[1], chan_id);
1225         confirm_transaction(&nodes[0], &bs_txn[0]);
1226         check_closed_broadcast!(&nodes[0], true);
1227         check_added_monitors!(nodes[0], 1);
1228
1229         let mut events = nodes[0].node.get_and_clear_pending_events();
1230         assert_eq!(events.len(), 2);
1231         let mut found_probe_failed = false;
1232         for event in events.drain(..) {
1233                 match event {
1234                         Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1235                                 assert_eq!(payment_id, ev_pid);
1236                                 assert_eq!(payment_hash, ev_ph);
1237                                 found_probe_failed = true;
1238                         },
1239                         Event::ChannelClosed { .. } => {},
1240                         _ => panic!(),
1241                 }
1242         }
1243         assert!(found_probe_failed);
1244         assert!(!nodes[0].node.has_pending_payments());
1245 }
1246
1247 #[test]
1248 fn claimed_send_payment_idempotent() {
1249         // Tests that `send_payment` (and friends) are (reasonably) idempotent.
1250         let chanmon_cfgs = create_chanmon_cfgs(2);
1251         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1252         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1253         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1254
1255         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1256
1257         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1258         let (first_payment_preimage, _, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1259
1260         macro_rules! check_send_rejected {
1261                 () => {
1262                         // If we try to resend a new payment with a different payment_hash but with the same
1263                         // payment_id, it should be rejected.
1264                         let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1265                                 RecipientOnionFields::secret_only(second_payment_secret), payment_id);
1266                         match send_result {
1267                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1268                                 _ => panic!("Unexpected send result: {:?}", send_result),
1269                         }
1270
1271                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1272                         // also be rejected.
1273                         let send_result = nodes[0].node.send_spontaneous_payment(
1274                                 &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
1275                         match send_result {
1276                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1277                                 _ => panic!("Unexpected send result: {:?}", send_result),
1278                         }
1279                 }
1280         }
1281
1282         check_send_rejected!();
1283
1284         // Claim the payment backwards, but note that the PaymentSent event is still pending and has
1285         // not been seen by the user. At this point, from the user perspective nothing has changed, so
1286         // we must remain just as idempotent as we were before.
1287         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, first_payment_preimage);
1288
1289         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1290                 nodes[0].node.timer_tick_occurred();
1291         }
1292
1293         check_send_rejected!();
1294
1295         // Once the user sees and handles the `PaymentSent` event, we expect them to no longer call
1296         // `send_payment`, and our idempotency guarantees are off - they should have atomically marked
1297         // the payment complete. However, they could have called `send_payment` while the event was
1298         // being processed, leading to a race in our idempotency guarantees. Thus, even immediately
1299         // after the event is handled a duplicate payment should sitll be rejected.
1300         expect_payment_sent!(&nodes[0], first_payment_preimage, Some(0));
1301         check_send_rejected!();
1302
1303         // If relatively little time has passed, a duplicate payment should still fail.
1304         nodes[0].node.timer_tick_occurred();
1305         check_send_rejected!();
1306
1307         // However, after some time has passed (at least more than the one timer tick above), a
1308         // duplicate payment should go through, as ChannelManager should no longer have any remaining
1309         // references to the old payment data.
1310         for _ in 0..IDEMPOTENCY_TIMEOUT_TICKS {
1311                 nodes[0].node.timer_tick_occurred();
1312         }
1313
1314         nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1315                 RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
1316         check_added_monitors!(nodes[0], 1);
1317         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1318         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1319 }
1320
1321 #[test]
1322 fn abandoned_send_payment_idempotent() {
1323         // Tests that `send_payment` (and friends) allow duplicate PaymentIds immediately after
1324         // abandon_payment.
1325         let chanmon_cfgs = create_chanmon_cfgs(2);
1326         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1327         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1328         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1329
1330         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1331
1332         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1333         let (_, first_payment_hash, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1334
1335         macro_rules! check_send_rejected {
1336                 () => {
1337                         // If we try to resend a new payment with a different payment_hash but with the same
1338                         // payment_id, it should be rejected.
1339                         let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1340                                 RecipientOnionFields::secret_only(second_payment_secret), payment_id);
1341                         match send_result {
1342                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1343                                 _ => panic!("Unexpected send result: {:?}", send_result),
1344                         }
1345
1346                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1347                         // also be rejected.
1348                         let send_result = nodes[0].node.send_spontaneous_payment(
1349                                 &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
1350                         match send_result {
1351                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1352                                 _ => panic!("Unexpected send result: {:?}", send_result),
1353                         }
1354                 }
1355         }
1356
1357         check_send_rejected!();
1358
1359         nodes[1].node.fail_htlc_backwards(&first_payment_hash);
1360         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash: first_payment_hash }]);
1361
1362         // Until we abandon the payment upon path failure, no matter how many timer ticks pass, we still cannot reuse the
1363         // PaymentId.
1364         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1365                 nodes[0].node.timer_tick_occurred();
1366         }
1367         check_send_rejected!();
1368
1369         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash, PaymentFailureReason::RecipientRejected);
1370
1371         // However, we can reuse the PaymentId immediately after we `abandon_payment` upon passing the
1372         // failed payment back.
1373         nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1374                 RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
1375         check_added_monitors!(nodes[0], 1);
1376         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1377         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1378 }
1379
1380 #[derive(PartialEq)]
1381 enum InterceptTest {
1382         Forward,
1383         Fail,
1384         Timeout,
1385 }
1386
1387 #[test]
1388 fn test_trivial_inflight_htlc_tracking(){
1389         // In this test, we test three scenarios:
1390         // (1) Sending + claiming a payment successfully should return `None` when querying InFlightHtlcs
1391         // (2) Sending a payment without claiming it should return the payment's value (500000) when querying InFlightHtlcs
1392         // (3) After we claim the payment sent in (2), InFlightHtlcs should return `None` for the query.
1393         let chanmon_cfgs = create_chanmon_cfgs(3);
1394         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1395         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1396         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1397
1398         let (_, _, chan_1_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
1399         let (_, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
1400
1401         // Send and claim the payment. Inflight HTLCs should be empty.
1402         let payment_hash = send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 500000).1;
1403         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1404         {
1405                 let mut node_0_per_peer_lock;
1406                 let mut node_0_peer_state_lock;
1407                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1408
1409                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1410                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1411                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1412                         channel_1.context.get_short_channel_id().unwrap()
1413                 );
1414                 assert_eq!(chan_1_used_liquidity, None);
1415         }
1416         {
1417                 let mut node_1_per_peer_lock;
1418                 let mut node_1_peer_state_lock;
1419                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1420
1421                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1422                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1423                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1424                         channel_2.context.get_short_channel_id().unwrap()
1425                 );
1426
1427                 assert_eq!(chan_2_used_liquidity, None);
1428         }
1429         let pending_payments = nodes[0].node.list_recent_payments();
1430         assert_eq!(pending_payments.len(), 1);
1431         assert_eq!(pending_payments[0], RecentPaymentDetails::Fulfilled { payment_hash: Some(payment_hash) });
1432
1433         // Remove fulfilled payment
1434         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1435                 nodes[0].node.timer_tick_occurred();
1436         }
1437
1438         // Send the payment, but do not claim it. Our inflight HTLCs should contain the pending payment.
1439         let (payment_preimage, payment_hash,  _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 500000);
1440         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1441         {
1442                 let mut node_0_per_peer_lock;
1443                 let mut node_0_peer_state_lock;
1444                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1445
1446                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1447                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1448                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1449                         channel_1.context.get_short_channel_id().unwrap()
1450                 );
1451                 // First hop accounts for expected 1000 msat fee
1452                 assert_eq!(chan_1_used_liquidity, Some(501000));
1453         }
1454         {
1455                 let mut node_1_per_peer_lock;
1456                 let mut node_1_peer_state_lock;
1457                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1458
1459                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1460                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1461                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1462                         channel_2.context.get_short_channel_id().unwrap()
1463                 );
1464
1465                 assert_eq!(chan_2_used_liquidity, Some(500000));
1466         }
1467         let pending_payments = nodes[0].node.list_recent_payments();
1468         assert_eq!(pending_payments.len(), 1);
1469         assert_eq!(pending_payments[0], RecentPaymentDetails::Pending { payment_hash, total_msat: 500000 });
1470
1471         // Now, let's claim the payment. This should result in the used liquidity to return `None`.
1472         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
1473
1474         // Remove fulfilled payment
1475         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1476                 nodes[0].node.timer_tick_occurred();
1477         }
1478
1479         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1480         {
1481                 let mut node_0_per_peer_lock;
1482                 let mut node_0_peer_state_lock;
1483                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1484
1485                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1486                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1487                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1488                         channel_1.context.get_short_channel_id().unwrap()
1489                 );
1490                 assert_eq!(chan_1_used_liquidity, None);
1491         }
1492         {
1493                 let mut node_1_per_peer_lock;
1494                 let mut node_1_peer_state_lock;
1495                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1496
1497                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1498                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1499                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1500                         channel_2.context.get_short_channel_id().unwrap()
1501                 );
1502                 assert_eq!(chan_2_used_liquidity, None);
1503         }
1504
1505         let pending_payments = nodes[0].node.list_recent_payments();
1506         assert_eq!(pending_payments.len(), 0);
1507 }
1508
1509 #[test]
1510 fn test_holding_cell_inflight_htlcs() {
1511         let chanmon_cfgs = create_chanmon_cfgs(2);
1512         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1513         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1514         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1515         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1516
1517         let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
1518         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1519
1520         // Queue up two payments - one will be delivered right away, one immediately goes into the
1521         // holding cell as nodes[0] is AwaitingRAA.
1522         {
1523                 nodes[0].node.send_payment_with_route(&route, payment_hash_1,
1524                         RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
1525                 check_added_monitors!(nodes[0], 1);
1526                 nodes[0].node.send_payment_with_route(&route, payment_hash_2,
1527                         RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
1528                 check_added_monitors!(nodes[0], 0);
1529         }
1530
1531         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1532
1533         {
1534                 let mut node_0_per_peer_lock;
1535                 let mut node_0_peer_state_lock;
1536                 let channel =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, channel_id);
1537
1538                 let used_liquidity = inflight_htlcs.used_liquidity_msat(
1539                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1540                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1541                         channel.context.get_short_channel_id().unwrap()
1542                 );
1543
1544                 assert_eq!(used_liquidity, Some(2000000));
1545         }
1546
1547         // Clear pending events so test doesn't throw a "Had excess message on node..." error
1548         nodes[0].node.get_and_clear_pending_msg_events();
1549 }
1550
1551 #[test]
1552 fn intercepted_payment() {
1553         // Test that detecting an intercept scid on payment forward will signal LDK to generate an
1554         // intercept event, which the LSP can then use to either (a) open a JIT channel to forward the
1555         // payment or (b) fail the payment.
1556         do_test_intercepted_payment(InterceptTest::Forward);
1557         do_test_intercepted_payment(InterceptTest::Fail);
1558         // Make sure that intercepted payments will be automatically failed back if too many blocks pass.
1559         do_test_intercepted_payment(InterceptTest::Timeout);
1560 }
1561
1562 fn do_test_intercepted_payment(test: InterceptTest) {
1563         let chanmon_cfgs = create_chanmon_cfgs(3);
1564         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1565
1566         let mut zero_conf_chan_config = test_default_channel_config();
1567         zero_conf_chan_config.manually_accept_inbound_channels = true;
1568         let mut intercept_forwards_config = test_default_channel_config();
1569         intercept_forwards_config.accept_intercept_htlcs = true;
1570         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(zero_conf_chan_config)]);
1571
1572         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1573         let scorer = test_utils::TestScorer::new();
1574         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
1575
1576         let _ = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1577
1578         let amt_msat = 100_000;
1579         let intercept_scid = nodes[1].node.get_intercept_scid();
1580         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1581                 .with_route_hints(vec![
1582                         RouteHint(vec![RouteHintHop {
1583                                 src_node_id: nodes[1].node.get_our_node_id(),
1584                                 short_channel_id: intercept_scid,
1585                                 fees: RoutingFees {
1586                                         base_msat: 1000,
1587                                         proportional_millionths: 0,
1588                                 },
1589                                 cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
1590                                 htlc_minimum_msat: None,
1591                                 htlc_maximum_msat: None,
1592                         }])
1593                 ]).unwrap()
1594                 .with_bolt11_features(nodes[2].node.invoice_features()).unwrap();
1595         let route_params = RouteParameters {
1596                 payment_params,
1597                 final_value_msat: amt_msat,
1598         };
1599         let route = get_route(
1600                 &nodes[0].node.get_our_node_id(), &route_params.payment_params,
1601                 &nodes[0].network_graph.read_only(), None, route_params.final_value_msat,
1602                 nodes[0].logger, &scorer, &(), &random_seed_bytes,
1603         ).unwrap();
1604
1605         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
1606         nodes[0].node.send_payment_with_route(&route, payment_hash,
1607                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
1608         let payment_event = {
1609                 {
1610                         let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
1611                         assert_eq!(added_monitors.len(), 1);
1612                         added_monitors.clear();
1613                 }
1614                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1615                 assert_eq!(events.len(), 1);
1616                 SendEvent::from_event(events.remove(0))
1617         };
1618         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1619         commitment_signed_dance!(nodes[1], nodes[0], &payment_event.commitment_msg, false, true);
1620
1621         // Check that we generate the PaymentIntercepted event when an intercept forward is detected.
1622         let events = nodes[1].node.get_and_clear_pending_events();
1623         assert_eq!(events.len(), 1);
1624         let (intercept_id, expected_outbound_amount_msat) = match events[0] {
1625                 crate::events::Event::HTLCIntercepted {
1626                         intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, inbound_amount_msat, requested_next_hop_scid: short_channel_id
1627                 } => {
1628                         assert_eq!(pmt_hash, payment_hash);
1629                         assert_eq!(inbound_amount_msat, route.get_total_amount() + route.get_total_fees());
1630                         assert_eq!(short_channel_id, intercept_scid);
1631                         (intercept_id, expected_outbound_amount_msat)
1632                 },
1633                 _ => panic!()
1634         };
1635
1636         // Check for unknown channel id error.
1637         let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &[42; 32], nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1638         assert_eq!(unknown_chan_id_err , APIError::ChannelUnavailable  {
1639                 err: format!("Funded channel with id {} not found for the passed counterparty node_id {}. Channel may still be opening.",
1640                         log_bytes!([42; 32]), nodes[2].node.get_our_node_id()) });
1641
1642         if test == InterceptTest::Fail {
1643                 // Ensure we can fail the intercepted payment back.
1644                 nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap();
1645                 expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::UnknownNextHop { requested_forward_scid: intercept_scid }]);
1646                 nodes[1].node.process_pending_htlc_forwards();
1647                 let update_fail = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1648                 check_added_monitors!(&nodes[1], 1);
1649                 assert!(update_fail.update_fail_htlcs.len() == 1);
1650                 let fail_msg = update_fail.update_fail_htlcs[0].clone();
1651                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1652                 commitment_signed_dance!(nodes[0], nodes[1], update_fail.commitment_signed, false);
1653
1654                 // Ensure the payment fails with the expected error.
1655                 let fail_conditions = PaymentFailedConditions::new()
1656                         .blamed_scid(intercept_scid)
1657                         .blamed_chan_closed(true)
1658                         .expected_htlc_error_data(0x4000 | 10, &[]);
1659                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
1660         } else if test == InterceptTest::Forward {
1661                 // Check that we'll fail as expected when sending to a channel that isn't in `ChannelReady` yet.
1662                 let temp_chan_id = nodes[1].node.create_channel(nodes[2].node.get_our_node_id(), 100_000, 0, 42, None).unwrap();
1663                 let unusable_chan_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &temp_chan_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1664                 assert_eq!(unusable_chan_err , APIError::ChannelUnavailable {
1665                         err: format!("Funded channel with id {} not found for the passed counterparty node_id {}. Channel may still be opening.",
1666                                 log_bytes!(temp_chan_id), nodes[2].node.get_our_node_id()) });
1667                 assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
1668
1669                 // Open the just-in-time channel so the payment can then be forwarded.
1670                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1671
1672                 // Finally, forward the intercepted payment through and claim it.
1673                 nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap();
1674                 expect_pending_htlcs_forwardable!(nodes[1]);
1675
1676                 let payment_event = {
1677                         {
1678                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
1679                                 assert_eq!(added_monitors.len(), 1);
1680                                 added_monitors.clear();
1681                         }
1682                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
1683                         assert_eq!(events.len(), 1);
1684                         SendEvent::from_event(events.remove(0))
1685                 };
1686                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1687                 commitment_signed_dance!(nodes[2], nodes[1], &payment_event.commitment_msg, false, true);
1688                 expect_pending_htlcs_forwardable!(nodes[2]);
1689
1690                 let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1691                 expect_payment_claimable!(&nodes[2], payment_hash, payment_secret, amt_msat, Some(payment_preimage), nodes[2].node.get_our_node_id());
1692                 do_claim_payment_along_route(&nodes[0], &vec!(&vec!(&nodes[1], &nodes[2])[..]), false, payment_preimage);
1693                 let events = nodes[0].node.get_and_clear_pending_events();
1694                 assert_eq!(events.len(), 2);
1695                 match events[0] {
1696                         Event::PaymentSent { payment_preimage: ref ev_preimage, payment_hash: ref ev_hash, ref fee_paid_msat, .. } => {
1697                                 assert_eq!(payment_preimage, *ev_preimage);
1698                                 assert_eq!(payment_hash, *ev_hash);
1699                                 assert_eq!(fee_paid_msat, &Some(1000));
1700                         },
1701                         _ => panic!("Unexpected event")
1702                 }
1703                 match events[1] {
1704                         Event::PaymentPathSuccessful { payment_hash: hash, .. } => {
1705                                 assert_eq!(hash, Some(payment_hash));
1706                         },
1707                         _ => panic!("Unexpected event")
1708                 }
1709         } else if test == InterceptTest::Timeout {
1710                 let mut block = create_dummy_block(nodes[0].best_block_hash(), 42, Vec::new());
1711                 connect_block(&nodes[0], &block);
1712                 connect_block(&nodes[1], &block);
1713                 for _ in 0..TEST_FINAL_CLTV {
1714                         block.header.prev_blockhash = block.block_hash();
1715                         connect_block(&nodes[0], &block);
1716                         connect_block(&nodes[1], &block);
1717                 }
1718                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::InvalidForward { requested_forward_scid: intercept_scid }]);
1719                 check_added_monitors!(nodes[1], 1);
1720                 let htlc_timeout_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1721                 assert!(htlc_timeout_updates.update_add_htlcs.is_empty());
1722                 assert_eq!(htlc_timeout_updates.update_fail_htlcs.len(), 1);
1723                 assert!(htlc_timeout_updates.update_fail_malformed_htlcs.is_empty());
1724                 assert!(htlc_timeout_updates.update_fee.is_none());
1725
1726                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_timeout_updates.update_fail_htlcs[0]);
1727                 commitment_signed_dance!(nodes[0], nodes[1], htlc_timeout_updates.commitment_signed, false);
1728                 expect_payment_failed!(nodes[0], payment_hash, false, 0x2000 | 2, []);
1729
1730                 // Check for unknown intercept id error.
1731                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1732                 let unknown_intercept_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1733                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1734                 let unknown_intercept_id_err = nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap_err();
1735                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1736         }
1737 }
1738
1739 #[derive(PartialEq)]
1740 enum AutoRetry {
1741         Success,
1742         Spontaneous,
1743         FailAttempts,
1744         FailTimeout,
1745         FailOnRestart,
1746         FailOnRetry,
1747 }
1748
1749 #[test]
1750 fn automatic_retries() {
1751         do_automatic_retries(AutoRetry::Success);
1752         do_automatic_retries(AutoRetry::Spontaneous);
1753         do_automatic_retries(AutoRetry::FailAttempts);
1754         do_automatic_retries(AutoRetry::FailTimeout);
1755         do_automatic_retries(AutoRetry::FailOnRestart);
1756         do_automatic_retries(AutoRetry::FailOnRetry);
1757 }
1758 fn do_automatic_retries(test: AutoRetry) {
1759         // Test basic automatic payment retries in ChannelManager. See individual `test` variant comments
1760         // below.
1761         let chanmon_cfgs = create_chanmon_cfgs(3);
1762         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1763         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1764
1765         let persister;
1766         let new_chain_monitor;
1767         let node_0_deserialized;
1768
1769         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1770         let channel_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1771         let channel_id_2 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1772
1773         // Marshall data to send the payment
1774         #[cfg(feature = "std")]
1775         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1776         #[cfg(not(feature = "std"))]
1777         let payment_expiry_secs = 60 * 60;
1778         let amt_msat = 1000;
1779         let mut invoice_features = InvoiceFeatures::empty();
1780         invoice_features.set_variable_length_onion_required();
1781         invoice_features.set_payment_secret_required();
1782         invoice_features.set_basic_mpp_optional();
1783         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1784                 .with_expiry_time(payment_expiry_secs as u64)
1785                 .with_bolt11_features(invoice_features).unwrap();
1786         let route_params = RouteParameters {
1787                 payment_params,
1788                 final_value_msat: amt_msat,
1789         };
1790         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
1791
1792         macro_rules! pass_failed_attempt_with_retry_along_path {
1793                 ($failing_channel_id: expr, $expect_pending_htlcs_forwardable: expr) => {
1794                         // Send a payment attempt that fails due to lack of liquidity on the second hop
1795                         check_added_monitors!(nodes[0], 1);
1796                         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1797                         let mut update_add = update_0.update_add_htlcs[0].clone();
1798                         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
1799                         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
1800                         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
1801                         nodes[1].node.process_pending_htlc_forwards();
1802                         expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1],
1803                                 vec![HTLCDestination::NextHopChannel {
1804                                         node_id: Some(nodes[2].node.get_our_node_id()),
1805                                         channel_id: $failing_channel_id,
1806                                 }]);
1807                         nodes[1].node.process_pending_htlc_forwards();
1808                         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1809                         check_added_monitors!(&nodes[1], 1);
1810                         assert!(update_1.update_fail_htlcs.len() == 1);
1811                         let fail_msg = update_1.update_fail_htlcs[0].clone();
1812                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1813                         commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
1814
1815                         // Ensure the attempt fails and a new PendingHTLCsForwardable event is generated for the retry
1816                         let mut events = nodes[0].node.get_and_clear_pending_events();
1817                         assert_eq!(events.len(), 2);
1818                         match events[0] {
1819                                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
1820                                         assert_eq!(payment_hash, ev_payment_hash);
1821                                         assert_eq!(payment_failed_permanently, false);
1822                                 },
1823                                 _ => panic!("Unexpected event"),
1824                         }
1825                         if $expect_pending_htlcs_forwardable {
1826                                 match events[1] {
1827                                         Event::PendingHTLCsForwardable { .. } => {},
1828                                         _ => panic!("Unexpected event"),
1829                                 }
1830                         } else {
1831                                 match events[1] {
1832                                         Event::PaymentFailed { payment_hash: ev_payment_hash, .. } => {
1833                                                 assert_eq!(payment_hash, ev_payment_hash);
1834                                         },
1835                                         _ => panic!("Unexpected event"),
1836                                 }
1837                         }
1838                 }
1839         }
1840
1841         if test == AutoRetry::Success {
1842                 // Test that we can succeed on the first retry.
1843                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
1844                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1845                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1846
1847                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
1848                 // attempt, since the initial second hop channel will be excluded from pathfinding
1849                 create_announced_chan_between_nodes(&nodes, 1, 2);
1850
1851                 // We retry payments in `process_pending_htlc_forwards`
1852                 nodes[0].node.process_pending_htlc_forwards();
1853                 check_added_monitors!(nodes[0], 1);
1854                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1855                 assert_eq!(msg_events.len(), 1);
1856                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, Some(payment_secret), msg_events.pop().unwrap(), true, None);
1857                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
1858         } else if test == AutoRetry::Spontaneous {
1859                 nodes[0].node.send_spontaneous_payment_with_retry(Some(payment_preimage),
1860                         RecipientOnionFields::spontaneous_empty(), PaymentId(payment_hash.0), route_params,
1861                         Retry::Attempts(1)).unwrap();
1862                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1863
1864                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
1865                 // attempt, since the initial second hop channel will be excluded from pathfinding
1866                 create_announced_chan_between_nodes(&nodes, 1, 2);
1867
1868                 // We retry payments in `process_pending_htlc_forwards`
1869                 nodes[0].node.process_pending_htlc_forwards();
1870                 check_added_monitors!(nodes[0], 1);
1871                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1872                 assert_eq!(msg_events.len(), 1);
1873                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, None, msg_events.pop().unwrap(), true, Some(payment_preimage));
1874                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
1875         } else if test == AutoRetry::FailAttempts {
1876                 // Ensure ChannelManager will not retry a payment if it has run out of payment attempts.
1877                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
1878                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1879                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1880
1881                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
1882                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
1883                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1884
1885                 // We retry payments in `process_pending_htlc_forwards`
1886                 nodes[0].node.process_pending_htlc_forwards();
1887                 pass_failed_attempt_with_retry_along_path!(channel_id_3, false);
1888
1889                 // Ensure we won't retry a second time.
1890                 nodes[0].node.process_pending_htlc_forwards();
1891                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1892                 assert_eq!(msg_events.len(), 0);
1893         } else if test == AutoRetry::FailTimeout {
1894                 #[cfg(not(feature = "no-std"))] {
1895                         // Ensure ChannelManager will not retry a payment if it times out due to Retry::Timeout.
1896                         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
1897                                 PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
1898                         pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1899
1900                         // Advance the time so the second attempt fails due to timeout.
1901                         SinceEpoch::advance(Duration::from_secs(61));
1902
1903                         // Make sure we don't retry again.
1904                         nodes[0].node.process_pending_htlc_forwards();
1905                         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1906                         assert_eq!(msg_events.len(), 0);
1907
1908                         let mut events = nodes[0].node.get_and_clear_pending_events();
1909                         assert_eq!(events.len(), 1);
1910                         match events[0] {
1911                                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
1912                                         assert_eq!(payment_hash, *ev_payment_hash);
1913                                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1914                                         assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
1915                                 },
1916                                 _ => panic!("Unexpected event"),
1917                         }
1918                 }
1919         } else if test == AutoRetry::FailOnRestart {
1920                 // Ensure ChannelManager will not retry a payment after restart, even if there were retry
1921                 // attempts remaining prior to restart.
1922                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
1923                         PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
1924                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1925
1926                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
1927                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
1928                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
1929
1930                 // Ensure the first retry attempt fails, with 1 retry attempt remaining
1931                 nodes[0].node.process_pending_htlc_forwards();
1932                 pass_failed_attempt_with_retry_along_path!(channel_id_3, true);
1933
1934                 // Restart the node and ensure that ChannelManager does not use its remaining retry attempt
1935                 let node_encoded = nodes[0].node.encode();
1936                 let chan_1_monitor_serialized = get_monitor!(nodes[0], channel_id_1).encode();
1937                 reload_node!(nodes[0], node_encoded, &[&chan_1_monitor_serialized], persister, new_chain_monitor, node_0_deserialized);
1938
1939                 let mut events = nodes[0].node.get_and_clear_pending_events();
1940                 expect_pending_htlcs_forwardable_from_events!(nodes[0], events, true);
1941                 // Make sure we don't retry again.
1942                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1943                 assert_eq!(msg_events.len(), 0);
1944
1945                 let mut events = nodes[0].node.get_and_clear_pending_events();
1946                 assert_eq!(events.len(), 1);
1947                 match events[0] {
1948                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
1949                                 assert_eq!(payment_hash, *ev_payment_hash);
1950                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1951                                 assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
1952                         },
1953                         _ => panic!("Unexpected event"),
1954                 }
1955         } else if test == AutoRetry::FailOnRetry {
1956                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
1957                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
1958                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
1959
1960                 // We retry payments in `process_pending_htlc_forwards`. Since our channel closed, we should
1961                 // fail to find a route.
1962                 nodes[0].node.process_pending_htlc_forwards();
1963                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
1964                 assert_eq!(msg_events.len(), 0);
1965
1966                 let mut events = nodes[0].node.get_and_clear_pending_events();
1967                 assert_eq!(events.len(), 1);
1968                 match events[0] {
1969                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
1970                                 assert_eq!(payment_hash, *ev_payment_hash);
1971                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
1972                                 assert_eq!(PaymentFailureReason::RouteNotFound, ev_reason.unwrap());
1973                         },
1974                         _ => panic!("Unexpected event"),
1975                 }
1976         }
1977 }
1978
1979 #[test]
1980 fn auto_retry_partial_failure() {
1981         // Test that we'll retry appropriately on send partial failure and retry partial failure.
1982         let chanmon_cfgs = create_chanmon_cfgs(2);
1983         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1984         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1985         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1986
1987         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1988         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1989         let chan_3_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
1990
1991         // Marshall data to send the payment
1992         let amt_msat = 20_000;
1993         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
1994         #[cfg(feature = "std")]
1995         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
1996         #[cfg(not(feature = "std"))]
1997         let payment_expiry_secs = 60 * 60;
1998         let mut invoice_features = InvoiceFeatures::empty();
1999         invoice_features.set_variable_length_onion_required();
2000         invoice_features.set_payment_secret_required();
2001         invoice_features.set_basic_mpp_optional();
2002         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2003                 .with_expiry_time(payment_expiry_secs as u64)
2004                 .with_bolt11_features(invoice_features).unwrap();
2005         let route_params = RouteParameters {
2006                 payment_params,
2007                 final_value_msat: amt_msat,
2008         };
2009
2010         // Ensure the first monitor update (for the initial send path1 over chan_1) succeeds, but the
2011         // second (for the initial send path2 over chan_2) fails.
2012         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
2013         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2014         // Ensure third monitor update (for the retry1's path1 over chan_1) succeeds, but the fourth (for
2015         // the retry1's path2 over chan_3) fails, and monitor updates succeed after that.
2016         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
2017         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2018         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
2019
2020         // Configure the initial send, retry1 and retry2's paths.
2021         let send_route = Route {
2022                 paths: vec![
2023                         Path { hops: vec![RouteHop {
2024                                 pubkey: nodes[1].node.get_our_node_id(),
2025                                 node_features: nodes[1].node.node_features(),
2026                                 short_channel_id: chan_1_id,
2027                                 channel_features: nodes[1].node.channel_features(),
2028                                 fee_msat: amt_msat / 2,
2029                                 cltv_expiry_delta: 100,
2030                         }], blinded_tail: None },
2031                         Path { hops: vec![RouteHop {
2032                                 pubkey: nodes[1].node.get_our_node_id(),
2033                                 node_features: nodes[1].node.node_features(),
2034                                 short_channel_id: chan_2_id,
2035                                 channel_features: nodes[1].node.channel_features(),
2036                                 fee_msat: amt_msat / 2,
2037                                 cltv_expiry_delta: 100,
2038                         }], blinded_tail: None },
2039                 ],
2040                 payment_params: Some(route_params.payment_params.clone()),
2041         };
2042         let retry_1_route = Route {
2043                 paths: vec![
2044                         Path { hops: vec![RouteHop {
2045                                 pubkey: nodes[1].node.get_our_node_id(),
2046                                 node_features: nodes[1].node.node_features(),
2047                                 short_channel_id: chan_1_id,
2048                                 channel_features: nodes[1].node.channel_features(),
2049                                 fee_msat: amt_msat / 4,
2050                                 cltv_expiry_delta: 100,
2051                         }], blinded_tail: None },
2052                         Path { hops: vec![RouteHop {
2053                                 pubkey: nodes[1].node.get_our_node_id(),
2054                                 node_features: nodes[1].node.node_features(),
2055                                 short_channel_id: chan_3_id,
2056                                 channel_features: nodes[1].node.channel_features(),
2057                                 fee_msat: amt_msat / 4,
2058                                 cltv_expiry_delta: 100,
2059                         }], blinded_tail: None },
2060                 ],
2061                 payment_params: Some(route_params.payment_params.clone()),
2062         };
2063         let retry_2_route = Route {
2064                 paths: vec![
2065                         Path { hops: vec![RouteHop {
2066                                 pubkey: nodes[1].node.get_our_node_id(),
2067                                 node_features: nodes[1].node.node_features(),
2068                                 short_channel_id: chan_1_id,
2069                                 channel_features: nodes[1].node.channel_features(),
2070                                 fee_msat: amt_msat / 4,
2071                                 cltv_expiry_delta: 100,
2072                         }], blinded_tail: None },
2073                 ],
2074                 payment_params: Some(route_params.payment_params.clone()),
2075         };
2076         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
2077         let mut payment_params = route_params.payment_params.clone();
2078         payment_params.previously_failed_channels.push(chan_2_id);
2079         nodes[0].router.expect_find_route(RouteParameters {
2080                         payment_params, final_value_msat: amt_msat / 2,
2081                 }, Ok(retry_1_route));
2082         let mut payment_params = route_params.payment_params.clone();
2083         payment_params.previously_failed_channels.push(chan_3_id);
2084         nodes[0].router.expect_find_route(RouteParameters {
2085                         payment_params, final_value_msat: amt_msat / 4,
2086                 }, Ok(retry_2_route));
2087
2088         // Send a payment that will partially fail on send, then partially fail on retry, then succeed.
2089         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2090                 PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap();
2091         let closed_chan_events = nodes[0].node.get_and_clear_pending_events();
2092         assert_eq!(closed_chan_events.len(), 4);
2093         match closed_chan_events[0] {
2094                 Event::ChannelClosed { .. } => {},
2095                 _ => panic!("Unexpected event"),
2096         }
2097         match closed_chan_events[1] {
2098                 Event::PaymentPathFailed { .. } => {},
2099                 _ => panic!("Unexpected event"),
2100         }
2101         match closed_chan_events[2] {
2102                 Event::ChannelClosed { .. } => {},
2103                 _ => panic!("Unexpected event"),
2104         }
2105         match closed_chan_events[3] {
2106                 Event::PaymentPathFailed { .. } => {},
2107                 _ => panic!("Unexpected event"),
2108         }
2109
2110         // Pass the first part of the payment along the path.
2111         check_added_monitors!(nodes[0], 5); // three outbound channel updates succeeded, two permanently failed
2112         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2113
2114         // First message is the first update_add, remaining messages are broadcasting channel updates and
2115         // errors for the permfailed channels
2116         assert_eq!(msg_events.len(), 5);
2117         let mut payment_event = SendEvent::from_event(msg_events.remove(0));
2118
2119         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2120         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
2121         check_added_monitors!(nodes[1], 1);
2122         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2123
2124         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2125         check_added_monitors!(nodes[0], 1);
2126         let as_second_htlc_updates = SendEvent::from_node(&nodes[0]);
2127
2128         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2129         check_added_monitors!(nodes[0], 1);
2130         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2131
2132         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2133         check_added_monitors!(nodes[1], 1);
2134
2135         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[0]);
2136         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[1]);
2137         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.commitment_msg);
2138         check_added_monitors!(nodes[1], 1);
2139         let (bs_second_raa, bs_second_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2140
2141         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2142         check_added_monitors!(nodes[0], 1);
2143
2144         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_cs);
2145         check_added_monitors!(nodes[0], 1);
2146         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2147
2148         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2149         check_added_monitors!(nodes[1], 1);
2150
2151         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
2152         nodes[1].node.process_pending_htlc_forwards();
2153         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, amt_msat);
2154         nodes[1].node.claim_funds(payment_preimage);
2155         expect_payment_claimed!(nodes[1], payment_hash, amt_msat);
2156         let bs_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2157         assert_eq!(bs_claim_update.update_fulfill_htlcs.len(), 1);
2158
2159         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_claim_update.update_fulfill_htlcs[0]);
2160         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_claim_update.commitment_signed);
2161         check_added_monitors!(nodes[0], 1);
2162         let (as_third_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2163
2164         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2165         check_added_monitors!(nodes[1], 4);
2166         let bs_second_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2167
2168         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2169         check_added_monitors!(nodes[1], 1);
2170         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2171
2172         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2173         check_added_monitors!(nodes[0], 1);
2174
2175         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[0]);
2176         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[1]);
2177         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.commitment_signed);
2178         check_added_monitors!(nodes[0], 1);
2179         let (as_fourth_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2180
2181         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_fourth_raa);
2182         check_added_monitors!(nodes[1], 1);
2183
2184         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2185         check_added_monitors!(nodes[1], 1);
2186         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2187
2188         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2189         check_added_monitors!(nodes[0], 1);
2190         expect_payment_sent!(nodes[0], payment_preimage);
2191 }
2192
2193 #[test]
2194 fn auto_retry_zero_attempts_send_error() {
2195         let chanmon_cfgs = create_chanmon_cfgs(2);
2196         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2197         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2198         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2199
2200         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2201         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2202
2203         // Marshall data to send the payment
2204         let amt_msat = 20_000;
2205         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2206         #[cfg(feature = "std")]
2207         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2208         #[cfg(not(feature = "std"))]
2209         let payment_expiry_secs = 60 * 60;
2210         let mut invoice_features = InvoiceFeatures::empty();
2211         invoice_features.set_variable_length_onion_required();
2212         invoice_features.set_payment_secret_required();
2213         invoice_features.set_basic_mpp_optional();
2214         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2215                 .with_expiry_time(payment_expiry_secs as u64)
2216                 .with_bolt11_features(invoice_features).unwrap();
2217         let route_params = RouteParameters {
2218                 payment_params,
2219                 final_value_msat: amt_msat,
2220         };
2221
2222         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
2223         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2224                 PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
2225         assert_eq!(nodes[0].node.get_and_clear_pending_msg_events().len(), 2); // channel close messages
2226         let events = nodes[0].node.get_and_clear_pending_events();
2227         assert_eq!(events.len(), 3);
2228         if let Event::ChannelClosed { .. } = events[0] { } else { panic!(); }
2229         if let Event::PaymentPathFailed { .. } = events[1] { } else { panic!(); }
2230         if let Event::PaymentFailed { .. } = events[2] { } else { panic!(); }
2231         check_added_monitors!(nodes[0], 2);
2232 }
2233
2234 #[test]
2235 fn fails_paying_after_rejected_by_payee() {
2236         let chanmon_cfgs = create_chanmon_cfgs(2);
2237         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2238         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2239         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2240
2241         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2242
2243         // Marshall data to send the payment
2244         let amt_msat = 20_000;
2245         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2246         #[cfg(feature = "std")]
2247         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2248         #[cfg(not(feature = "std"))]
2249         let payment_expiry_secs = 60 * 60;
2250         let mut invoice_features = InvoiceFeatures::empty();
2251         invoice_features.set_variable_length_onion_required();
2252         invoice_features.set_payment_secret_required();
2253         invoice_features.set_basic_mpp_optional();
2254         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2255                 .with_expiry_time(payment_expiry_secs as u64)
2256                 .with_bolt11_features(invoice_features).unwrap();
2257         let route_params = RouteParameters {
2258                 payment_params,
2259                 final_value_msat: amt_msat,
2260         };
2261
2262         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2263                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2264         check_added_monitors!(nodes[0], 1);
2265         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
2266         assert_eq!(events.len(), 1);
2267         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
2268         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2269         check_added_monitors!(nodes[1], 0);
2270         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
2271         expect_pending_htlcs_forwardable!(nodes[1]);
2272         expect_payment_claimable!(&nodes[1], payment_hash, payment_secret, amt_msat);
2273
2274         nodes[1].node.fail_htlc_backwards(&payment_hash);
2275         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash }]);
2276         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
2277 }
2278
2279 #[test]
2280 fn retry_multi_path_single_failed_payment() {
2281         // Tests that we can/will retry after a single path of an MPP payment failed immediately
2282         let chanmon_cfgs = create_chanmon_cfgs(2);
2283         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2284         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2285         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2286
2287         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2288         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2289
2290         let amt_msat = 100_010_000;
2291
2292         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2293         #[cfg(feature = "std")]
2294         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2295         #[cfg(not(feature = "std"))]
2296         let payment_expiry_secs = 60 * 60;
2297         let mut invoice_features = InvoiceFeatures::empty();
2298         invoice_features.set_variable_length_onion_required();
2299         invoice_features.set_payment_secret_required();
2300         invoice_features.set_basic_mpp_optional();
2301         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2302                 .with_expiry_time(payment_expiry_secs as u64)
2303                 .with_bolt11_features(invoice_features).unwrap();
2304         let route_params = RouteParameters {
2305                 payment_params: payment_params.clone(),
2306                 final_value_msat: amt_msat,
2307         };
2308
2309         let chans = nodes[0].node.list_usable_channels();
2310         let mut route = Route {
2311                 paths: vec![
2312                         Path { hops: vec![RouteHop {
2313                                 pubkey: nodes[1].node.get_our_node_id(),
2314                                 node_features: nodes[1].node.node_features(),
2315                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2316                                 channel_features: nodes[1].node.channel_features(),
2317                                 fee_msat: 10_000,
2318                                 cltv_expiry_delta: 100,
2319                         }], blinded_tail: None },
2320                         Path { hops: vec![RouteHop {
2321                                 pubkey: nodes[1].node.get_our_node_id(),
2322                                 node_features: nodes[1].node.node_features(),
2323                                 short_channel_id: chans[1].short_channel_id.unwrap(),
2324                                 channel_features: nodes[1].node.channel_features(),
2325                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2326                                 cltv_expiry_delta: 100,
2327                         }], blinded_tail: None },
2328                 ],
2329                 payment_params: Some(payment_params),
2330         };
2331         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2332         // On retry, split the payment across both channels.
2333         route.paths[0].hops[0].fee_msat = 50_000_001;
2334         route.paths[1].hops[0].fee_msat = 50_000_000;
2335         let mut pay_params = route.payment_params.clone().unwrap();
2336         pay_params.previously_failed_channels.push(chans[1].short_channel_id.unwrap());
2337         nodes[0].router.expect_find_route(RouteParameters {
2338                         payment_params: pay_params,
2339                         // Note that the second request here requests the amount we originally failed to send,
2340                         // not the amount remaining on the full payment, which should be changed.
2341                         final_value_msat: 100_000_001,
2342                 }, Ok(route.clone()));
2343
2344         {
2345                 let scorer = chanmon_cfgs[0].scorer.lock().unwrap();
2346                 // The initial send attempt, 2 paths
2347                 scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 10_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2348                 scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 100_000_001, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2349                 // The retry, 2 paths. Ensure that the in-flight HTLC amount is factored in.
2350                 scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_001, inflight_htlc_msat: 10_000, effective_capacity: EffectiveCapacity::Unknown });
2351                 scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2352         }
2353
2354         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2355                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2356         let events = nodes[0].node.get_and_clear_pending_events();
2357         assert_eq!(events.len(), 1);
2358         match events[0] {
2359                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false,
2360                         failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }},
2361                         short_channel_id: Some(expected_scid), .. } =>
2362                 {
2363                         assert_eq!(payment_hash, ev_payment_hash);
2364                         assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
2365                 },
2366                 _ => panic!("Unexpected event"),
2367         }
2368         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2369         assert_eq!(htlc_msgs.len(), 2);
2370         check_added_monitors!(nodes[0], 2);
2371 }
2372
2373 #[test]
2374 fn immediate_retry_on_failure() {
2375         // Tests that we can/will retry immediately after a failure
2376         let chanmon_cfgs = create_chanmon_cfgs(2);
2377         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2378         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2379         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2380
2381         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2382         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2383
2384         let amt_msat = 100_000_001;
2385         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2386         #[cfg(feature = "std")]
2387         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2388         #[cfg(not(feature = "std"))]
2389         let payment_expiry_secs = 60 * 60;
2390         let mut invoice_features = InvoiceFeatures::empty();
2391         invoice_features.set_variable_length_onion_required();
2392         invoice_features.set_payment_secret_required();
2393         invoice_features.set_basic_mpp_optional();
2394         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2395                 .with_expiry_time(payment_expiry_secs as u64)
2396                 .with_bolt11_features(invoice_features).unwrap();
2397         let route_params = RouteParameters {
2398                 payment_params,
2399                 final_value_msat: amt_msat,
2400         };
2401
2402         let chans = nodes[0].node.list_usable_channels();
2403         let mut route = Route {
2404                 paths: vec![
2405                         Path { hops: vec![RouteHop {
2406                                 pubkey: nodes[1].node.get_our_node_id(),
2407                                 node_features: nodes[1].node.node_features(),
2408                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2409                                 channel_features: nodes[1].node.channel_features(),
2410                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2411                                 cltv_expiry_delta: 100,
2412                         }], blinded_tail: None },
2413                 ],
2414                 payment_params: Some(PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)),
2415         };
2416         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2417         // On retry, split the payment across both channels.
2418         route.paths.push(route.paths[0].clone());
2419         route.paths[0].hops[0].short_channel_id = chans[1].short_channel_id.unwrap();
2420         route.paths[0].hops[0].fee_msat = 50_000_000;
2421         route.paths[1].hops[0].fee_msat = 50_000_001;
2422         let mut pay_params = route_params.payment_params.clone();
2423         pay_params.previously_failed_channels.push(chans[0].short_channel_id.unwrap());
2424         nodes[0].router.expect_find_route(RouteParameters {
2425                         payment_params: pay_params, final_value_msat: amt_msat,
2426                 }, Ok(route.clone()));
2427
2428         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2429                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2430         let events = nodes[0].node.get_and_clear_pending_events();
2431         assert_eq!(events.len(), 1);
2432         match events[0] {
2433                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false,
2434                         failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }},
2435                         short_channel_id: Some(expected_scid), .. } =>
2436                 {
2437                         assert_eq!(payment_hash, ev_payment_hash);
2438                         assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
2439                 },
2440                 _ => panic!("Unexpected event"),
2441         }
2442         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2443         assert_eq!(htlc_msgs.len(), 2);
2444         check_added_monitors!(nodes[0], 2);
2445 }
2446
2447 #[test]
2448 fn no_extra_retries_on_back_to_back_fail() {
2449         // In a previous release, we had a race where we may exceed the payment retry count if we
2450         // get two failures in a row with the second indicating that all paths had failed (this field,
2451         // `all_paths_failed`, has since been removed).
2452         // Generally, when we give up trying to retry a payment, we don't know for sure what the
2453         // current state of the ChannelManager event queue is. Specifically, we cannot be sure that
2454         // there are not multiple additional `PaymentPathFailed` or even `PaymentSent` events
2455         // pending which we will see later. Thus, when we previously removed the retry tracking map
2456         // entry after a `all_paths_failed` `PaymentPathFailed` event, we may have dropped the
2457         // retry entry even though more events for the same payment were still pending. This led to
2458         // us retrying a payment again even though we'd already given up on it.
2459         //
2460         // We now have a separate event - `PaymentFailed` which indicates no HTLCs remain and which
2461         // is used to remove the payment retry counter entries instead. This tests for the specific
2462         // excess-retry case while also testing `PaymentFailed` generation.
2463
2464         let chanmon_cfgs = create_chanmon_cfgs(3);
2465         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2466         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2467         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2468
2469         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2470         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2471
2472         let amt_msat = 200_000_000;
2473         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2474         #[cfg(feature = "std")]
2475         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2476         #[cfg(not(feature = "std"))]
2477         let payment_expiry_secs = 60 * 60;
2478         let mut invoice_features = InvoiceFeatures::empty();
2479         invoice_features.set_variable_length_onion_required();
2480         invoice_features.set_payment_secret_required();
2481         invoice_features.set_basic_mpp_optional();
2482         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2483                 .with_expiry_time(payment_expiry_secs as u64)
2484                 .with_bolt11_features(invoice_features).unwrap();
2485         let route_params = RouteParameters {
2486                 payment_params,
2487                 final_value_msat: amt_msat,
2488         };
2489
2490         let mut route = Route {
2491                 paths: vec![
2492                         Path { hops: vec![RouteHop {
2493                                 pubkey: nodes[1].node.get_our_node_id(),
2494                                 node_features: nodes[1].node.node_features(),
2495                                 short_channel_id: chan_1_scid,
2496                                 channel_features: nodes[1].node.channel_features(),
2497                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2498                                 cltv_expiry_delta: 100,
2499                         }, RouteHop {
2500                                 pubkey: nodes[2].node.get_our_node_id(),
2501                                 node_features: nodes[2].node.node_features(),
2502                                 short_channel_id: chan_2_scid,
2503                                 channel_features: nodes[2].node.channel_features(),
2504                                 fee_msat: 100_000_000,
2505                                 cltv_expiry_delta: 100,
2506                         }], blinded_tail: None },
2507                         Path { hops: vec![RouteHop {
2508                                 pubkey: nodes[1].node.get_our_node_id(),
2509                                 node_features: nodes[1].node.node_features(),
2510                                 short_channel_id: chan_1_scid,
2511                                 channel_features: nodes[1].node.channel_features(),
2512                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2513                                 cltv_expiry_delta: 100,
2514                         }, RouteHop {
2515                                 pubkey: nodes[2].node.get_our_node_id(),
2516                                 node_features: nodes[2].node.node_features(),
2517                                 short_channel_id: chan_2_scid,
2518                                 channel_features: nodes[2].node.channel_features(),
2519                                 fee_msat: 100_000_000,
2520                                 cltv_expiry_delta: 100,
2521                         }], blinded_tail: None }
2522                 ],
2523                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2524         };
2525         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2526         let mut second_payment_params = route_params.payment_params.clone();
2527         second_payment_params.previously_failed_channels = vec![chan_2_scid, chan_2_scid];
2528         // On retry, we'll only return one path
2529         route.paths.remove(1);
2530         route.paths[0].hops[1].fee_msat = amt_msat;
2531         nodes[0].router.expect_find_route(RouteParameters {
2532                         payment_params: second_payment_params,
2533                         final_value_msat: amt_msat,
2534                 }, Ok(route.clone()));
2535
2536         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2537                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2538         let htlc_updates = SendEvent::from_node(&nodes[0]);
2539         check_added_monitors!(nodes[0], 1);
2540         assert_eq!(htlc_updates.msgs.len(), 1);
2541
2542         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2543         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2544         check_added_monitors!(nodes[1], 1);
2545         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2546
2547         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2548         check_added_monitors!(nodes[0], 1);
2549         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2550
2551         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2552         check_added_monitors!(nodes[0], 1);
2553         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2554
2555         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2556         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2557         check_added_monitors!(nodes[1], 1);
2558         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2559
2560         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2561         check_added_monitors!(nodes[1], 1);
2562         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2563
2564         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2565         check_added_monitors!(nodes[0], 1);
2566
2567         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2568         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2569         check_added_monitors!(nodes[0], 1);
2570         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2571
2572         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2573         check_added_monitors!(nodes[1], 1);
2574         let bs_second_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2575
2576         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2577         check_added_monitors!(nodes[1], 1);
2578         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2579
2580         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.update_fail_htlcs[0]);
2581         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.commitment_signed);
2582         check_added_monitors!(nodes[0], 1);
2583
2584         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2585         check_added_monitors!(nodes[0], 1);
2586         let (as_third_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2587
2588         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2589         check_added_monitors!(nodes[1], 1);
2590         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2591         check_added_monitors!(nodes[1], 1);
2592         let bs_fourth_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2593
2594         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_fourth_raa);
2595         check_added_monitors!(nodes[0], 1);
2596
2597         // At this point A has sent two HTLCs which both failed due to lack of fee. It now has two
2598         // pending `PaymentPathFailed` events, one with `all_paths_failed` unset, and the second
2599         // with it set.
2600         //
2601         // Previously, we retried payments in an event consumer, which would retry each
2602         // `PaymentPathFailed` individually. In that setup, we had retried the payment in response to
2603         // the first `PaymentPathFailed`, then seen the second `PaymentPathFailed` with
2604         // `all_paths_failed` set and assumed the payment was completely failed. We ultimately fixed it
2605         // by adding the `PaymentFailed` event.
2606         //
2607         // Because we now retry payments as a batch, we simply return a single-path route in the
2608         // second, batched, request, have that fail, ensure the payment was abandoned.
2609         let mut events = nodes[0].node.get_and_clear_pending_events();
2610         assert_eq!(events.len(), 3);
2611         match events[0] {
2612                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2613                         assert_eq!(payment_hash, ev_payment_hash);
2614                         assert_eq!(payment_failed_permanently, false);
2615                 },
2616                 _ => panic!("Unexpected event"),
2617         }
2618         match events[1] {
2619                 Event::PendingHTLCsForwardable { .. } => {},
2620                 _ => panic!("Unexpected event"),
2621         }
2622         match events[2] {
2623                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2624                         assert_eq!(payment_hash, ev_payment_hash);
2625                         assert_eq!(payment_failed_permanently, false);
2626                 },
2627                 _ => panic!("Unexpected event"),
2628         }
2629
2630         nodes[0].node.process_pending_htlc_forwards();
2631         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2632         check_added_monitors!(nodes[0], 1);
2633
2634         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2635         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2636         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2637         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2638         commitment_signed_dance!(nodes[0], nodes[1], &bs_fail_update.commitment_signed, false, true);
2639
2640         let mut events = nodes[0].node.get_and_clear_pending_events();
2641         assert_eq!(events.len(), 2);
2642         match events[0] {
2643                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2644                         assert_eq!(payment_hash, ev_payment_hash);
2645                         assert_eq!(payment_failed_permanently, false);
2646                 },
2647                 _ => panic!("Unexpected event"),
2648         }
2649         match events[1] {
2650                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2651                         assert_eq!(payment_hash, *ev_payment_hash);
2652                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2653                         assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
2654                 },
2655                 _ => panic!("Unexpected event"),
2656         }
2657 }
2658
2659 #[test]
2660 fn test_simple_partial_retry() {
2661         // In the first version of the in-`ChannelManager` payment retries, retries were sent for the
2662         // full amount of the payment, rather than only the missing amount. Here we simply test for
2663         // this by sending a payment with two parts, failing one, and retrying the second. Note that
2664         // `TestRouter` will check that the `RouteParameters` (which contain the amount) matches the
2665         // request.
2666         let chanmon_cfgs = create_chanmon_cfgs(3);
2667         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2668         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2669         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2670
2671         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2672         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2673
2674         let amt_msat = 200_000_000;
2675         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
2676         #[cfg(feature = "std")]
2677         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2678         #[cfg(not(feature = "std"))]
2679         let payment_expiry_secs = 60 * 60;
2680         let mut invoice_features = InvoiceFeatures::empty();
2681         invoice_features.set_variable_length_onion_required();
2682         invoice_features.set_payment_secret_required();
2683         invoice_features.set_basic_mpp_optional();
2684         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2685                 .with_expiry_time(payment_expiry_secs as u64)
2686                 .with_bolt11_features(invoice_features).unwrap();
2687         let route_params = RouteParameters {
2688                 payment_params,
2689                 final_value_msat: amt_msat,
2690         };
2691
2692         let mut route = Route {
2693                 paths: vec![
2694                         Path { hops: vec![RouteHop {
2695                                 pubkey: nodes[1].node.get_our_node_id(),
2696                                 node_features: nodes[1].node.node_features(),
2697                                 short_channel_id: chan_1_scid,
2698                                 channel_features: nodes[1].node.channel_features(),
2699                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2700                                 cltv_expiry_delta: 100,
2701                         }, RouteHop {
2702                                 pubkey: nodes[2].node.get_our_node_id(),
2703                                 node_features: nodes[2].node.node_features(),
2704                                 short_channel_id: chan_2_scid,
2705                                 channel_features: nodes[2].node.channel_features(),
2706                                 fee_msat: 100_000_000,
2707                                 cltv_expiry_delta: 100,
2708                         }], blinded_tail: None },
2709                         Path { hops: vec![RouteHop {
2710                                 pubkey: nodes[1].node.get_our_node_id(),
2711                                 node_features: nodes[1].node.node_features(),
2712                                 short_channel_id: chan_1_scid,
2713                                 channel_features: nodes[1].node.channel_features(),
2714                                 fee_msat: 100_000,
2715                                 cltv_expiry_delta: 100,
2716                         }, RouteHop {
2717                                 pubkey: nodes[2].node.get_our_node_id(),
2718                                 node_features: nodes[2].node.node_features(),
2719                                 short_channel_id: chan_2_scid,
2720                                 channel_features: nodes[2].node.channel_features(),
2721                                 fee_msat: 100_000_000,
2722                                 cltv_expiry_delta: 100,
2723                         }], blinded_tail: None }
2724                 ],
2725                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2726         };
2727         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2728         let mut second_payment_params = route_params.payment_params.clone();
2729         second_payment_params.previously_failed_channels = vec![chan_2_scid];
2730         // On retry, we'll only be asked for one path (or 100k sats)
2731         route.paths.remove(0);
2732         nodes[0].router.expect_find_route(RouteParameters {
2733                         payment_params: second_payment_params,
2734                         final_value_msat: amt_msat / 2,
2735                 }, Ok(route.clone()));
2736
2737         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2738                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2739         let htlc_updates = SendEvent::from_node(&nodes[0]);
2740         check_added_monitors!(nodes[0], 1);
2741         assert_eq!(htlc_updates.msgs.len(), 1);
2742
2743         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2744         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2745         check_added_monitors!(nodes[1], 1);
2746         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2747
2748         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2749         check_added_monitors!(nodes[0], 1);
2750         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2751
2752         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2753         check_added_monitors!(nodes[0], 1);
2754         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2755
2756         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2757         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2758         check_added_monitors!(nodes[1], 1);
2759         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2760
2761         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2762         check_added_monitors!(nodes[1], 1);
2763         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2764
2765         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2766         check_added_monitors!(nodes[0], 1);
2767
2768         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2769         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2770         check_added_monitors!(nodes[0], 1);
2771         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2772
2773         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2774         check_added_monitors!(nodes[1], 1);
2775
2776         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2777         check_added_monitors!(nodes[1], 1);
2778
2779         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2780
2781         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2782         check_added_monitors!(nodes[0], 1);
2783
2784         let mut events = nodes[0].node.get_and_clear_pending_events();
2785         assert_eq!(events.len(), 2);
2786         match events[0] {
2787                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2788                         assert_eq!(payment_hash, ev_payment_hash);
2789                         assert_eq!(payment_failed_permanently, false);
2790                 },
2791                 _ => panic!("Unexpected event"),
2792         }
2793         match events[1] {
2794                 Event::PendingHTLCsForwardable { .. } => {},
2795                 _ => panic!("Unexpected event"),
2796         }
2797
2798         nodes[0].node.process_pending_htlc_forwards();
2799         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2800         check_added_monitors!(nodes[0], 1);
2801
2802         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2803         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2804
2805         expect_pending_htlcs_forwardable!(nodes[1]);
2806         check_added_monitors!(nodes[1], 1);
2807
2808         let bs_forward_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
2809         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[0]);
2810         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[1]);
2811         commitment_signed_dance!(nodes[2], nodes[1], &bs_forward_update.commitment_signed, false);
2812
2813         expect_pending_htlcs_forwardable!(nodes[2]);
2814         expect_payment_claimable!(nodes[2], payment_hash, payment_secret, amt_msat);
2815 }
2816
2817 #[test]
2818 #[cfg(feature = "std")]
2819 fn test_threaded_payment_retries() {
2820         // In the first version of the in-`ChannelManager` payment retries, retries weren't limited to
2821         // a single thread and would happily let multiple threads run retries at the same time. Because
2822         // retries are done by first calculating the amount we need to retry, then dropping the
2823         // relevant lock, then actually sending, we would happily let multiple threads retry the same
2824         // amount at the same time, overpaying our original HTLC!
2825         let chanmon_cfgs = create_chanmon_cfgs(4);
2826         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
2827         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
2828         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
2829
2830         // There is one mitigating guardrail when retrying payments - we can never over-pay by more
2831         // than 10% of the original value. Thus, we want all our retries to be below that. In order to
2832         // keep things simple, we route one HTLC for 0.1% of the payment over channel 1 and the rest
2833         // out over channel 3+4. This will let us ignore 99% of the payment value and deal with only
2834         // our channel.
2835         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2836         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 10_000_000, 0);
2837         let chan_3_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 10_000_000, 0).0.contents.short_channel_id;
2838         let chan_4_scid = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 10_000_000, 0).0.contents.short_channel_id;
2839
2840         let amt_msat = 100_000_000;
2841         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
2842         #[cfg(feature = "std")]
2843         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2844         #[cfg(not(feature = "std"))]
2845         let payment_expiry_secs = 60 * 60;
2846         let mut invoice_features = InvoiceFeatures::empty();
2847         invoice_features.set_variable_length_onion_required();
2848         invoice_features.set_payment_secret_required();
2849         invoice_features.set_basic_mpp_optional();
2850         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2851                 .with_expiry_time(payment_expiry_secs as u64)
2852                 .with_bolt11_features(invoice_features).unwrap();
2853         let mut route_params = RouteParameters {
2854                 payment_params,
2855                 final_value_msat: amt_msat,
2856         };
2857
2858         let mut route = Route {
2859                 paths: vec![
2860                         Path { hops: vec![RouteHop {
2861                                 pubkey: nodes[1].node.get_our_node_id(),
2862                                 node_features: nodes[1].node.node_features(),
2863                                 short_channel_id: chan_1_scid,
2864                                 channel_features: nodes[1].node.channel_features(),
2865                                 fee_msat: 0,
2866                                 cltv_expiry_delta: 100,
2867                         }, RouteHop {
2868                                 pubkey: nodes[3].node.get_our_node_id(),
2869                                 node_features: nodes[2].node.node_features(),
2870                                 short_channel_id: 42, // Set a random SCID which nodes[1] will fail as unknown
2871                                 channel_features: nodes[2].node.channel_features(),
2872                                 fee_msat: amt_msat / 1000,
2873                                 cltv_expiry_delta: 100,
2874                         }], blinded_tail: None },
2875                         Path { hops: vec![RouteHop {
2876                                 pubkey: nodes[2].node.get_our_node_id(),
2877                                 node_features: nodes[2].node.node_features(),
2878                                 short_channel_id: chan_3_scid,
2879                                 channel_features: nodes[2].node.channel_features(),
2880                                 fee_msat: 100_000,
2881                                 cltv_expiry_delta: 100,
2882                         }, RouteHop {
2883                                 pubkey: nodes[3].node.get_our_node_id(),
2884                                 node_features: nodes[3].node.node_features(),
2885                                 short_channel_id: chan_4_scid,
2886                                 channel_features: nodes[3].node.channel_features(),
2887                                 fee_msat: amt_msat - amt_msat / 1000,
2888                                 cltv_expiry_delta: 100,
2889                         }], blinded_tail: None }
2890                 ],
2891                 payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
2892         };
2893         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2894
2895         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2896                 PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(0xdeadbeef)).unwrap();
2897         check_added_monitors!(nodes[0], 2);
2898         let mut send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2899         assert_eq!(send_msg_events.len(), 2);
2900         send_msg_events.retain(|msg|
2901                 if let MessageSendEvent::UpdateHTLCs { node_id, .. } = msg {
2902                         // Drop the commitment update for nodes[2], we can just let that one sit pending
2903                         // forever.
2904                         *node_id == nodes[1].node.get_our_node_id()
2905                 } else { panic!(); }
2906         );
2907
2908         // from here on out, the retry `RouteParameters` amount will be amt/1000
2909         route_params.final_value_msat /= 1000;
2910         route.paths.pop();
2911
2912         let end_time = Instant::now() + Duration::from_secs(1);
2913         macro_rules! thread_body { () => { {
2914                 // We really want std::thread::scope, but its not stable until 1.63. Until then, we get unsafe.
2915                 let node_ref = NodePtr::from_node(&nodes[0]);
2916                 move || {
2917                         let node_a = unsafe { &*node_ref.0 };
2918                         while Instant::now() < end_time {
2919                                 node_a.node.get_and_clear_pending_events(); // wipe the PendingHTLCsForwardable
2920                                 // Ignore if we have any pending events, just always pretend we just got a
2921                                 // PendingHTLCsForwardable
2922                                 node_a.node.process_pending_htlc_forwards();
2923                         }
2924                 }
2925         } } }
2926         let mut threads = Vec::new();
2927         for _ in 0..16 { threads.push(std::thread::spawn(thread_body!())); }
2928
2929         // Back in the main thread, poll pending messages and make sure that we never have more than
2930         // one HTLC pending at a time. Note that the commitment_signed_dance will fail horribly if
2931         // there are HTLC messages shoved in while its running. This allows us to test that we never
2932         // generate an additional update_add_htlc until we've fully failed the first.
2933         let mut previously_failed_channels = Vec::new();
2934         loop {
2935                 assert_eq!(send_msg_events.len(), 1);
2936                 let send_event = SendEvent::from_event(send_msg_events.pop().unwrap());
2937                 assert_eq!(send_event.msgs.len(), 1);
2938
2939                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
2940                 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
2941
2942                 // Note that we only push one route into `expect_find_route` at a time, because that's all
2943                 // the retries (should) need. If the bug is reintroduced "real" routes may be selected, but
2944                 // we should still ultimately fail for the same reason - because we're trying to send too
2945                 // many HTLCs at once.
2946                 let mut new_route_params = route_params.clone();
2947                 previously_failed_channels.push(route.paths[0].hops[1].short_channel_id);
2948                 new_route_params.payment_params.previously_failed_channels = previously_failed_channels.clone();
2949                 route.paths[0].hops[1].short_channel_id += 1;
2950                 nodes[0].router.expect_find_route(new_route_params, Ok(route.clone()));
2951
2952                 let bs_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2953                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_updates.update_fail_htlcs[0]);
2954                 // The "normal" commitment_signed_dance delivers the final RAA and then calls
2955                 // `check_added_monitors` to ensure only the one RAA-generated monitor update was created.
2956                 // This races with our other threads which may generate an add-HTLCs commitment update via
2957                 // `process_pending_htlc_forwards`. Instead, we defer the monitor update check until after
2958                 // *we've* called `process_pending_htlc_forwards` when its guaranteed to have two updates.
2959                 let last_raa = commitment_signed_dance!(nodes[0], nodes[1], bs_fail_updates.commitment_signed, false, true, false, true);
2960                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &last_raa);
2961
2962                 let cur_time = Instant::now();
2963                 if cur_time > end_time {
2964                         for thread in threads.drain(..) { thread.join().unwrap(); }
2965                 }
2966
2967                 // Make sure we have some events to handle when we go around...
2968                 nodes[0].node.get_and_clear_pending_events(); // wipe the PendingHTLCsForwardable
2969                 nodes[0].node.process_pending_htlc_forwards();
2970                 send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2971                 check_added_monitors!(nodes[0], 2);
2972
2973                 if cur_time > end_time {
2974                         break;
2975                 }
2976         }
2977 }
2978
2979 fn do_no_missing_sent_on_midpoint_reload(persist_manager_with_payment: bool) {
2980         // Test that if we reload in the middle of an HTLC claim commitment signed dance we'll still
2981         // receive the PaymentSent event even if the ChannelManager had no idea about the payment when
2982         // it was last persisted.
2983         let chanmon_cfgs = create_chanmon_cfgs(2);
2984         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2985         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2986         let (persister_a, persister_b, persister_c);
2987         let (chain_monitor_a, chain_monitor_b, chain_monitor_c);
2988         let (nodes_0_deserialized, nodes_0_deserialized_b, nodes_0_deserialized_c);
2989         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2990
2991         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
2992
2993         let mut nodes_0_serialized = Vec::new();
2994         if !persist_manager_with_payment {
2995                 nodes_0_serialized = nodes[0].node.encode();
2996         }
2997
2998         let (our_payment_preimage, our_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 1_000_000);
2999
3000         if persist_manager_with_payment {
3001                 nodes_0_serialized = nodes[0].node.encode();
3002         }
3003
3004         nodes[1].node.claim_funds(our_payment_preimage);
3005         check_added_monitors!(nodes[1], 1);
3006         expect_payment_claimed!(nodes[1], our_payment_hash, 1_000_000);
3007
3008         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3009         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3010         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
3011         check_added_monitors!(nodes[0], 1);
3012
3013         // The ChannelMonitor should always be the latest version, as we're required to persist it
3014         // during the commitment signed handling.
3015         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3016         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister_a, chain_monitor_a, nodes_0_deserialized);
3017
3018         let events = nodes[0].node.get_and_clear_pending_events();
3019         assert_eq!(events.len(), 2);
3020         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[0] {} else { panic!(); }
3021         if let Event::PaymentSent { payment_preimage, .. } = events[1] { assert_eq!(payment_preimage, our_payment_preimage); } else { panic!(); }
3022         // Note that we don't get a PaymentPathSuccessful here as we leave the HTLC pending to avoid
3023         // the double-claim that would otherwise appear at the end of this test.
3024         nodes[0].node.timer_tick_occurred();
3025         let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3026         assert_eq!(as_broadcasted_txn.len(), 1);
3027
3028         // Ensure that, even after some time, if we restart we still include *something* in the current
3029         // `ChannelManager` which prevents a `PaymentFailed` when we restart even if pending resolved
3030         // payments have since been timed out thanks to `IDEMPOTENCY_TIMEOUT_TICKS`.
3031         // A naive implementation of the fix here would wipe the pending payments set, causing a
3032         // failure event when we restart.
3033         for _ in 0..(IDEMPOTENCY_TIMEOUT_TICKS * 2) { nodes[0].node.timer_tick_occurred(); }
3034
3035         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3036         reload_node!(nodes[0], test_default_channel_config(), &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister_b, chain_monitor_b, nodes_0_deserialized_b);
3037         let events = nodes[0].node.get_and_clear_pending_events();
3038         assert!(events.is_empty());
3039
3040         // Ensure that we don't generate any further events even after the channel-closing commitment
3041         // transaction is confirmed on-chain.
3042         confirm_transaction(&nodes[0], &as_broadcasted_txn[0]);
3043         for _ in 0..(IDEMPOTENCY_TIMEOUT_TICKS * 2) { nodes[0].node.timer_tick_occurred(); }
3044
3045         let events = nodes[0].node.get_and_clear_pending_events();
3046         assert!(events.is_empty());
3047
3048         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3049         reload_node!(nodes[0], test_default_channel_config(), &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister_c, chain_monitor_c, nodes_0_deserialized_c);
3050         let events = nodes[0].node.get_and_clear_pending_events();
3051         assert!(events.is_empty());
3052         check_added_monitors(&nodes[0], 1);
3053 }
3054
3055 #[test]
3056 fn no_missing_sent_on_midpoint_reload() {
3057         do_no_missing_sent_on_midpoint_reload(false);
3058         do_no_missing_sent_on_midpoint_reload(true);
3059 }
3060
3061 fn do_claim_from_closed_chan(fail_payment: bool) {
3062         // Previously, LDK would refuse to claim a payment if a channel on which the payment was
3063         // received had been closed between when the HTLC was received and when we went to claim it.
3064         // This makes sense in the payment case - why pay an on-chain fee to claim the HTLC when
3065         // presumably the sender may retry later. Long ago it also reduced total code in the claim
3066         // pipeline.
3067         //
3068         // However, this doesn't make sense if you're trying to do an atomic swap or some other
3069         // protocol that requires atomicity with some other action - if your money got claimed
3070         // elsewhere you need to be able to claim the HTLC in lightning no matter what. Further, this
3071         // is an over-optimization - there should be a very, very low likelihood that a channel closes
3072         // between when we receive the last HTLC for a payment and the user goes to claim the payment.
3073         // Since we now have code to handle this anyway we should allow it.
3074
3075         // Build 4 nodes and send an MPP payment across two paths. By building a route manually set the
3076         // CLTVs on the paths to different value resulting in a different claim deadline.
3077         let chanmon_cfgs = create_chanmon_cfgs(4);
3078         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3079         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3080         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3081
3082         create_announced_chan_between_nodes(&nodes, 0, 1);
3083         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
3084         let chan_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
3085         create_announced_chan_between_nodes(&nodes, 2, 3);
3086
3087         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3]);
3088         let mut route_params = RouteParameters {
3089                 payment_params: PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3090                         .with_bolt11_features(nodes[1].node.invoice_features()).unwrap(),
3091                 final_value_msat: 10_000_000,
3092         };
3093         let mut route = nodes[0].router.find_route(&nodes[0].node.get_our_node_id(), &route_params,
3094                 None, &nodes[0].node.compute_inflight_htlcs()).unwrap();
3095         // Make sure the route is ordered as the B->D path before C->D
3096         route.paths.sort_by(|a, _| if a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
3097                 std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater });
3098
3099         // Note that we add an extra 1 in the send pipeline to compensate for any blocks found while
3100         // the HTLC is being relayed.
3101         route.paths[0].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 8;
3102         route.paths[1].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 12;
3103         let final_cltv = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 8 + 1;
3104
3105         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3106         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3107                 PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(1)).unwrap();
3108         check_added_monitors(&nodes[0], 2);
3109         let mut send_msgs = nodes[0].node.get_and_clear_pending_msg_events();
3110         send_msgs.sort_by(|a, _| {
3111                 let a_node_id =
3112                         if let MessageSendEvent::UpdateHTLCs { node_id, .. } = a { node_id } else { panic!() };
3113                 let node_b_id = nodes[1].node.get_our_node_id();
3114                 if *a_node_id == node_b_id { std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater }
3115         });
3116
3117         assert_eq!(send_msgs.len(), 2);
3118         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 10_000_000,
3119                 payment_hash, Some(payment_secret), send_msgs.remove(0), false, None);
3120         let receive_event = pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 10_000_000,
3121                 payment_hash, Some(payment_secret), send_msgs.remove(0), true, None);
3122
3123         match receive_event.unwrap() {
3124                 Event::PaymentClaimable { claim_deadline, .. } => {
3125                         assert_eq!(claim_deadline.unwrap(), final_cltv - HTLC_FAIL_BACK_BUFFER);
3126                 },
3127                 _ => panic!(),
3128         }
3129
3130         // Ensure that the claim_deadline is correct, with the payment failing at exactly the given
3131         // height.
3132         connect_blocks(&nodes[3], final_cltv - HTLC_FAIL_BACK_BUFFER - nodes[3].best_block_info().1
3133                 - if fail_payment { 0 } else { 2 });
3134         if fail_payment {
3135                 // We fail the HTLC on the A->B->D path first as it expires 4 blocks earlier. We go ahead
3136                 // and expire both immediately, though, by connecting another 4 blocks.
3137                 let reason = HTLCDestination::FailedPayment { payment_hash };
3138                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason.clone()]);
3139                 connect_blocks(&nodes[3], 4);
3140                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason]);
3141                 pass_failed_payment_back(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
3142         } else {
3143                 nodes[1].node.force_close_broadcasting_latest_txn(&chan_bd, &nodes[3].node.get_our_node_id()).unwrap();
3144                 check_closed_event(&nodes[1], 1, ClosureReason::HolderForceClosed, false);
3145                 check_closed_broadcast(&nodes[1], 1, true);
3146                 let bs_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3147                 assert_eq!(bs_tx.len(), 1);
3148
3149                 mine_transaction(&nodes[3], &bs_tx[0]);
3150                 check_added_monitors(&nodes[3], 1);
3151                 check_closed_broadcast(&nodes[3], 1, true);
3152                 check_closed_event(&nodes[3], 1, ClosureReason::CommitmentTxConfirmed, false);
3153
3154                 nodes[3].node.claim_funds(payment_preimage);
3155                 check_added_monitors(&nodes[3], 2);
3156                 expect_payment_claimed!(nodes[3], payment_hash, 10_000_000);
3157
3158                 let ds_tx = nodes[3].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3159                 assert_eq!(ds_tx.len(), 1);
3160                 check_spends!(&ds_tx[0], &bs_tx[0]);
3161
3162                 mine_transactions(&nodes[1], &[&bs_tx[0], &ds_tx[0]]);
3163                 check_added_monitors(&nodes[1], 1);
3164                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[3], Some(1000), false, true);
3165
3166                 let bs_claims = nodes[1].node.get_and_clear_pending_msg_events();
3167                 check_added_monitors(&nodes[1], 1);
3168                 assert_eq!(bs_claims.len(), 1);
3169                 if let MessageSendEvent::UpdateHTLCs { updates, .. } = &bs_claims[0] {
3170                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3171                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
3172                 } else { panic!(); }
3173
3174                 expect_payment_sent!(nodes[0], payment_preimage);
3175
3176                 let ds_claim_msgs = nodes[3].node.get_and_clear_pending_msg_events();
3177                 assert_eq!(ds_claim_msgs.len(), 1);
3178                 let cs_claim_msgs = if let MessageSendEvent::UpdateHTLCs { updates, .. } = &ds_claim_msgs[0] {
3179                         nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3180                         let cs_claim_msgs = nodes[2].node.get_and_clear_pending_msg_events();
3181                         check_added_monitors(&nodes[2], 1);
3182                         commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
3183                         expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
3184                         cs_claim_msgs
3185                 } else { panic!(); };
3186
3187                 assert_eq!(cs_claim_msgs.len(), 1);
3188                 if let MessageSendEvent::UpdateHTLCs { updates, .. } = &cs_claim_msgs[0] {
3189                         nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3190                         commitment_signed_dance!(nodes[0], nodes[2], updates.commitment_signed, false, true);
3191                 } else { panic!(); }
3192
3193                 expect_payment_path_successful!(nodes[0]);
3194         }
3195 }
3196
3197 #[test]
3198 fn claim_from_closed_chan() {
3199         do_claim_from_closed_chan(true);
3200         do_claim_from_closed_chan(false);
3201 }
3202
3203 fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) {
3204         // Check that a payment metadata received on one HTLC that doesn't match the one received on
3205         // another results in the HTLC being rejected.
3206         //
3207         // We first set up a diamond shaped network, allowing us to split a payment into two HTLCs, the
3208         // first of which we'll deliver and the second of which we'll fail and then re-send with
3209         // modified payment metadata, which will in turn result in it being failed by the recipient.
3210         let chanmon_cfgs = create_chanmon_cfgs(4);
3211         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3212         let mut config = test_default_channel_config();
3213         config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 50;
3214         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, Some(config), Some(config), Some(config)]);
3215
3216         let persister;
3217         let new_chain_monitor;
3218         let nodes_0_deserialized;
3219
3220         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3221
3222         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
3223         let chan_id_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
3224         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
3225         let chan_id_cd = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
3226
3227         // Pay more than half of each channel's max, requiring MPP
3228         let amt_msat = 750_000_000;
3229         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3], Some(amt_msat));
3230         let payment_id = PaymentId(payment_hash.0);
3231         let payment_metadata = vec![44, 49, 52, 142];
3232
3233         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3234                 .with_bolt11_features(nodes[1].node.invoice_features()).unwrap();
3235         let mut route_params = RouteParameters {
3236                 payment_params,
3237                 final_value_msat: amt_msat,
3238         };
3239
3240         // Send the MPP payment, delivering the updated commitment state to nodes[1].
3241         nodes[0].node.send_payment(payment_hash, RecipientOnionFields {
3242                         payment_secret: Some(payment_secret), payment_metadata: Some(payment_metadata),
3243                 }, payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
3244         check_added_monitors!(nodes[0], 2);
3245
3246         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
3247         assert_eq!(send_events.len(), 2);
3248         let first_send = SendEvent::from_event(send_events.pop().unwrap());
3249         let second_send = SendEvent::from_event(send_events.pop().unwrap());
3250
3251         let (b_recv_ev, c_recv_ev) = if first_send.node_id == nodes[1].node.get_our_node_id() {
3252                 (&first_send, &second_send)
3253         } else {
3254                 (&second_send, &first_send)
3255         };
3256         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &b_recv_ev.msgs[0]);
3257         commitment_signed_dance!(nodes[1], nodes[0], b_recv_ev.commitment_msg, false, true);
3258
3259         expect_pending_htlcs_forwardable!(nodes[1]);
3260         check_added_monitors(&nodes[1], 1);
3261         let b_forward_ev = SendEvent::from_node(&nodes[1]);
3262         nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &b_forward_ev.msgs[0]);
3263         commitment_signed_dance!(nodes[3], nodes[1], b_forward_ev.commitment_msg, false, true);
3264
3265         expect_pending_htlcs_forwardable!(nodes[3]);
3266
3267         // Before delivering the second MPP HTLC to nodes[2], disconnect nodes[2] and nodes[3], which
3268         // will result in nodes[2] failing the HTLC back.
3269         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
3270         nodes[3].node.peer_disconnected(&nodes[2].node.get_our_node_id());
3271
3272         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &c_recv_ev.msgs[0]);
3273         commitment_signed_dance!(nodes[2], nodes[0], c_recv_ev.commitment_msg, false, true);
3274
3275         let cs_fail = get_htlc_update_msgs(&nodes[2], &nodes[0].node.get_our_node_id());
3276         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail.update_fail_htlcs[0]);
3277         commitment_signed_dance!(nodes[0], nodes[2], cs_fail.commitment_signed, false, true);
3278
3279         let payment_fail_retryable_evs = nodes[0].node.get_and_clear_pending_events();
3280         assert_eq!(payment_fail_retryable_evs.len(), 2);
3281         if let Event::PaymentPathFailed { .. } = payment_fail_retryable_evs[0] {} else { panic!(); }
3282         if let Event::PendingHTLCsForwardable { .. } = payment_fail_retryable_evs[1] {} else { panic!(); }
3283
3284         // Before we allow the HTLC to be retried, optionally change the payment_metadata we have
3285         // stored for our payment.
3286         if do_modify {
3287                 nodes[0].node.test_set_payment_metadata(payment_id, Some(Vec::new()));
3288         }
3289
3290         // Optionally reload nodes[3] to check that the payment_metadata is properly serialized with
3291         // the payment state.
3292         if do_reload {
3293                 let mon_bd = get_monitor!(nodes[3], chan_id_bd).encode();
3294                 let mon_cd = get_monitor!(nodes[3], chan_id_cd).encode();
3295                 reload_node!(nodes[3], config, &nodes[3].node.encode(), &[&mon_bd, &mon_cd],
3296                         persister, new_chain_monitor, nodes_0_deserialized);
3297                 nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
3298                 reconnect_nodes(&nodes[1], &nodes[3], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
3299         }
3300         reconnect_nodes(&nodes[2], &nodes[3], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
3301
3302         // Create a new channel between C and D as A will refuse to retry on the existing one because
3303         // it just failed.
3304         let chan_id_cd_2 = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
3305
3306         // Now retry the failed HTLC.
3307         nodes[0].node.process_pending_htlc_forwards();
3308         check_added_monitors(&nodes[0], 1);
3309         let as_resend = SendEvent::from_node(&nodes[0]);
3310         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_resend.msgs[0]);
3311         commitment_signed_dance!(nodes[2], nodes[0], as_resend.commitment_msg, false, true);
3312
3313         expect_pending_htlcs_forwardable!(nodes[2]);
3314         check_added_monitors(&nodes[2], 1);
3315         let cs_forward = SendEvent::from_node(&nodes[2]);
3316         nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &cs_forward.msgs[0]);
3317         commitment_signed_dance!(nodes[3], nodes[2], cs_forward.commitment_msg, false, true);
3318
3319         // Finally, check that nodes[3] does the correct thing - either accepting the payment or, if
3320         // the payment metadata was modified, failing only the one modified HTLC and retaining the
3321         // other.
3322         if do_modify {
3323                 expect_pending_htlcs_forwardable_ignore!(nodes[3]);
3324                 nodes[3].node.process_pending_htlc_forwards();
3325                 expect_pending_htlcs_forwardable_conditions(nodes[3].node.get_and_clear_pending_events(),
3326                         &[HTLCDestination::FailedPayment {payment_hash}]);
3327                 nodes[3].node.process_pending_htlc_forwards();
3328
3329                 check_added_monitors(&nodes[3], 1);
3330                 let ds_fail = get_htlc_update_msgs(&nodes[3], &nodes[2].node.get_our_node_id());
3331
3332                 nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &ds_fail.update_fail_htlcs[0]);
3333                 commitment_signed_dance!(nodes[2], nodes[3], ds_fail.commitment_signed, false, true);
3334                 expect_pending_htlcs_forwardable_conditions(nodes[2].node.get_and_clear_pending_events(),
3335                         &[HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_id_cd_2 }]);
3336         } else {
3337                 expect_pending_htlcs_forwardable!(nodes[3]);
3338                 expect_payment_claimable!(nodes[3], payment_hash, payment_secret, amt_msat);
3339                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
3340         }
3341 }
3342
3343 #[test]
3344 fn test_payment_metadata_consistency() {
3345         do_test_payment_metadata_consistency(true, true);
3346         do_test_payment_metadata_consistency(true, false);
3347         do_test_payment_metadata_consistency(false, true);
3348         do_test_payment_metadata_consistency(false, false);
3349 }