_ Git - rust-lightning/blob - lightning/src/ln/payment_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Tests that test the payment retry logic in ChannelManager, including various edge-cases around
  11 //! serialization ordering between ChannelManager/ChannelMonitors and ensuring we can still retry
  12 //! payments thereafter.
  13
  14 use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Listen, Watch};
  15 use crate::chain::channelmonitor::{ANTI_REORG_DELAY, HTLC_FAIL_BACK_BUFFER, LATENCY_GRACE_PERIOD_BLOCKS};
  16 use crate::sign::EntropySource;
  17 use crate::chain::transaction::OutPoint;
  18 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentFailureReason, PaymentPurpose};
  19 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
  20 use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, RecentPaymentDetails, RecipientOnionFields, HTLCForwardInfo, PendingHTLCRouting, PendingAddHTLCInfo};
  21 use crate::ln::features::Bolt11InvoiceFeatures;
  22 use crate::ln::{msgs, ChannelId, PaymentSecret, PaymentPreimage};
  23 use crate::ln::msgs::ChannelMessageHandler;
  24 use crate::ln::outbound_payment::{IDEMPOTENCY_TIMEOUT_TICKS, Retry};
  25 use crate::routing::gossip::{EffectiveCapacity, RoutingFees};
  26 use crate::routing::router::{get_route, Path, PaymentParameters, Route, Router, RouteHint, RouteHintHop, RouteHop, RouteParameters, find_route};
  27 use crate::routing::scoring::ChannelUsage;
  28 use crate::util::config::UserConfig;
  29 use crate::util::test_utils;
  30 use crate::util::errors::APIError;
  31 use crate::util::ser::Writeable;
  32 use crate::util::string::UntrustedString;
  33
  34 use bitcoin::network::constants::Network;
  35
  36 use crate::prelude::*;
  37
  38 use crate::ln::functional_test_utils::*;
  39 use crate::routing::gossip::NodeId;
  40 #[cfg(feature = "std")]
  41 use {
  42         crate::util::time::tests::SinceEpoch,
  43         std::time::{SystemTime, Instant, Duration}
  44 };
  45
  46 #[test]
  47 fn mpp_failure() {
  48         let chanmon_cfgs = create_chanmon_cfgs(4);
  49         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
  50         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
  51         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
  52
  53         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
  54         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
  55         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
  56         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3).0.contents.short_channel_id;
  57
  58         let (mut route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
  59         let path = route.paths[0].clone();
  60         route.paths.push(path);
  61         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
  62         route.paths[0].hops[0].short_channel_id = chan_1_id;
  63         route.paths[0].hops[1].short_channel_id = chan_3_id;
  64         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
  65         route.paths[1].hops[0].short_channel_id = chan_2_id;
  66         route.paths[1].hops[1].short_channel_id = chan_4_id;
  67         send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
  68         fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
  69 }
  70
  71 #[test]
  72 fn mpp_retry() {
  73         let chanmon_cfgs = create_chanmon_cfgs(4);
  74         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
  75         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
  76         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
  77
  78         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
  79         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
  80         let (chan_3_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
  81         let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes(&nodes, 3, 2);
  82         // Rebalance
  83         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
  84
  85         let amt_msat = 1_000_000;
  86         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], amt_msat);
  87         let path = route.paths[0].clone();
  88         route.paths.push(path);
  89         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
  90         route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
  91         route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
  92         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
  93         route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
  94         route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
  95
  96         // Initiate the MPP payment.
  97         let payment_id = PaymentId(payment_hash.0);
  98         let mut route_params = route.route_params.clone().unwrap();
  99
 100         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
 101         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 102                 payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
 103         check_added_monitors!(nodes[0], 2); // one monitor per path
 104         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 105         assert_eq!(events.len(), 2);
 106
 107         // Pass half of the payment along the success path.
 108         let success_path_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 109         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), success_path_msgs, false, None);
 110
 111         // Add the HTLC along the first hop.
 112         let fail_path_msgs_1 = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 113         let (update_add, commitment_signed) = match fail_path_msgs_1 {
 114                 MessageSendEvent::UpdateHTLCs { node_id: _, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 115                         assert_eq!(update_add_htlcs.len(), 1);
 116                         assert!(update_fail_htlcs.is_empty());
 117                         assert!(update_fulfill_htlcs.is_empty());
 118                         assert!(update_fail_malformed_htlcs.is_empty());
 119                         assert!(update_fee.is_none());
 120                         (update_add_htlcs[0].clone(), commitment_signed.clone())
 121                 },
 122                 _ => panic!("Unexpected event"),
 123         };
 124         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
 125         commitment_signed_dance!(nodes[2], nodes[0], commitment_signed, false);
 126
 127         // Attempt to forward the payment and complete the 2nd path's failure.
 128         expect_pending_htlcs_forwardable!(&nodes[2]);
 129         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id }]);
 130         let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 131         assert!(htlc_updates.update_add_htlcs.is_empty());
 132         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 133         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 134         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 135         check_added_monitors!(nodes[2], 1);
 136         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 137         commitment_signed_dance!(nodes[0], nodes[2], htlc_updates.commitment_signed, false);
 138         let mut events = nodes[0].node.get_and_clear_pending_events();
 139         match events[1] {
 140                 Event::PendingHTLCsForwardable { .. } => {},
 141                 _ => panic!("Unexpected event")
 142         }
 143         events.remove(1);
 144         expect_payment_failed_conditions_event(events, payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 145
 146         // Rebalance the channel so the second half of the payment can succeed.
 147         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 148
 149         // Retry the second half of the payment and make sure it succeeds.
 150         route.paths.remove(0);
 151         route_params.final_value_msat = 1_000_000;
 152         route_params.payment_params.previously_failed_channels.push(chan_4_update.contents.short_channel_id);
 153         nodes[0].router.expect_find_route(route_params, Ok(route));
 154         nodes[0].node.process_pending_htlc_forwards();
 155         check_added_monitors!(nodes[0], 1);
 156         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 157         assert_eq!(events.len(), 1);
 158         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 159         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 160 }
 161
 162 fn do_mpp_receive_timeout(send_partial_mpp: bool) {
 163         let chanmon_cfgs = create_chanmon_cfgs(4);
 164         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 165         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 166         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 167
 168         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 169         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 170         let (chan_3_update, _, chan_3_id, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 171         let (chan_4_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 172
 173         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 100_000);
 174         let path = route.paths[0].clone();
 175         route.paths.push(path);
 176         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
 177         route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
 178         route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
 179         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
 180         route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
 181         route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
 182
 183         // Initiate the MPP payment.
 184         nodes[0].node.send_payment_with_route(&route, payment_hash,
 185                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 186         check_added_monitors!(nodes[0], 2); // one monitor per path
 187         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 188         assert_eq!(events.len(), 2);
 189
 190         // Pass half of the payment along the first path.
 191         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 192         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_1_msgs, false, None);
 193
 194         if send_partial_mpp {
 195                 // Time out the partial MPP
 196                 for _ in 0..MPP_TIMEOUT_TICKS {
 197                         nodes[3].node.timer_tick_occurred();
 198                 }
 199
 200                 // Failed HTLC from node 3 -> 1
 201                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 202                 let htlc_fail_updates_3_1 = get_htlc_update_msgs!(nodes[3], nodes[1].node.get_our_node_id());
 203                 assert_eq!(htlc_fail_updates_3_1.update_fail_htlcs.len(), 1);
 204                 nodes[1].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &htlc_fail_updates_3_1.update_fail_htlcs[0]);
 205                 check_added_monitors!(nodes[3], 1);
 206                 commitment_signed_dance!(nodes[1], nodes[3], htlc_fail_updates_3_1.commitment_signed, false);
 207
 208                 // Failed HTLC from node 1 -> 0
 209                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_3_id }]);
 210                 let htlc_fail_updates_1_0 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 211                 assert_eq!(htlc_fail_updates_1_0.update_fail_htlcs.len(), 1);
 212                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates_1_0.update_fail_htlcs[0]);
 213                 check_added_monitors!(nodes[1], 1);
 214                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates_1_0.commitment_signed, false);
 215
 216                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain().expected_htlc_error_data(23, &[][..]));
 217         } else {
 218                 // Pass half of the payment along the second path.
 219                 let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 220                 pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_2_msgs, true, None);
 221
 222                 // Even after MPP_TIMEOUT_TICKS we should not timeout the MPP if we have all the parts
 223                 for _ in 0..MPP_TIMEOUT_TICKS {
 224                         nodes[3].node.timer_tick_occurred();
 225                 }
 226
 227                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 228         }
 229 }
 230
 231 #[test]
 232 fn mpp_receive_timeout() {
 233         do_mpp_receive_timeout(true);
 234         do_mpp_receive_timeout(false);
 235 }
 236
 237 #[test]
 238 fn test_keysend_payments() {
 239         do_test_keysend_payments(false, false);
 240         do_test_keysend_payments(false, true);
 241         do_test_keysend_payments(true, false);
 242         do_test_keysend_payments(true, true);
 243 }
 244
 245 fn do_test_keysend_payments(public_node: bool, with_retry: bool) {
 246         let chanmon_cfgs = create_chanmon_cfgs(2);
 247         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 248         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 249         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 250
 251         if public_node {
 252                 create_announced_chan_between_nodes(&nodes, 0, 1);
 253         } else {
 254                 create_chan_between_nodes(&nodes[0], &nodes[1]);
 255         }
 256         let payer_pubkey = nodes[0].node.get_our_node_id();
 257         let payee_pubkey = nodes[1].node.get_our_node_id();
 258         let route_params = RouteParameters::from_payment_params_and_value(
 259                 PaymentParameters::for_keysend(payee_pubkey, 40, false), 10000);
 260
 261         let network_graph = nodes[0].network_graph.clone();
 262         let channels = nodes[0].node.list_usable_channels();
 263         let first_hops = channels.iter().collect::<Vec<_>>();
 264         let first_hops = if public_node { None } else { Some(first_hops.as_slice()) };
 265
 266         let scorer = test_utils::TestScorer::new();
 267         let random_seed_bytes = chanmon_cfgs[1].keys_manager.get_secure_random_bytes();
 268         let route = find_route(
 269                 &payer_pubkey, &route_params, &network_graph, first_hops,
 270                 nodes[0].logger, &scorer, &(), &random_seed_bytes
 271         ).unwrap();
 272
 273         {
 274                 let test_preimage = PaymentPreimage([42; 32]);
 275                 if with_retry {
 276                         nodes[0].node.send_spontaneous_payment_with_retry(Some(test_preimage),
 277                                 RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0),
 278                                 route_params, Retry::Attempts(1)).unwrap()
 279                 } else {
 280                         nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage),
 281                                 RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0)).unwrap()
 282                 };
 283         }
 284         check_added_monitors!(nodes[0], 1);
 285         let send_event = SendEvent::from_node(&nodes[0]);
 286         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 287         do_commitment_signed_dance(&nodes[1], &nodes[0], &send_event.commitment_msg, false, false);
 288         expect_pending_htlcs_forwardable!(nodes[1]);
 289         // Previously, a refactor caused us to stop including the payment preimage in the onion which
 290         // is sent as a part of keysend payments. Thus, to be extra careful here, we scope the preimage
 291         // above to demonstrate that we have no way to get the preimage at this point except by
 292         // extracting it from the onion nodes[1] received.
 293         let event = nodes[1].node.get_and_clear_pending_events();
 294         assert_eq!(event.len(), 1);
 295         if let Event::PaymentClaimable { purpose: PaymentPurpose::SpontaneousPayment(preimage), .. } = event[0] {
 296                 claim_payment(&nodes[0], &[&nodes[1]], preimage);
 297         } else { panic!(); }
 298 }
 299
 300 #[test]
 301 fn test_mpp_keysend() {
 302         let mut mpp_keysend_config = test_default_channel_config();
 303         mpp_keysend_config.accept_mpp_keysend = true;
 304         let chanmon_cfgs = create_chanmon_cfgs(4);
 305         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 306         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, Some(mpp_keysend_config)]);
 307         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 308
 309         create_announced_chan_between_nodes(&nodes, 0, 1);
 310         create_announced_chan_between_nodes(&nodes, 0, 2);
 311         create_announced_chan_between_nodes(&nodes, 1, 3);
 312         create_announced_chan_between_nodes(&nodes, 2, 3);
 313         let network_graph = nodes[0].network_graph.clone();
 314
 315         let payer_pubkey = nodes[0].node.get_our_node_id();
 316         let payee_pubkey = nodes[3].node.get_our_node_id();
 317         let recv_value = 15_000_000;
 318         let route_params = RouteParameters::from_payment_params_and_value(
 319                 PaymentParameters::for_keysend(payee_pubkey, 40, true), recv_value);
 320         let scorer = test_utils::TestScorer::new();
 321         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
 322         let route = find_route(&payer_pubkey, &route_params, &network_graph, None, nodes[0].logger,
 323                 &scorer, &(), &random_seed_bytes).unwrap();
 324
 325         let payment_preimage = PaymentPreimage([42; 32]);
 326         let payment_secret = PaymentSecret(payment_preimage.0);
 327         let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage),
 328                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_preimage.0)).unwrap();
 329         check_added_monitors!(nodes[0], 2);
 330
 331         let expected_route: &[&[&Node]] = &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]];
 332         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 333         assert_eq!(events.len(), 2);
 334
 335         let ev = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 336         pass_along_path(&nodes[0], expected_route[0], recv_value, payment_hash.clone(),
 337                 Some(payment_secret), ev.clone(), false, Some(payment_preimage));
 338
 339         let ev = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 340         pass_along_path(&nodes[0], expected_route[1], recv_value, payment_hash.clone(),
 341                 Some(payment_secret), ev.clone(), true, Some(payment_preimage));
 342         claim_payment_along_route(&nodes[0], expected_route, false, payment_preimage);
 343 }
 344
 345 #[test]
 346 fn test_reject_mpp_keysend_htlc() {
 347         // This test enforces that we reject MPP keysend HTLCs if our config states we don't support
 348         // MPP keysend. When receiving a payment, if we don't support MPP keysend we'll reject the
 349         // payment if it's keysend and has a payment secret, never reaching our payment validation
 350         // logic. To check that we enforce rejecting MPP keysends in our payment logic, here we send
 351         // keysend payments without payment secrets, then modify them by adding payment secrets in the
 352         // final node in between receiving the HTLCs and actually processing them.
 353         let mut reject_mpp_keysend_cfg = test_default_channel_config();
 354         reject_mpp_keysend_cfg.accept_mpp_keysend = false;
 355
 356         let chanmon_cfgs = create_chanmon_cfgs(4);
 357         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 358         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, Some(reject_mpp_keysend_cfg)]);
 359         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 360         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
 361         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
 362         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
 363         let (update_a, _, chan_4_channel_id, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 364         let chan_4_id = update_a.contents.short_channel_id;
 365         let amount = 40_000;
 366         let (mut route, payment_hash, payment_preimage, _) = get_route_and_payment_hash!(nodes[0], nodes[3], amount);
 367
 368         // Pay along nodes[1]
 369         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
 370         route.paths[0].hops[0].short_channel_id = chan_1_id;
 371         route.paths[0].hops[1].short_channel_id = chan_3_id;
 372
 373         let payment_id_0 = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 374         nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), RecipientOnionFields::spontaneous_empty(), payment_id_0).unwrap();
 375         check_added_monitors!(nodes[0], 1);
 376
 377         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 378         let update_add_0 = update_0.update_add_htlcs[0].clone();
 379         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_0);
 380         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
 381         expect_pending_htlcs_forwardable!(nodes[1]);
 382
 383         check_added_monitors!(&nodes[1], 1);
 384         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[3].node.get_our_node_id());
 385         let update_add_1 = update_1.update_add_htlcs[0].clone();
 386         nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &update_add_1);
 387         commitment_signed_dance!(nodes[3], nodes[1], update_1.commitment_signed, false, true);
 388
 389         assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
 390         for (_, pending_forwards) in nodes[3].node.forward_htlcs.lock().unwrap().iter_mut() {
 391                 for f in pending_forwards.iter_mut() {
 392                         match f {
 393                                 &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) => {
 394                                         match forward_info.routing {
 395                                                 PendingHTLCRouting::ReceiveKeysend { ref mut payment_data, .. } => {
 396                                                         *payment_data = Some(msgs::FinalOnionHopData {
 397                                                                 payment_secret: PaymentSecret([42; 32]),
 398                                                                 total_msat: amount * 2,
 399                                                         });
 400                                                 },
 401                                                 _ => panic!("Expected PendingHTLCRouting::ReceiveKeysend"),
 402                                         }
 403                                 },
 404                                 _ => {},
 405                         }
 406                 }
 407         }
 408         expect_pending_htlcs_forwardable!(nodes[3]);
 409
 410         // Pay along nodes[2]
 411         route.paths[0].hops[0].pubkey = nodes[2].node.get_our_node_id();
 412         route.paths[0].hops[0].short_channel_id = chan_2_id;
 413         route.paths[0].hops[1].short_channel_id = chan_4_id;
 414
 415         let payment_id_1 = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 416         nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), RecipientOnionFields::spontaneous_empty(), payment_id_1).unwrap();
 417         check_added_monitors!(nodes[0], 1);
 418
 419         let update_2 = get_htlc_update_msgs!(nodes[0], nodes[2].node.get_our_node_id());
 420         let update_add_2 = update_2.update_add_htlcs[0].clone();
 421         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_2);
 422         commitment_signed_dance!(nodes[2], nodes[0], &update_2.commitment_signed, false, true);
 423         expect_pending_htlcs_forwardable!(nodes[2]);
 424
 425         check_added_monitors!(&nodes[2], 1);
 426         let update_3 = get_htlc_update_msgs!(nodes[2], nodes[3].node.get_our_node_id());
 427         let update_add_3 = update_3.update_add_htlcs[0].clone();
 428         nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &update_add_3);
 429         commitment_signed_dance!(nodes[3], nodes[2], update_3.commitment_signed, false, true);
 430
 431         assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
 432         for (_, pending_forwards) in nodes[3].node.forward_htlcs.lock().unwrap().iter_mut() {
 433                 for f in pending_forwards.iter_mut() {
 434                         match f {
 435                                 &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) => {
 436                                         match forward_info.routing {
 437                                                 PendingHTLCRouting::ReceiveKeysend { ref mut payment_data, .. } => {
 438                                                         *payment_data = Some(msgs::FinalOnionHopData {
 439                                                                 payment_secret: PaymentSecret([42; 32]),
 440                                                                 total_msat: amount * 2,
 441                                                         });
 442                                                 },
 443                                                 _ => panic!("Expected PendingHTLCRouting::ReceiveKeysend"),
 444                                         }
 445                                 },
 446                                 _ => {},
 447                         }
 448                 }
 449         }
 450         expect_pending_htlcs_forwardable!(nodes[3]);
 451         check_added_monitors!(nodes[3], 1);
 452
 453         // Fail back along nodes[2]
 454         let update_fail_0 = get_htlc_update_msgs!(&nodes[3], &nodes[2].node.get_our_node_id());
 455         nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &update_fail_0.update_fail_htlcs[0]);
 456         commitment_signed_dance!(nodes[2], nodes[3], update_fail_0.commitment_signed, false);
 457         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_channel_id }]);
 458         check_added_monitors!(nodes[2], 1);
 459
 460         let update_fail_1 = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 461         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_1.update_fail_htlcs[0]);
 462         commitment_signed_dance!(nodes[0], nodes[2], update_fail_1.commitment_signed, false);
 463
 464         expect_payment_failed_conditions(&nodes[0], payment_hash, true, PaymentFailedConditions::new());
 465         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 466 }
 467
 468
 469 #[test]
 470 fn no_pending_leak_on_initial_send_failure() {
 471         // In an earlier version of our payment tracking, we'd have a retry entry even when the initial
 472         // HTLC for payment failed to send due to local channel errors (e.g. peer disconnected). In this
 473         // case, the user wouldn't have a PaymentId to retry the payment with, but we'd think we have a
 474         // pending payment forever and never time it out.
 475         // Here we test exactly that - retrying a payment when a peer was disconnected on the first
 476         // try, and then check that no pending payment is being tracked.
 477         let chanmon_cfgs = create_chanmon_cfgs(2);
 478         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 479         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 480         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 481
 482         create_announced_chan_between_nodes(&nodes, 0, 1);
 483
 484         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 485
 486         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 487         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 488
 489         unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash,
 490                         RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
 491                 ), true, APIError::ChannelUnavailable { ref err },
 492                 assert_eq!(err, "Peer for first hop currently disconnected"));
 493
 494         assert!(!nodes[0].node.has_pending_payments());
 495 }
 496
 497 fn do_retry_with_no_persist(confirm_before_reload: bool) {
 498         // If we send a pending payment and `send_payment` returns success, we should always either
 499         // return a payment failure event or a payment success event, and on failure the payment should
 500         // be retryable.
 501         //
 502         // In order to do so when the ChannelManager isn't immediately persisted (which is normal - its
 503         // always persisted asynchronously), the ChannelManager has to reload some payment data from
 504         // ChannelMonitor(s) in some cases. This tests that reloading.
 505         //
 506         // `confirm_before_reload` confirms the channel-closing commitment transaction on-chain prior
 507         // to reloading the ChannelManager, increasing test coverage in ChannelMonitor HTLC tracking
 508         // which has separate codepaths for "commitment transaction already confirmed" and not.
 509         let chanmon_cfgs = create_chanmon_cfgs(3);
 510         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 511         let persister;
 512         let new_chain_monitor;
 513         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 514         let nodes_0_deserialized;
 515         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 516
 517         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 518         let (_, _, chan_id_2, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
 519
 520         // Serialize the ChannelManager prior to sending payments
 521         let nodes_0_serialized = nodes[0].node.encode();
 522
 523         // Send two payments - one which will get to nodes[2] and will be claimed, one which we'll time
 524         // out and retry.
 525         let amt_msat = 1_000_000;
 526         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
 527         let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000);
 528         let route_params = route.route_params.unwrap().clone();
 529         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 530                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
 531         check_added_monitors!(nodes[0], 1);
 532
 533         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 534         assert_eq!(events.len(), 1);
 535         let payment_event = SendEvent::from_event(events.pop().unwrap());
 536         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 537
 538         // We relay the payment to nodes[1] while its disconnected from nodes[2], causing the payment
 539         // to be returned immediately to nodes[0], without having nodes[2] fail the inbound payment
 540         // which would prevent retry.
 541         nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id());
 542         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 543
 544         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 545         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true);
 546         // nodes[1] now immediately fails the HTLC as the next-hop channel is disconnected
 547         let _ = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 548
 549         reconnect_nodes(ReconnectArgs::new(&nodes[1], &nodes[2]));
 550
 551         let as_commitment_tx = get_local_commitment_txn!(nodes[0], chan_id)[0].clone();
 552         if confirm_before_reload {
 553                 mine_transaction(&nodes[0], &as_commitment_tx);
 554                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 555         }
 556
 557         // The ChannelMonitor should always be the latest version, as we're required to persist it
 558         // during the `commitment_signed_dance!()`.
 559         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 560         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 561
 562         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 563         // force-close the channel.
 564         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[1].node.get_our_node_id()], 100000);
 565         assert!(nodes[0].node.list_channels().is_empty());
 566         assert!(nodes[0].node.has_pending_payments());
 567         nodes[0].node.timer_tick_occurred();
 568         if !confirm_before_reload {
 569                 let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 570                 assert_eq!(as_broadcasted_txn.len(), 1);
 571                 assert_eq!(as_broadcasted_txn[0].txid(), as_commitment_tx.txid());
 572         } else {
 573                 assert!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
 574         }
 575         check_added_monitors!(nodes[0], 1);
 576
 577         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 578         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 579                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 580         }, true).unwrap();
 581         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 582
 583         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 584         // error, as the channel has hit the chain.
 585         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 586                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 587         }, false).unwrap();
 588         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 589         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 590         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 591         assert_eq!(as_err.len(), 1);
 592         match as_err[0] {
 593                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 594                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 595                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 596                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}",
 597                                 &nodes[1].node.get_our_node_id())) }, [nodes[0].node.get_our_node_id()], 100000);
 598                         check_added_monitors!(nodes[1], 1);
 599                         assert_eq!(nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 600                 },
 601                 _ => panic!("Unexpected event"),
 602         }
 603         check_closed_broadcast!(nodes[1], false);
 604
 605         // Now claim the first payment, which should allow nodes[1] to claim the payment on-chain when
 606         // we close in a moment.
 607         nodes[2].node.claim_funds(payment_preimage_1);
 608         check_added_monitors!(nodes[2], 1);
 609         expect_payment_claimed!(nodes[2], payment_hash_1, 1_000_000);
 610
 611         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 612         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 613         check_added_monitors!(nodes[1], 1);
 614         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 615         expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], None, true, false);
 616
 617         if confirm_before_reload {
 618                 let best_block = nodes[0].blocks.lock().unwrap().last().unwrap().clone();
 619                 nodes[0].node.best_block_updated(&best_block.0.header, best_block.1);
 620         }
 621
 622         // Create a new channel on which to retry the payment before we fail the payment via the
 623         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 624         // connecting several blocks while creating the channel (implying time has passed).
 625         create_announced_chan_between_nodes(&nodes, 0, 1);
 626         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 627
 628         mine_transaction(&nodes[1], &as_commitment_tx);
 629         let bs_htlc_claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 630         assert_eq!(bs_htlc_claim_txn.len(), 1);
 631         check_spends!(bs_htlc_claim_txn[0], as_commitment_tx);
 632
 633         if !confirm_before_reload {
 634                 mine_transaction(&nodes[0], &as_commitment_tx);
 635         }
 636         mine_transaction(&nodes[0], &bs_htlc_claim_txn[0]);
 637         expect_payment_sent(&nodes[0], payment_preimage_1, None, true, false);
 638         connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
 639         let (first_htlc_timeout_tx, second_htlc_timeout_tx) = {
 640                 let mut txn = nodes[0].tx_broadcaster.unique_txn_broadcast();
 641                 assert_eq!(txn.len(), 2);
 642                 (txn.remove(0), txn.remove(0))
 643         };
 644         check_spends!(first_htlc_timeout_tx, as_commitment_tx);
 645         check_spends!(second_htlc_timeout_tx, as_commitment_tx);
 646         if first_htlc_timeout_tx.input[0].previous_output == bs_htlc_claim_txn[0].input[0].previous_output {
 647                 confirm_transaction(&nodes[0], &second_htlc_timeout_tx);
 648         } else {
 649                 confirm_transaction(&nodes[0], &first_htlc_timeout_tx);
 650         }
 651         nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 652         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new());
 653
 654         // Finally, retry the payment (which was reloaded from the ChannelMonitor when nodes[0] was
 655         // reloaded) via a route over the new channel, which work without issue and eventually be
 656         // received and claimed at the recipient just like any other payment.
 657         let (mut new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 658
 659         // Update the fee on the middle hop to ensure PaymentSent events have the correct (retried) fee
 660         // and not the original fee. We also update node[1]'s relevant config as
 661         // do_claim_payment_along_route expects us to never overpay.
 662         {
 663                 let per_peer_state = nodes[1].node.per_peer_state.read().unwrap();
 664                 let mut peer_state = per_peer_state.get(&nodes[2].node.get_our_node_id())
 665                         .unwrap().lock().unwrap();
 666                 let mut channel = peer_state.channel_by_id.get_mut(&chan_id_2).unwrap();
 667                 let mut new_config = channel.context().config();
 668                 new_config.forwarding_fee_base_msat += 100_000;
 669                 channel.context_mut().update_config(&new_config);
 670                 new_route.paths[0].hops[0].fee_msat += 100_000;
 671         }
 672
 673         // Force expiration of the channel's previous config.
 674         for _ in 0..EXPIRE_PREV_CONFIG_TICKS {
 675                 nodes[1].node.timer_tick_occurred();
 676         }
 677
 678         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash, // Shouldn't be allowed to retry a fulfilled payment
 679                 RecipientOnionFields::secret_only(payment_secret), payment_id_1).is_err());
 680         nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 681                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 682         check_added_monitors!(nodes[0], 1);
 683         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 684         assert_eq!(events.len(), 1);
 685         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 686         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
 687         expect_payment_sent!(nodes[0], payment_preimage, Some(new_route.paths[0].hops[0].fee_msat));
 688 }
 689
 690 #[test]
 691 fn retry_with_no_persist() {
 692         do_retry_with_no_persist(true);
 693         do_retry_with_no_persist(false);
 694 }
 695
 696 fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 697         // Test that an off-chain completed payment is not retryable on restart. This was previously
 698         // broken for dust payments, but we test for both dust and non-dust payments.
 699         //
 700         // `use_dust` switches to using a dust HTLC, which results in the HTLC not having an on-chain
 701         // output at all.
 702         let chanmon_cfgs = create_chanmon_cfgs(3);
 703         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 704
 705         let mut manually_accept_config = test_default_channel_config();
 706         manually_accept_config.manually_accept_inbound_channels = true;
 707
 708         let first_persister;
 709         let first_new_chain_monitor;
 710         let second_persister;
 711         let second_new_chain_monitor;
 712         let third_persister;
 713         let third_new_chain_monitor;
 714
 715         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(manually_accept_config), None]);
 716         let first_nodes_0_deserialized;
 717         let second_nodes_0_deserialized;
 718         let third_nodes_0_deserialized;
 719
 720         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 721
 722         // Because we set nodes[1] to manually accept channels, just open a 0-conf channel.
 723         let (funding_tx, chan_id) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 724         confirm_transaction(&nodes[0], &funding_tx);
 725         confirm_transaction(&nodes[1], &funding_tx);
 726         // Ignore the announcement_signatures messages
 727         nodes[0].node.get_and_clear_pending_msg_events();
 728         nodes[1].node.get_and_clear_pending_msg_events();
 729         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 730
 731         // Serialize the ChannelManager prior to sending payments
 732         let mut nodes_0_serialized = nodes[0].node.encode();
 733
 734         let route = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 }).0;
 735         let (payment_preimage, payment_hash, payment_secret, payment_id) = send_along_route(&nodes[0], route, &[&nodes[1], &nodes[2]], if use_dust { 1_000 } else { 1_000_000 });
 736
 737         // The ChannelMonitor should always be the latest version, as we're required to persist it
 738         // during the `commitment_signed_dance!()`.
 739         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 740
 741         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized], first_persister, first_new_chain_monitor, first_nodes_0_deserialized);
 742         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 743
 744         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 745         // force-close the channel.
 746         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[1].node.get_our_node_id()], 100000);
 747         nodes[0].node.timer_tick_occurred();
 748         assert!(nodes[0].node.list_channels().is_empty());
 749         assert!(nodes[0].node.has_pending_payments());
 750         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 751         check_added_monitors!(nodes[0], 1);
 752
 753         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 754                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 755         }, true).unwrap();
 756         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 757
 758         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 759         // error, as the channel has hit the chain.
 760         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 761                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 762         }, false).unwrap();
 763         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 764         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 765         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 766         assert_eq!(as_err.len(), 1);
 767         let bs_commitment_tx;
 768         match as_err[0] {
 769                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 770                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 771                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 772                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) }
 773                                 , [nodes[0].node.get_our_node_id()], 100000);
 774                         check_added_monitors!(nodes[1], 1);
 775                         bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 776                 },
 777                 _ => panic!("Unexpected event"),
 778         }
 779         check_closed_broadcast!(nodes[1], false);
 780
 781         // Now fail back the payment from nodes[2] to nodes[1]. This doesn't really matter as the
 782         // previous hop channel is already on-chain, but it makes nodes[2] willing to see additional
 783         // incoming HTLCs with the same payment hash later.
 784         nodes[2].node.fail_htlc_backwards(&payment_hash);
 785         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
 786         check_added_monitors!(nodes[2], 1);
 787
 788         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 789         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fail_htlcs[0]);
 790         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 791         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1],
 792                 [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 793
 794         // Connect the HTLC-Timeout transaction, timing out the HTLC on both nodes (but not confirming
 795         // the HTLC-Timeout transaction beyond 1 conf). For dust HTLCs, the HTLC is considered resolved
 796         // after the commitment transaction, so always connect the commitment transaction.
 797         mine_transaction(&nodes[0], &bs_commitment_tx[0]);
 798         mine_transaction(&nodes[1], &bs_commitment_tx[0]);
 799         if !use_dust {
 800                 connect_blocks(&nodes[0], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
 801                 connect_blocks(&nodes[1], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
 802                 let as_htlc_timeout = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 803                 check_spends!(as_htlc_timeout[0], bs_commitment_tx[0]);
 804                 assert_eq!(as_htlc_timeout.len(), 1);
 805
 806                 mine_transaction(&nodes[0], &as_htlc_timeout[0]);
 807                 // nodes[0] may rebroadcast (or RBF-bump) its HTLC-Timeout, so wipe the announced set.
 808                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 809                 mine_transaction(&nodes[1], &as_htlc_timeout[0]);
 810         }
 811
 812         // Create a new channel on which to retry the payment before we fail the payment via the
 813         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 814         // connecting several blocks while creating the channel (implying time has passed).
 815         // We do this with a zero-conf channel to avoid connecting blocks as a side-effect.
 816         let (_, chan_id_3) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 817         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 818
 819         // If we attempt to retry prior to the HTLC-Timeout (or commitment transaction, for dust HTLCs)
 820         // confirming, we will fail as it's considered still-pending...
 821         let (new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 });
 822         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 823                 Err(PaymentSendFailure::DuplicatePayment) => {},
 824                 _ => panic!("Unexpected error")
 825         }
 826         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 827
 828         // After ANTI_REORG_DELAY confirmations, the HTLC should be failed and we can try the payment
 829         // again. We serialize the node first as we'll then test retrying the HTLC after a restart
 830         // (which should also still work).
 831         connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
 832         connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 833         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new());
 834
 835         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 836         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 837         nodes_0_serialized = nodes[0].node.encode();
 838
 839         // After the payment failed, we're free to send it again.
 840         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 841                 RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
 842         assert!(!nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 843
 844         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], second_persister, second_new_chain_monitor, second_nodes_0_deserialized);
 845         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 846
 847         nodes[0].node.test_process_background_events();
 848         check_added_monitors(&nodes[0], 1);
 849
 850         let mut reconnect_args = ReconnectArgs::new(&nodes[0], &nodes[1]);
 851         reconnect_args.send_channel_ready = (true, true);
 852         reconnect_nodes(reconnect_args);
 853
 854         // Now resend the payment, delivering the HTLC and actually claiming it this time. This ensures
 855         // the payment is not (spuriously) listed as still pending.
 856         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 857                 RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
 858         check_added_monitors!(nodes[0], 1);
 859         pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], if use_dust { 1_000 } else { 1_000_000 }, payment_hash, payment_secret);
 860         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
 861
 862         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 863                 Err(PaymentSendFailure::DuplicatePayment) => {},
 864                 _ => panic!("Unexpected error")
 865         }
 866         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 867
 868         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 869         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 870         nodes_0_serialized = nodes[0].node.encode();
 871
 872         // Check that after reload we can send the payment again (though we shouldn't, since it was
 873         // claimed previously).
 874         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], third_persister, third_new_chain_monitor, third_nodes_0_deserialized);
 875         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 876
 877         nodes[0].node.test_process_background_events();
 878         check_added_monitors(&nodes[0], 1);
 879
 880         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
 881
 882         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 883                 Err(PaymentSendFailure::DuplicatePayment) => {},
 884                 _ => panic!("Unexpected error")
 885         }
 886         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 887 }
 888
 889 #[test]
 890 fn test_completed_payment_not_retryable_on_reload() {
 891         do_test_completed_payment_not_retryable_on_reload(true);
 892         do_test_completed_payment_not_retryable_on_reload(false);
 893 }
 894
 895
 896 fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, confirm_commitment_tx: bool, payment_timeout: bool) {
 897         // When a Channel is closed, any outbound HTLCs which were relayed through it are simply
 898         // dropped when the Channel is. From there, the ChannelManager relies on the ChannelMonitor
 899         // having a copy of the relevant fail-/claim-back data and processes the HTLC fail/claim when
 900         // the ChannelMonitor tells it to.
 901         //
 902         // If, due to an on-chain event, an HTLC is failed/claimed, we should avoid providing the
 903         // ChannelManager the HTLC event until after the monitor is re-persisted. This should prevent a
 904         // duplicate HTLC fail/claim (e.g. via a PaymentPathFailed event).
 905         let chanmon_cfgs = create_chanmon_cfgs(2);
 906         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 907         let persister;
 908         let new_chain_monitor;
 909         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 910         let nodes_0_deserialized;
 911         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 912
 913         let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1);
 914
 915         // Route a payment, but force-close the channel before the HTLC fulfill message arrives at
 916         // nodes[0].
 917         let (payment_preimage, payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 10_000_000);
 918         nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap();
 919         check_closed_broadcast!(nodes[0], true);
 920         check_added_monitors!(nodes[0], 1);
 921         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 100000);
 922
 923         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 924         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 925
 926         // Connect blocks until the CLTV timeout is up so that we get an HTLC-Timeout transaction
 927         connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
 928         let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 929         assert_eq!(node_txn.len(), 3);
 930         assert_eq!(node_txn[0].txid(), node_txn[1].txid());
 931         check_spends!(node_txn[1], funding_tx);
 932         check_spends!(node_txn[2], node_txn[1]);
 933         let timeout_txn = vec![node_txn[2].clone()];
 934
 935         nodes[1].node.claim_funds(payment_preimage);
 936         check_added_monitors!(nodes[1], 1);
 937         expect_payment_claimed!(nodes[1], payment_hash, 10_000_000);
 938
 939         connect_block(&nodes[1], &create_dummy_block(nodes[1].best_block_hash(), 42, vec![node_txn[1].clone()]));
 940         check_closed_broadcast!(nodes[1], true);
 941         check_added_monitors!(nodes[1], 1);
 942         check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed, [nodes[0].node.get_our_node_id()], 100000);
 943         let claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 944         assert_eq!(claim_txn.len(), 1);
 945         check_spends!(claim_txn[0], node_txn[1]);
 946
 947         connect_block(&nodes[0], &create_dummy_block(nodes[0].best_block_hash(), 42, vec![node_txn[1].clone()]));
 948
 949         if confirm_commitment_tx {
 950                 connect_blocks(&nodes[0], BREAKDOWN_TIMEOUT as u32 - 1);
 951         }
 952
 953         let claim_block = create_dummy_block(nodes[0].best_block_hash(), 42, if payment_timeout { timeout_txn } else { vec![claim_txn[0].clone()] });
 954
 955         if payment_timeout {
 956                 assert!(confirm_commitment_tx); // Otherwise we're spending below our CSV!
 957                 connect_block(&nodes[0], &claim_block);
 958                 connect_blocks(&nodes[0], ANTI_REORG_DELAY - 2);
 959         }
 960
 961         // Now connect the HTLC claim transaction with the ChainMonitor-generated ChannelMonitor update
 962         // returning InProgress. This should cause the claim event to never make its way to the
 963         // ChannelManager.
 964         chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap().clear();
 965         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
 966
 967         if payment_timeout {
 968                 connect_blocks(&nodes[0], 1);
 969         } else {
 970                 connect_block(&nodes[0], &claim_block);
 971         }
 972
 973         let funding_txo = OutPoint { txid: funding_tx.txid(), index: 0 };
 974         let mon_updates: Vec<_> = chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap()
 975                 .get_mut(&funding_txo).unwrap().drain().collect();
 976         // If we are using chain::Confirm instead of chain::Listen, we will get the same update twice.
 977         // If we're testing connection idempotency we may get substantially more.
 978         assert!(mon_updates.len() >= 1);
 979         assert!(nodes[0].chain_monitor.release_pending_monitor_events().is_empty());
 980         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 981
 982         // If we persist the ChannelManager here, we should get the PaymentSent event after
 983         // deserialization.
 984         let mut chan_manager_serialized = Vec::new();
 985         if !persist_manager_post_event {
 986                 chan_manager_serialized = nodes[0].node.encode();
 987         }
 988
 989         // Now persist the ChannelMonitor and inform the ChainMonitor that we're done, generating the
 990         // payment sent event.
 991         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
 992         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 993         for update in mon_updates {
 994                 nodes[0].chain_monitor.chain_monitor.channel_monitor_updated(funding_txo, update).unwrap();
 995         }
 996         if payment_timeout {
 997                 expect_payment_failed!(nodes[0], payment_hash, false);
 998         } else {
 999                 expect_payment_sent(&nodes[0], payment_preimage, None, true, false);
1000         }
1001
1002         // If we persist the ChannelManager after we get the PaymentSent event, we shouldn't get it
1003         // twice.
1004         if persist_manager_post_event {
1005                 chan_manager_serialized = nodes[0].node.encode();
1006         }
1007
1008         // Now reload nodes[0]...
1009         reload_node!(nodes[0], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
1010
1011         if persist_manager_post_event {
1012                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1013         } else if payment_timeout {
1014                 expect_payment_failed!(nodes[0], payment_hash, false);
1015         } else {
1016                 expect_payment_sent(&nodes[0], payment_preimage, None, true, false);
1017         }
1018
1019         // Note that if we re-connect the block which exposed nodes[0] to the payment preimage (but
1020         // which the current ChannelMonitor has not seen), the ChannelManager's de-duplication of
1021         // payment events should kick in, leaving us with no pending events here.
1022         let height = nodes[0].blocks.lock().unwrap().len() as u32 - 1;
1023         nodes[0].chain_monitor.chain_monitor.block_connected(&claim_block, height);
1024         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1025         check_added_monitors(&nodes[0], 1);
1026 }
1027
1028 #[test]
1029 fn test_dup_htlc_onchain_fails_on_reload() {
1030         do_test_dup_htlc_onchain_fails_on_reload(true, true, true);
1031         do_test_dup_htlc_onchain_fails_on_reload(true, true, false);
1032         do_test_dup_htlc_onchain_fails_on_reload(true, false, false);
1033         do_test_dup_htlc_onchain_fails_on_reload(false, true, true);
1034         do_test_dup_htlc_onchain_fails_on_reload(false, true, false);
1035         do_test_dup_htlc_onchain_fails_on_reload(false, false, false);
1036 }
1037
1038 #[test]
1039 fn test_fulfill_restart_failure() {
1040         // When we receive an update_fulfill_htlc message, we immediately consider the HTLC fully
1041         // fulfilled. At this point, the peer can reconnect and decide to either fulfill the HTLC
1042         // again, or fail it, giving us free money.
1043         //
1044         // Of course probably they won't fail it and give us free money, but because we have code to
1045         // handle it, we should test the logic for it anyway. We do that here.
1046         let chanmon_cfgs = create_chanmon_cfgs(2);
1047         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1048         let persister;
1049         let new_chain_monitor;
1050         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1051         let nodes_1_deserialized;
1052         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1053
1054         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1055         let (payment_preimage, payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 100_000);
1056
1057         // The simplest way to get a failure after a fulfill is to reload nodes[1] from a state
1058         // pre-fulfill, which we do by serializing it here.
1059         let chan_manager_serialized = nodes[1].node.encode();
1060         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id).encode();
1061
1062         nodes[1].node.claim_funds(payment_preimage);
1063         check_added_monitors!(nodes[1], 1);
1064         expect_payment_claimed!(nodes[1], payment_hash, 100_000);
1065
1066         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1067         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
1068         expect_payment_sent(&nodes[0], payment_preimage, None, false, false);
1069
1070         // Now reload nodes[1]...
1071         reload_node!(nodes[1], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1072
1073         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1074         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1075
1076         nodes[1].node.fail_htlc_backwards(&payment_hash);
1077         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]);
1078         check_added_monitors!(nodes[1], 1);
1079         let htlc_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1080         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates.update_fail_htlcs[0]);
1081         commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates.commitment_signed, false);
1082         // nodes[0] shouldn't generate any events here, while it just got a payment failure completion
1083         // it had already considered the payment fulfilled, and now they just got free money.
1084         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1085 }
1086
1087 #[test]
1088 fn get_ldk_payment_preimage() {
1089         // Ensure that `ChannelManager::get_payment_preimage` can successfully be used to claim a payment.
1090         let chanmon_cfgs = create_chanmon_cfgs(2);
1091         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1092         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1093         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1094         create_announced_chan_between_nodes(&nodes, 0, 1);
1095
1096         let amt_msat = 60_000;
1097         let expiry_secs = 60 * 60;
1098         let (payment_hash, payment_secret) = nodes[1].node.create_inbound_payment(Some(amt_msat), expiry_secs, None).unwrap();
1099
1100         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
1101                 .with_bolt11_features(nodes[1].node.invoice_features()).unwrap();
1102         let scorer = test_utils::TestScorer::new();
1103         let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet);
1104         let random_seed_bytes = keys_manager.get_secure_random_bytes();
1105         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
1106         let route = get_route( &nodes[0].node.get_our_node_id(), &route_params,
1107                 &nodes[0].network_graph.read_only(),
1108                 Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()), nodes[0].logger,
1109                 &scorer, &(), &random_seed_bytes).unwrap();
1110         nodes[0].node.send_payment_with_route(&route, payment_hash,
1111                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
1112         check_added_monitors!(nodes[0], 1);
1113
1114         // Make sure to use `get_payment_preimage`
1115         let payment_preimage = nodes[1].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1116         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1117         assert_eq!(events.len(), 1);
1118         pass_along_path(&nodes[0], &[&nodes[1]], amt_msat, payment_hash, Some(payment_secret), events.pop().unwrap(), true, Some(payment_preimage));
1119         claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, payment_preimage);
1120 }
1121
1122 #[test]
1123 fn sent_probe_is_probe_of_sending_node() {
1124         let chanmon_cfgs = create_chanmon_cfgs(3);
1125         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1126         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1127         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1128
1129         create_announced_chan_between_nodes(&nodes, 0, 1);
1130         create_announced_chan_between_nodes(&nodes, 1, 2);
1131
1132         // First check we refuse to build a single-hop probe
1133         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[1], 100_000);
1134         assert!(nodes[0].node.send_probe(route.paths[0].clone()).is_err());
1135
1136         // Then build an actual two-hop probing path
1137         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
1138
1139         match nodes[0].node.send_probe(route.paths[0].clone()) {
1140                 Ok((payment_hash, payment_id)) => {
1141                         assert!(nodes[0].node.payment_is_probe(&payment_hash, &payment_id));
1142                         assert!(!nodes[1].node.payment_is_probe(&payment_hash, &payment_id));
1143                         assert!(!nodes[2].node.payment_is_probe(&payment_hash, &payment_id));
1144                 },
1145                 _ => panic!(),
1146         }
1147
1148         get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1149         check_added_monitors!(nodes[0], 1);
1150 }
1151
1152 #[test]
1153 fn successful_probe_yields_event() {
1154         let chanmon_cfgs = create_chanmon_cfgs(3);
1155         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1156         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1157         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1158
1159         create_announced_chan_between_nodes(&nodes, 0, 1);
1160         create_announced_chan_between_nodes(&nodes, 1, 2);
1161
1162         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
1163
1164         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1165
1166         // node[0] -- update_add_htlcs -> node[1]
1167         check_added_monitors!(nodes[0], 1);
1168         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1169         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1170         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1171         check_added_monitors!(nodes[1], 0);
1172         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1173         expect_pending_htlcs_forwardable!(nodes[1]);
1174
1175         // node[1] -- update_add_htlcs -> node[2]
1176         check_added_monitors!(nodes[1], 1);
1177         let updates = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1178         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1179         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &probe_event.msgs[0]);
1180         check_added_monitors!(nodes[2], 0);
1181         commitment_signed_dance!(nodes[2], nodes[1], probe_event.commitment_msg, true, true);
1182
1183         // node[1] <- update_fail_htlcs -- node[2]
1184         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1185         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1186         check_added_monitors!(nodes[1], 0);
1187         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, true);
1188
1189         // node[0] <- update_fail_htlcs -- node[1]
1190         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1191         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1192         check_added_monitors!(nodes[0], 0);
1193         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1194
1195         let mut events = nodes[0].node.get_and_clear_pending_events();
1196         assert_eq!(events.len(), 1);
1197         match events.drain(..).next().unwrap() {
1198                 crate::events::Event::ProbeSuccessful { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1199                         assert_eq!(payment_id, ev_pid);
1200                         assert_eq!(payment_hash, ev_ph);
1201                 },
1202                 _ => panic!(),
1203         };
1204         assert!(!nodes[0].node.has_pending_payments());
1205 }
1206
1207 #[test]
1208 fn failed_probe_yields_event() {
1209         let chanmon_cfgs = create_chanmon_cfgs(3);
1210         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1211         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1212         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1213
1214         create_announced_chan_between_nodes(&nodes, 0, 1);
1215         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 100000, 90000000);
1216
1217         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1218
1219         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], payment_params, 9_998_000);
1220
1221         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1222
1223         // node[0] -- update_add_htlcs -> node[1]
1224         check_added_monitors!(nodes[0], 1);
1225         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1226         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1227         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1228         check_added_monitors!(nodes[1], 0);
1229         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1230         expect_pending_htlcs_forwardable!(nodes[1]);
1231
1232         // node[0] <- update_fail_htlcs -- node[1]
1233         check_added_monitors!(nodes[1], 1);
1234         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1235         // Skip the PendingHTLCsForwardable event
1236         let _events = nodes[1].node.get_and_clear_pending_events();
1237         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1238         check_added_monitors!(nodes[0], 0);
1239         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1240
1241         let mut events = nodes[0].node.get_and_clear_pending_events();
1242         assert_eq!(events.len(), 1);
1243         match events.drain(..).next().unwrap() {
1244                 crate::events::Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1245                         assert_eq!(payment_id, ev_pid);
1246                         assert_eq!(payment_hash, ev_ph);
1247                 },
1248                 _ => panic!(),
1249         };
1250         assert!(!nodes[0].node.has_pending_payments());
1251 }
1252
1253 #[test]
1254 fn onchain_failed_probe_yields_event() {
1255         // Tests that an attempt to probe over a channel that is eventaully closed results in a failure
1256         // event.
1257         let chanmon_cfgs = create_chanmon_cfgs(3);
1258         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1259         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1260         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1261
1262         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1263         create_announced_chan_between_nodes(&nodes, 1, 2);
1264
1265         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1266
1267         // Send a dust HTLC, which will be treated as if it timed out once the channel hits the chain.
1268         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], payment_params, 1_000);
1269         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1270
1271         // node[0] -- update_add_htlcs -> node[1]
1272         check_added_monitors!(nodes[0], 1);
1273         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1274         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1275         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1276         check_added_monitors!(nodes[1], 0);
1277         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1278         expect_pending_htlcs_forwardable!(nodes[1]);
1279
1280         check_added_monitors!(nodes[1], 1);
1281         let _ = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1282
1283         // Don't bother forwarding the HTLC onwards and just confirm the force-close transaction on
1284         // Node A, which after 6 confirmations should result in a probe failure event.
1285         let bs_txn = get_local_commitment_txn!(nodes[1], chan_id);
1286         confirm_transaction(&nodes[0], &bs_txn[0]);
1287         check_closed_broadcast!(&nodes[0], true);
1288         check_added_monitors!(nodes[0], 1);
1289
1290         let mut events = nodes[0].node.get_and_clear_pending_events();
1291         assert_eq!(events.len(), 2);
1292         let mut found_probe_failed = false;
1293         for event in events.drain(..) {
1294                 match event {
1295                         Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1296                                 assert_eq!(payment_id, ev_pid);
1297                                 assert_eq!(payment_hash, ev_ph);
1298                                 found_probe_failed = true;
1299                         },
1300                         Event::ChannelClosed { .. } => {},
1301                         _ => panic!(),
1302                 }
1303         }
1304         assert!(found_probe_failed);
1305         assert!(!nodes[0].node.has_pending_payments());
1306 }
1307
1308 #[test]
1309 fn preflight_probes_yield_event_and_skip() {
1310         let chanmon_cfgs = create_chanmon_cfgs(5);
1311         let node_cfgs = create_node_cfgs(5, &chanmon_cfgs);
1312
1313         // We alleviate the HTLC max-in-flight limit, as otherwise we'd always be limited through that.
1314         let mut no_htlc_limit_config = test_default_channel_config();
1315         no_htlc_limit_config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 100;
1316
1317         let user_configs = std::iter::repeat(no_htlc_limit_config).take(5).map(|c| Some(c)).collect::<Vec<Option<UserConfig>>>();
1318         let node_chanmgrs = create_node_chanmgrs(5, &node_cfgs, &user_configs);
1319         let nodes = create_network(5, &node_cfgs, &node_chanmgrs);
1320
1321         // Setup channel topology:
1322         //                    (30k:0)- N2 -(1M:0)
1323         //                   /                  \
1324         //  N0 -(100k:0)-> N1                    N4
1325         //                   \                  /
1326         //                    (70k:0)- N3 -(1M:0)
1327         //
1328         let first_chan_update = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0).0;
1329         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 30_000, 0);
1330         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 70_000, 0);
1331         create_announced_chan_between_nodes_with_value(&nodes, 2, 4, 1_000_000, 0);
1332         create_announced_chan_between_nodes_with_value(&nodes, 3, 4, 1_000_000, 0);
1333
1334         let mut invoice_features = Bolt11InvoiceFeatures::empty();
1335         invoice_features.set_basic_mpp_optional();
1336
1337         let mut payment_params = PaymentParameters::from_node_id(nodes[4].node.get_our_node_id(), TEST_FINAL_CLTV)
1338                 .with_bolt11_features(invoice_features).unwrap();
1339
1340         let route_params = RouteParameters { payment_params, final_value_msat: 80_000_000 };
1341         let res = nodes[0].node.send_preflight_probes(route_params, None).unwrap();
1342
1343         // We check that only one probe was sent, the other one was skipped due to limited liquidity.
1344         assert_eq!(res.len(), 1);
1345         let log_msg = format!("Skipped sending payment probe to avoid putting channel {} under the liquidity limit.",
1346                 first_chan_update.contents.short_channel_id);
1347         node_cfgs[0].logger.assert_log_contains("lightning::ln::channelmanager", &log_msg, 1);
1348
1349         let (payment_hash, payment_id) = res.first().unwrap();
1350
1351         // node[0] -- update_add_htlcs -> node[1]
1352         check_added_monitors!(nodes[0], 1);
1353         let probe_event = SendEvent::from_node(&nodes[0]);
1354         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1355         check_added_monitors!(nodes[1], 0);
1356         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1357         expect_pending_htlcs_forwardable!(nodes[1]);
1358
1359         // node[1] -- update_add_htlcs -> node[2]
1360         check_added_monitors!(nodes[1], 1);
1361         let probe_event = SendEvent::from_node(&nodes[1]);
1362         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &probe_event.msgs[0]);
1363         check_added_monitors!(nodes[2], 0);
1364         commitment_signed_dance!(nodes[2], nodes[1], probe_event.commitment_msg, false);
1365         expect_pending_htlcs_forwardable!(nodes[2]);
1366
1367         // node[2] -- update_add_htlcs -> node[4]
1368         check_added_monitors!(nodes[2], 1);
1369         let probe_event = SendEvent::from_node(&nodes[2]);
1370         nodes[4].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &probe_event.msgs[0]);
1371         check_added_monitors!(nodes[4], 0);
1372         commitment_signed_dance!(nodes[4], nodes[2], probe_event.commitment_msg, true, true);
1373
1374         // node[2] <- update_fail_htlcs -- node[4]
1375         let updates = get_htlc_update_msgs!(nodes[4], nodes[2].node.get_our_node_id());
1376         nodes[2].node.handle_update_fail_htlc(&nodes[4].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1377         check_added_monitors!(nodes[2], 0);
1378         commitment_signed_dance!(nodes[2], nodes[4], updates.commitment_signed, true);
1379
1380         // node[1] <- update_fail_htlcs -- node[2]
1381         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1382         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1383         check_added_monitors!(nodes[1], 0);
1384         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, true);
1385
1386         // node[0] <- update_fail_htlcs -- node[1]
1387         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1388         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1389         check_added_monitors!(nodes[0], 0);
1390         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1391
1392         let mut events = nodes[0].node.get_and_clear_pending_events();
1393         assert_eq!(events.len(), 1);
1394         match events.drain(..).next().unwrap() {
1395                 crate::events::Event::ProbeSuccessful { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1396                         assert_eq!(*payment_id, ev_pid);
1397                         assert_eq!(*payment_hash, ev_ph);
1398                 },
1399                 _ => panic!(),
1400         };
1401         assert!(!nodes[0].node.has_pending_payments());
1402 }
1403
1404 #[test]
1405 fn claimed_send_payment_idempotent() {
1406         // Tests that `send_payment` (and friends) are (reasonably) idempotent.
1407         let chanmon_cfgs = create_chanmon_cfgs(2);
1408         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1409         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1410         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1411
1412         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1413
1414         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1415         let (first_payment_preimage, _, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1416
1417         macro_rules! check_send_rejected {
1418                 () => {
1419                         // If we try to resend a new payment with a different payment_hash but with the same
1420                         // payment_id, it should be rejected.
1421                         let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1422                                 RecipientOnionFields::secret_only(second_payment_secret), payment_id);
1423                         match send_result {
1424                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1425                                 _ => panic!("Unexpected send result: {:?}", send_result),
1426                         }
1427
1428                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1429                         // also be rejected.
1430                         let send_result = nodes[0].node.send_spontaneous_payment(
1431                                 &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
1432                         match send_result {
1433                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1434                                 _ => panic!("Unexpected send result: {:?}", send_result),
1435                         }
1436                 }
1437         }
1438
1439         check_send_rejected!();
1440
1441         // Claim the payment backwards, but note that the PaymentSent event is still pending and has
1442         // not been seen by the user. At this point, from the user perspective nothing has changed, so
1443         // we must remain just as idempotent as we were before.
1444         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, first_payment_preimage);
1445
1446         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1447                 nodes[0].node.timer_tick_occurred();
1448         }
1449
1450         check_send_rejected!();
1451
1452         // Once the user sees and handles the `PaymentSent` event, we expect them to no longer call
1453         // `send_payment`, and our idempotency guarantees are off - they should have atomically marked
1454         // the payment complete. However, they could have called `send_payment` while the event was
1455         // being processed, leading to a race in our idempotency guarantees. Thus, even immediately
1456         // after the event is handled a duplicate payment should sitll be rejected.
1457         expect_payment_sent!(&nodes[0], first_payment_preimage, Some(0));
1458         check_send_rejected!();
1459
1460         // If relatively little time has passed, a duplicate payment should still fail.
1461         nodes[0].node.timer_tick_occurred();
1462         check_send_rejected!();
1463
1464         // However, after some time has passed (at least more than the one timer tick above), a
1465         // duplicate payment should go through, as ChannelManager should no longer have any remaining
1466         // references to the old payment data.
1467         for _ in 0..IDEMPOTENCY_TIMEOUT_TICKS {
1468                 nodes[0].node.timer_tick_occurred();
1469         }
1470
1471         nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1472                 RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
1473         check_added_monitors!(nodes[0], 1);
1474         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1475         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1476 }
1477
1478 #[test]
1479 fn abandoned_send_payment_idempotent() {
1480         // Tests that `send_payment` (and friends) allow duplicate PaymentIds immediately after
1481         // abandon_payment.
1482         let chanmon_cfgs = create_chanmon_cfgs(2);
1483         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1484         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1485         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1486
1487         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1488
1489         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1490         let (_, first_payment_hash, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1491
1492         macro_rules! check_send_rejected {
1493                 () => {
1494                         // If we try to resend a new payment with a different payment_hash but with the same
1495                         // payment_id, it should be rejected.
1496                         let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1497                                 RecipientOnionFields::secret_only(second_payment_secret), payment_id);
1498                         match send_result {
1499                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1500                                 _ => panic!("Unexpected send result: {:?}", send_result),
1501                         }
1502
1503                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1504                         // also be rejected.
1505                         let send_result = nodes[0].node.send_spontaneous_payment(
1506                                 &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
1507                         match send_result {
1508                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1509                                 _ => panic!("Unexpected send result: {:?}", send_result),
1510                         }
1511                 }
1512         }
1513
1514         check_send_rejected!();
1515
1516         nodes[1].node.fail_htlc_backwards(&first_payment_hash);
1517         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash: first_payment_hash }]);
1518
1519         // Until we abandon the payment upon path failure, no matter how many timer ticks pass, we still cannot reuse the
1520         // PaymentId.
1521         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1522                 nodes[0].node.timer_tick_occurred();
1523         }
1524         check_send_rejected!();
1525
1526         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash, PaymentFailureReason::RecipientRejected);
1527
1528         // However, we can reuse the PaymentId immediately after we `abandon_payment` upon passing the
1529         // failed payment back.
1530         nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1531                 RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
1532         check_added_monitors!(nodes[0], 1);
1533         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1534         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1535 }
1536
1537 #[derive(PartialEq)]
1538 enum InterceptTest {
1539         Forward,
1540         Fail,
1541         Timeout,
1542 }
1543
1544 #[test]
1545 fn test_trivial_inflight_htlc_tracking(){
1546         // In this test, we test three scenarios:
1547         // (1) Sending + claiming a payment successfully should return `None` when querying InFlightHtlcs
1548         // (2) Sending a payment without claiming it should return the payment's value (500000) when querying InFlightHtlcs
1549         // (3) After we claim the payment sent in (2), InFlightHtlcs should return `None` for the query.
1550         let chanmon_cfgs = create_chanmon_cfgs(3);
1551         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1552         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1553         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1554
1555         let (_, _, chan_1_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
1556         let (_, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
1557
1558         // Send and claim the payment. Inflight HTLCs should be empty.
1559         let (_, payment_hash, _, payment_id) = send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 500000);
1560         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1561         {
1562                 let mut node_0_per_peer_lock;
1563                 let mut node_0_peer_state_lock;
1564                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1565
1566                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1567                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1568                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1569                         channel_1.context().get_short_channel_id().unwrap()
1570                 );
1571                 assert_eq!(chan_1_used_liquidity, None);
1572         }
1573         {
1574                 let mut node_1_per_peer_lock;
1575                 let mut node_1_peer_state_lock;
1576                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1577
1578                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1579                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1580                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1581                         channel_2.context().get_short_channel_id().unwrap()
1582                 );
1583
1584                 assert_eq!(chan_2_used_liquidity, None);
1585         }
1586         let pending_payments = nodes[0].node.list_recent_payments();
1587         assert_eq!(pending_payments.len(), 1);
1588         assert_eq!(pending_payments[0], RecentPaymentDetails::Fulfilled { payment_hash: Some(payment_hash), payment_id });
1589
1590         // Remove fulfilled payment
1591         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1592                 nodes[0].node.timer_tick_occurred();
1593         }
1594
1595         // Send the payment, but do not claim it. Our inflight HTLCs should contain the pending payment.
1596         let (payment_preimage, payment_hash,  _, payment_id) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 500000);
1597         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1598         {
1599                 let mut node_0_per_peer_lock;
1600                 let mut node_0_peer_state_lock;
1601                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1602
1603                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1604                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1605                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1606                         channel_1.context().get_short_channel_id().unwrap()
1607                 );
1608                 // First hop accounts for expected 1000 msat fee
1609                 assert_eq!(chan_1_used_liquidity, Some(501000));
1610         }
1611         {
1612                 let mut node_1_per_peer_lock;
1613                 let mut node_1_peer_state_lock;
1614                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1615
1616                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1617                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1618                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1619                         channel_2.context().get_short_channel_id().unwrap()
1620                 );
1621
1622                 assert_eq!(chan_2_used_liquidity, Some(500000));
1623         }
1624         let pending_payments = nodes[0].node.list_recent_payments();
1625         assert_eq!(pending_payments.len(), 1);
1626         assert_eq!(pending_payments[0], RecentPaymentDetails::Pending { payment_id, payment_hash, total_msat: 500000 });
1627
1628         // Now, let's claim the payment. This should result in the used liquidity to return `None`.
1629         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
1630
1631         // Remove fulfilled payment
1632         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1633                 nodes[0].node.timer_tick_occurred();
1634         }
1635
1636         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1637         {
1638                 let mut node_0_per_peer_lock;
1639                 let mut node_0_peer_state_lock;
1640                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1641
1642                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1643                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1644                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1645                         channel_1.context().get_short_channel_id().unwrap()
1646                 );
1647                 assert_eq!(chan_1_used_liquidity, None);
1648         }
1649         {
1650                 let mut node_1_per_peer_lock;
1651                 let mut node_1_peer_state_lock;
1652                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1653
1654                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1655                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1656                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1657                         channel_2.context().get_short_channel_id().unwrap()
1658                 );
1659                 assert_eq!(chan_2_used_liquidity, None);
1660         }
1661
1662         let pending_payments = nodes[0].node.list_recent_payments();
1663         assert_eq!(pending_payments.len(), 0);
1664 }
1665
1666 #[test]
1667 fn test_holding_cell_inflight_htlcs() {
1668         let chanmon_cfgs = create_chanmon_cfgs(2);
1669         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1670         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1671         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1672         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1673
1674         let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
1675         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1676
1677         // Queue up two payments - one will be delivered right away, one immediately goes into the
1678         // holding cell as nodes[0] is AwaitingRAA.
1679         {
1680                 nodes[0].node.send_payment_with_route(&route, payment_hash_1,
1681                         RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
1682                 check_added_monitors!(nodes[0], 1);
1683                 nodes[0].node.send_payment_with_route(&route, payment_hash_2,
1684                         RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
1685                 check_added_monitors!(nodes[0], 0);
1686         }
1687
1688         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1689
1690         {
1691                 let mut node_0_per_peer_lock;
1692                 let mut node_0_peer_state_lock;
1693                 let channel =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, channel_id);
1694
1695                 let used_liquidity = inflight_htlcs.used_liquidity_msat(
1696                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1697                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1698                         channel.context().get_short_channel_id().unwrap()
1699                 );
1700
1701                 assert_eq!(used_liquidity, Some(2000000));
1702         }
1703
1704         // Clear pending events so test doesn't throw a "Had excess message on node..." error
1705         nodes[0].node.get_and_clear_pending_msg_events();
1706 }
1707
1708 #[test]
1709 fn intercepted_payment() {
1710         // Test that detecting an intercept scid on payment forward will signal LDK to generate an
1711         // intercept event, which the LSP can then use to either (a) open a JIT channel to forward the
1712         // payment or (b) fail the payment.
1713         do_test_intercepted_payment(InterceptTest::Forward);
1714         do_test_intercepted_payment(InterceptTest::Fail);
1715         // Make sure that intercepted payments will be automatically failed back if too many blocks pass.
1716         do_test_intercepted_payment(InterceptTest::Timeout);
1717 }
1718
1719 fn do_test_intercepted_payment(test: InterceptTest) {
1720         let chanmon_cfgs = create_chanmon_cfgs(3);
1721         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1722
1723         let mut zero_conf_chan_config = test_default_channel_config();
1724         zero_conf_chan_config.manually_accept_inbound_channels = true;
1725         let mut intercept_forwards_config = test_default_channel_config();
1726         intercept_forwards_config.accept_intercept_htlcs = true;
1727         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(zero_conf_chan_config)]);
1728
1729         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1730         let scorer = test_utils::TestScorer::new();
1731         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
1732
1733         let _ = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1734
1735         let amt_msat = 100_000;
1736         let intercept_scid = nodes[1].node.get_intercept_scid();
1737         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1738                 .with_route_hints(vec![
1739                         RouteHint(vec![RouteHintHop {
1740                                 src_node_id: nodes[1].node.get_our_node_id(),
1741                                 short_channel_id: intercept_scid,
1742                                 fees: RoutingFees {
1743                                         base_msat: 1000,
1744                                         proportional_millionths: 0,
1745                                 },
1746                                 cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
1747                                 htlc_minimum_msat: None,
1748                                 htlc_maximum_msat: None,
1749                         }])
1750                 ]).unwrap()
1751                 .with_bolt11_features(nodes[2].node.invoice_features()).unwrap();
1752         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat,);
1753         let route = get_route( &nodes[0].node.get_our_node_id(), &route_params,
1754                 &nodes[0].network_graph.read_only(), None, nodes[0].logger, &scorer, &(),
1755                 &random_seed_bytes,).unwrap();
1756
1757         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
1758         nodes[0].node.send_payment_with_route(&route, payment_hash,
1759                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
1760         let payment_event = {
1761                 {
1762                         let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
1763                         assert_eq!(added_monitors.len(), 1);
1764                         added_monitors.clear();
1765                 }
1766                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1767                 assert_eq!(events.len(), 1);
1768                 SendEvent::from_event(events.remove(0))
1769         };
1770         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1771         commitment_signed_dance!(nodes[1], nodes[0], &payment_event.commitment_msg, false, true);
1772
1773         // Check that we generate the PaymentIntercepted event when an intercept forward is detected.
1774         let events = nodes[1].node.get_and_clear_pending_events();
1775         assert_eq!(events.len(), 1);
1776         let (intercept_id, expected_outbound_amount_msat) = match events[0] {
1777                 crate::events::Event::HTLCIntercepted {
1778                         intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, inbound_amount_msat, requested_next_hop_scid: short_channel_id
1779                 } => {
1780                         assert_eq!(pmt_hash, payment_hash);
1781                         assert_eq!(inbound_amount_msat, route.get_total_amount() + route.get_total_fees());
1782                         assert_eq!(short_channel_id, intercept_scid);
1783                         (intercept_id, expected_outbound_amount_msat)
1784                 },
1785                 _ => panic!()
1786         };
1787
1788         // Check for unknown channel id error.
1789         let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &ChannelId::from_bytes([42; 32]), nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1790         assert_eq!(unknown_chan_id_err , APIError::ChannelUnavailable  {
1791                 err: format!("Channel with id {} not found for the passed counterparty node_id {}.",
1792                         log_bytes!([42; 32]), nodes[2].node.get_our_node_id()) });
1793
1794         if test == InterceptTest::Fail {
1795                 // Ensure we can fail the intercepted payment back.
1796                 nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap();
1797                 expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::UnknownNextHop { requested_forward_scid: intercept_scid }]);
1798                 nodes[1].node.process_pending_htlc_forwards();
1799                 let update_fail = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1800                 check_added_monitors!(&nodes[1], 1);
1801                 assert!(update_fail.update_fail_htlcs.len() == 1);
1802                 let fail_msg = update_fail.update_fail_htlcs[0].clone();
1803                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1804                 commitment_signed_dance!(nodes[0], nodes[1], update_fail.commitment_signed, false);
1805
1806                 // Ensure the payment fails with the expected error.
1807                 let fail_conditions = PaymentFailedConditions::new()
1808                         .blamed_scid(intercept_scid)
1809                         .blamed_chan_closed(true)
1810                         .expected_htlc_error_data(0x4000 | 10, &[]);
1811                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
1812         } else if test == InterceptTest::Forward {
1813                 // Check that we'll fail as expected when sending to a channel that isn't in `ChannelReady` yet.
1814                 let temp_chan_id = nodes[1].node.create_channel(nodes[2].node.get_our_node_id(), 100_000, 0, 42, None).unwrap();
1815                 let unusable_chan_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &temp_chan_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1816                 assert_eq!(unusable_chan_err , APIError::ChannelUnavailable {
1817                         err: format!("Channel with id {} for the passed counterparty node_id {} is still opening.",
1818                                 temp_chan_id, nodes[2].node.get_our_node_id()) });
1819                 assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
1820
1821                 // Open the just-in-time channel so the payment can then be forwarded.
1822                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1823
1824                 // Finally, forward the intercepted payment through and claim it.
1825                 nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap();
1826                 expect_pending_htlcs_forwardable!(nodes[1]);
1827
1828                 let payment_event = {
1829                         {
1830                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
1831                                 assert_eq!(added_monitors.len(), 1);
1832                                 added_monitors.clear();
1833                         }
1834                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
1835                         assert_eq!(events.len(), 1);
1836                         SendEvent::from_event(events.remove(0))
1837                 };
1838                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1839                 commitment_signed_dance!(nodes[2], nodes[1], &payment_event.commitment_msg, false, true);
1840                 expect_pending_htlcs_forwardable!(nodes[2]);
1841
1842                 let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1843                 expect_payment_claimable!(&nodes[2], payment_hash, payment_secret, amt_msat, Some(payment_preimage), nodes[2].node.get_our_node_id());
1844                 do_claim_payment_along_route(&nodes[0], &vec!(&vec!(&nodes[1], &nodes[2])[..]), false, payment_preimage);
1845                 let events = nodes[0].node.get_and_clear_pending_events();
1846                 assert_eq!(events.len(), 2);
1847                 match events[0] {
1848                         Event::PaymentSent { payment_preimage: ref ev_preimage, payment_hash: ref ev_hash, ref fee_paid_msat, .. } => {
1849                                 assert_eq!(payment_preimage, *ev_preimage);
1850                                 assert_eq!(payment_hash, *ev_hash);
1851                                 assert_eq!(fee_paid_msat, &Some(1000));
1852                         },
1853                         _ => panic!("Unexpected event")
1854                 }
1855                 match events[1] {
1856                         Event::PaymentPathSuccessful { payment_hash: hash, .. } => {
1857                                 assert_eq!(hash, Some(payment_hash));
1858                         },
1859                         _ => panic!("Unexpected event")
1860                 }
1861                 check_added_monitors(&nodes[0], 1);
1862         } else if test == InterceptTest::Timeout {
1863                 let mut block = create_dummy_block(nodes[0].best_block_hash(), 42, Vec::new());
1864                 connect_block(&nodes[0], &block);
1865                 connect_block(&nodes[1], &block);
1866                 for _ in 0..TEST_FINAL_CLTV {
1867                         block.header.prev_blockhash = block.block_hash();
1868                         connect_block(&nodes[0], &block);
1869                         connect_block(&nodes[1], &block);
1870                 }
1871                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::InvalidForward { requested_forward_scid: intercept_scid }]);
1872                 check_added_monitors!(nodes[1], 1);
1873                 let htlc_timeout_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1874                 assert!(htlc_timeout_updates.update_add_htlcs.is_empty());
1875                 assert_eq!(htlc_timeout_updates.update_fail_htlcs.len(), 1);
1876                 assert!(htlc_timeout_updates.update_fail_malformed_htlcs.is_empty());
1877                 assert!(htlc_timeout_updates.update_fee.is_none());
1878
1879                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_timeout_updates.update_fail_htlcs[0]);
1880                 commitment_signed_dance!(nodes[0], nodes[1], htlc_timeout_updates.commitment_signed, false);
1881                 expect_payment_failed!(nodes[0], payment_hash, false, 0x2000 | 2, []);
1882
1883                 // Check for unknown intercept id error.
1884                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1885                 let unknown_intercept_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1886                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1887                 let unknown_intercept_id_err = nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap_err();
1888                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
1889         }
1890 }
1891
1892 #[test]
1893 fn accept_underpaying_htlcs_config() {
1894         do_accept_underpaying_htlcs_config(1);
1895         do_accept_underpaying_htlcs_config(2);
1896         do_accept_underpaying_htlcs_config(3);
1897 }
1898
1899 fn do_accept_underpaying_htlcs_config(num_mpp_parts: usize) {
1900         let chanmon_cfgs = create_chanmon_cfgs(3);
1901         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1902         let mut intercept_forwards_config = test_default_channel_config();
1903         intercept_forwards_config.accept_intercept_htlcs = true;
1904         let mut underpay_config = test_default_channel_config();
1905         underpay_config.channel_config.accept_underpaying_htlcs = true;
1906         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(underpay_config)]);
1907         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1908
1909         let mut chan_ids = Vec::new();
1910         for _ in 0..num_mpp_parts {
1911                 let _ = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000, 0);
1912                 let channel_id = create_unannounced_chan_between_nodes_with_value(&nodes, 1, 2, 2_000_000, 0).0.channel_id;
1913                 chan_ids.push(channel_id);
1914         }
1915
1916         // Send the initial payment.
1917         let amt_msat = 900_000;
1918         let skimmed_fee_msat = 20;
1919         let mut route_hints = Vec::new();
1920         for _ in 0..num_mpp_parts {
1921                 route_hints.push(RouteHint(vec![RouteHintHop {
1922                         src_node_id: nodes[1].node.get_our_node_id(),
1923                         short_channel_id: nodes[1].node.get_intercept_scid(),
1924                         fees: RoutingFees {
1925                                 base_msat: 1000,
1926                                 proportional_millionths: 0,
1927                         },
1928                         cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
1929                         htlc_minimum_msat: None,
1930                         htlc_maximum_msat: Some(amt_msat / num_mpp_parts as u64 + 5),
1931                 }]));
1932         }
1933         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1934                 .with_route_hints(route_hints).unwrap()
1935                 .with_bolt11_features(nodes[2].node.invoice_features()).unwrap();
1936         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
1937         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
1938         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
1939                 PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
1940         check_added_monitors!(nodes[0], num_mpp_parts); // one monitor per path
1941         let mut events: Vec<SendEvent> = nodes[0].node.get_and_clear_pending_msg_events().into_iter().map(|e| SendEvent::from_event(e)).collect();
1942         assert_eq!(events.len(), num_mpp_parts);
1943
1944         // Forward the intercepted payments.
1945         for (idx, ev) in events.into_iter().enumerate() {
1946                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &ev.msgs[0]);
1947                 do_commitment_signed_dance(&nodes[1], &nodes[0], &ev.commitment_msg, false, true);
1948
1949                 let events = nodes[1].node.get_and_clear_pending_events();
1950                 assert_eq!(events.len(), 1);
1951                 let (intercept_id, expected_outbound_amt_msat) = match events[0] {
1952                         crate::events::Event::HTLCIntercepted {
1953                                 intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, ..
1954                         } => {
1955                                 assert_eq!(pmt_hash, payment_hash);
1956                                 (intercept_id, expected_outbound_amount_msat)
1957                         },
1958                         _ => panic!()
1959                 };
1960                 nodes[1].node.forward_intercepted_htlc(intercept_id, &chan_ids[idx],
1961                         nodes[2].node.get_our_node_id(), expected_outbound_amt_msat - skimmed_fee_msat).unwrap();
1962                 expect_pending_htlcs_forwardable!(nodes[1]);
1963                 let payment_event = {
1964                         {
1965                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
1966                                 assert_eq!(added_monitors.len(), 1);
1967                                 added_monitors.clear();
1968                         }
1969                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
1970                         assert_eq!(events.len(), 1);
1971                         SendEvent::from_event(events.remove(0))
1972                 };
1973                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1974                 do_commitment_signed_dance(&nodes[2], &nodes[1], &payment_event.commitment_msg, false, true);
1975                 if idx == num_mpp_parts - 1 {
1976                         expect_pending_htlcs_forwardable!(nodes[2]);
1977                 }
1978         }
1979
1980         // Claim the payment and check that the skimmed fee is as expected.
1981         let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1982         let events = nodes[2].node.get_and_clear_pending_events();
1983         assert_eq!(events.len(), 1);
1984         match events[0] {
1985                 crate::events::Event::PaymentClaimable {
1986                         ref payment_hash, ref purpose, amount_msat, counterparty_skimmed_fee_msat, receiver_node_id, ..
1987                 } => {
1988                         assert_eq!(payment_hash, payment_hash);
1989                         assert_eq!(amt_msat - skimmed_fee_msat * num_mpp_parts as u64, amount_msat);
1990                         assert_eq!(skimmed_fee_msat * num_mpp_parts as u64, counterparty_skimmed_fee_msat);
1991                         assert_eq!(nodes[2].node.get_our_node_id(), receiver_node_id.unwrap());
1992                         match purpose {
1993                                 crate::events::PaymentPurpose::InvoicePayment { payment_preimage: ev_payment_preimage,
1994                                         payment_secret: ev_payment_secret, .. } =>
1995                                 {
1996                                         assert_eq!(payment_preimage, ev_payment_preimage.unwrap());
1997                                         assert_eq!(payment_secret, *ev_payment_secret);
1998                                 },
1999                                 _ => panic!(),
2000                         }
2001                 },
2002                 _ => panic!("Unexpected event"),
2003         }
2004         let mut expected_paths_vecs = Vec::new();
2005         let mut expected_paths = Vec::new();
2006         for _ in 0..num_mpp_parts { expected_paths_vecs.push(vec!(&nodes[1], &nodes[2])); }
2007         for i in 0..num_mpp_parts { expected_paths.push(&expected_paths_vecs[i][..]); }
2008         let total_fee_msat = do_claim_payment_along_route_with_extra_penultimate_hop_fees(
2009                 &nodes[0], &expected_paths[..], &vec![skimmed_fee_msat as u32; num_mpp_parts][..], false,
2010                 payment_preimage);
2011         // The sender doesn't know that the penultimate hop took an extra fee.
2012         expect_payment_sent(&nodes[0], payment_preimage,
2013                 Some(Some(total_fee_msat - skimmed_fee_msat * num_mpp_parts as u64)), true, true);
2014 }
2015
2016 #[derive(PartialEq)]
2017 enum AutoRetry {
2018         Success,
2019         Spontaneous,
2020         FailAttempts,
2021         FailTimeout,
2022         FailOnRestart,
2023         FailOnRetry,
2024 }
2025
2026 #[test]
2027 fn automatic_retries() {
2028         do_automatic_retries(AutoRetry::Success);
2029         do_automatic_retries(AutoRetry::Spontaneous);
2030         do_automatic_retries(AutoRetry::FailAttempts);
2031         do_automatic_retries(AutoRetry::FailTimeout);
2032         do_automatic_retries(AutoRetry::FailOnRestart);
2033         do_automatic_retries(AutoRetry::FailOnRetry);
2034 }
2035 fn do_automatic_retries(test: AutoRetry) {
2036         // Test basic automatic payment retries in ChannelManager. See individual `test` variant comments
2037         // below.
2038         let chanmon_cfgs = create_chanmon_cfgs(3);
2039         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2040         let persister;
2041         let new_chain_monitor;
2042
2043         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2044         let node_0_deserialized;
2045
2046         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2047         let channel_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
2048         let channel_id_2 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
2049
2050         // Marshall data to send the payment
2051         #[cfg(feature = "std")]
2052         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2053         #[cfg(not(feature = "std"))]
2054         let payment_expiry_secs = 60 * 60;
2055         let amt_msat = 1000;
2056         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2057         invoice_features.set_variable_length_onion_required();
2058         invoice_features.set_payment_secret_required();
2059         invoice_features.set_basic_mpp_optional();
2060         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
2061                 .with_expiry_time(payment_expiry_secs as u64)
2062                 .with_bolt11_features(invoice_features).unwrap();
2063         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2064         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
2065
2066         macro_rules! pass_failed_attempt_with_retry_along_path {
2067                 ($failing_channel_id: expr, $expect_pending_htlcs_forwardable: expr) => {
2068                         // Send a payment attempt that fails due to lack of liquidity on the second hop
2069                         check_added_monitors!(nodes[0], 1);
2070                         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2071                         let mut update_add = update_0.update_add_htlcs[0].clone();
2072                         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
2073                         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
2074                         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
2075                         nodes[1].node.process_pending_htlc_forwards();
2076                         expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1],
2077                                 vec![HTLCDestination::NextHopChannel {
2078                                         node_id: Some(nodes[2].node.get_our_node_id()),
2079                                         channel_id: $failing_channel_id,
2080                                 }]);
2081                         nodes[1].node.process_pending_htlc_forwards();
2082                         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2083                         check_added_monitors!(&nodes[1], 1);
2084                         assert!(update_1.update_fail_htlcs.len() == 1);
2085                         let fail_msg = update_1.update_fail_htlcs[0].clone();
2086                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
2087                         commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
2088
2089                         // Ensure the attempt fails and a new PendingHTLCsForwardable event is generated for the retry
2090                         let mut events = nodes[0].node.get_and_clear_pending_events();
2091                         assert_eq!(events.len(), 2);
2092                         match events[0] {
2093                                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2094                                         assert_eq!(payment_hash, ev_payment_hash);
2095                                         assert_eq!(payment_failed_permanently, false);
2096                                 },
2097                                 _ => panic!("Unexpected event"),
2098                         }
2099                         if $expect_pending_htlcs_forwardable {
2100                                 match events[1] {
2101                                         Event::PendingHTLCsForwardable { .. } => {},
2102                                         _ => panic!("Unexpected event"),
2103                                 }
2104                         } else {
2105                                 match events[1] {
2106                                         Event::PaymentFailed { payment_hash: ev_payment_hash, .. } => {
2107                                                 assert_eq!(payment_hash, ev_payment_hash);
2108                                         },
2109                                         _ => panic!("Unexpected event"),
2110                                 }
2111                         }
2112                 }
2113         }
2114
2115         if test == AutoRetry::Success {
2116                 // Test that we can succeed on the first retry.
2117                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2118                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2119                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2120
2121                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
2122                 // attempt, since the initial second hop channel will be excluded from pathfinding
2123                 create_announced_chan_between_nodes(&nodes, 1, 2);
2124
2125                 // We retry payments in `process_pending_htlc_forwards`
2126                 nodes[0].node.process_pending_htlc_forwards();
2127                 check_added_monitors!(nodes[0], 1);
2128                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2129                 assert_eq!(msg_events.len(), 1);
2130                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, Some(payment_secret), msg_events.pop().unwrap(), true, None);
2131                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
2132         } else if test == AutoRetry::Spontaneous {
2133                 nodes[0].node.send_spontaneous_payment_with_retry(Some(payment_preimage),
2134                         RecipientOnionFields::spontaneous_empty(), PaymentId(payment_hash.0), route_params,
2135                         Retry::Attempts(1)).unwrap();
2136                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2137
2138                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
2139                 // attempt, since the initial second hop channel will be excluded from pathfinding
2140                 create_announced_chan_between_nodes(&nodes, 1, 2);
2141
2142                 // We retry payments in `process_pending_htlc_forwards`
2143                 nodes[0].node.process_pending_htlc_forwards();
2144                 check_added_monitors!(nodes[0], 1);
2145                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2146                 assert_eq!(msg_events.len(), 1);
2147                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, None, msg_events.pop().unwrap(), true, Some(payment_preimage));
2148                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
2149         } else if test == AutoRetry::FailAttempts {
2150                 // Ensure ChannelManager will not retry a payment if it has run out of payment attempts.
2151                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2152                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2153                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2154
2155                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
2156                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
2157                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
2158
2159                 // We retry payments in `process_pending_htlc_forwards`
2160                 nodes[0].node.process_pending_htlc_forwards();
2161                 pass_failed_attempt_with_retry_along_path!(channel_id_3, false);
2162
2163                 // Ensure we won't retry a second time.
2164                 nodes[0].node.process_pending_htlc_forwards();
2165                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2166                 assert_eq!(msg_events.len(), 0);
2167         } else if test == AutoRetry::FailTimeout {
2168                 #[cfg(not(feature = "no-std"))] {
2169                         // Ensure ChannelManager will not retry a payment if it times out due to Retry::Timeout.
2170                         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2171                                 PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
2172                         pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2173
2174                         // Advance the time so the second attempt fails due to timeout.
2175                         SinceEpoch::advance(Duration::from_secs(61));
2176
2177                         // Make sure we don't retry again.
2178                         nodes[0].node.process_pending_htlc_forwards();
2179                         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2180                         assert_eq!(msg_events.len(), 0);
2181
2182                         let mut events = nodes[0].node.get_and_clear_pending_events();
2183                         assert_eq!(events.len(), 1);
2184                         match events[0] {
2185                                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2186                                         assert_eq!(payment_hash, *ev_payment_hash);
2187                                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2188                                         assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
2189                                 },
2190                                 _ => panic!("Unexpected event"),
2191                         }
2192                 }
2193         } else if test == AutoRetry::FailOnRestart {
2194                 // Ensure ChannelManager will not retry a payment after restart, even if there were retry
2195                 // attempts remaining prior to restart.
2196                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2197                         PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
2198                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2199
2200                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
2201                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
2202                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
2203
2204                 // Ensure the first retry attempt fails, with 1 retry attempt remaining
2205                 nodes[0].node.process_pending_htlc_forwards();
2206                 pass_failed_attempt_with_retry_along_path!(channel_id_3, true);
2207
2208                 // Restart the node and ensure that ChannelManager does not use its remaining retry attempt
2209                 let node_encoded = nodes[0].node.encode();
2210                 let chan_1_monitor_serialized = get_monitor!(nodes[0], channel_id_1).encode();
2211                 reload_node!(nodes[0], node_encoded, &[&chan_1_monitor_serialized], persister, new_chain_monitor, node_0_deserialized);
2212
2213                 let mut events = nodes[0].node.get_and_clear_pending_events();
2214                 expect_pending_htlcs_forwardable_from_events!(nodes[0], events, true);
2215                 // Make sure we don't retry again.
2216                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2217                 assert_eq!(msg_events.len(), 0);
2218
2219                 let mut events = nodes[0].node.get_and_clear_pending_events();
2220                 assert_eq!(events.len(), 1);
2221                 match events[0] {
2222                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2223                                 assert_eq!(payment_hash, *ev_payment_hash);
2224                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2225                                 assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
2226                         },
2227                         _ => panic!("Unexpected event"),
2228                 }
2229         } else if test == AutoRetry::FailOnRetry {
2230                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2231                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2232                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2233
2234                 // We retry payments in `process_pending_htlc_forwards`. Since our channel closed, we should
2235                 // fail to find a route.
2236                 nodes[0].node.process_pending_htlc_forwards();
2237                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2238                 assert_eq!(msg_events.len(), 0);
2239
2240                 let mut events = nodes[0].node.get_and_clear_pending_events();
2241                 assert_eq!(events.len(), 1);
2242                 match events[0] {
2243                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2244                                 assert_eq!(payment_hash, *ev_payment_hash);
2245                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2246                                 assert_eq!(PaymentFailureReason::RouteNotFound, ev_reason.unwrap());
2247                         },
2248                         _ => panic!("Unexpected event"),
2249                 }
2250         }
2251 }
2252
2253 #[test]
2254 fn auto_retry_partial_failure() {
2255         // Test that we'll retry appropriately on send partial failure and retry partial failure.
2256         let chanmon_cfgs = create_chanmon_cfgs(2);
2257         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2258         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2259         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2260
2261         // Open three channels, the first has plenty of liquidity, the second and third have ~no
2262         // available liquidity, causing any outbound payments routed over it to fail immediately.
2263         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2264         let chan_2_id = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 989_000_000).0.contents.short_channel_id;
2265         let chan_3_id = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 989_000_000).0.contents.short_channel_id;
2266
2267         // Marshall data to send the payment
2268         let amt_msat = 10_000_000;
2269         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2270         #[cfg(feature = "std")]
2271         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2272         #[cfg(not(feature = "std"))]
2273         let payment_expiry_secs = 60 * 60;
2274         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2275         invoice_features.set_variable_length_onion_required();
2276         invoice_features.set_payment_secret_required();
2277         invoice_features.set_basic_mpp_optional();
2278         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2279                 .with_expiry_time(payment_expiry_secs as u64)
2280                 .with_bolt11_features(invoice_features).unwrap();
2281         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2282
2283         // Configure the initial send, retry1 and retry2's paths.
2284         let send_route = Route {
2285                 paths: vec![
2286                         Path { hops: vec![RouteHop {
2287                                 pubkey: nodes[1].node.get_our_node_id(),
2288                                 node_features: nodes[1].node.node_features(),
2289                                 short_channel_id: chan_1_id,
2290                                 channel_features: nodes[1].node.channel_features(),
2291                                 fee_msat: amt_msat / 2,
2292                                 cltv_expiry_delta: 100,
2293                                 maybe_announced_channel: true,
2294                         }], blinded_tail: None },
2295                         Path { hops: vec![RouteHop {
2296                                 pubkey: nodes[1].node.get_our_node_id(),
2297                                 node_features: nodes[1].node.node_features(),
2298                                 short_channel_id: chan_2_id,
2299                                 channel_features: nodes[1].node.channel_features(),
2300                                 fee_msat: amt_msat / 2,
2301                                 cltv_expiry_delta: 100,
2302                                 maybe_announced_channel: true,
2303                         }], blinded_tail: None },
2304                 ],
2305                 route_params: Some(route_params.clone()),
2306         };
2307         let retry_1_route = Route {
2308                 paths: vec![
2309                         Path { hops: vec![RouteHop {
2310                                 pubkey: nodes[1].node.get_our_node_id(),
2311                                 node_features: nodes[1].node.node_features(),
2312                                 short_channel_id: chan_1_id,
2313                                 channel_features: nodes[1].node.channel_features(),
2314                                 fee_msat: amt_msat / 4,
2315                                 cltv_expiry_delta: 100,
2316                                 maybe_announced_channel: true,
2317                         }], blinded_tail: None },
2318                         Path { hops: vec![RouteHop {
2319                                 pubkey: nodes[1].node.get_our_node_id(),
2320                                 node_features: nodes[1].node.node_features(),
2321                                 short_channel_id: chan_3_id,
2322                                 channel_features: nodes[1].node.channel_features(),
2323                                 fee_msat: amt_msat / 4,
2324                                 cltv_expiry_delta: 100,
2325                                 maybe_announced_channel: true,
2326                         }], blinded_tail: None },
2327                 ],
2328                 route_params: Some(route_params.clone()),
2329         };
2330         let retry_2_route = Route {
2331                 paths: vec![
2332                         Path { hops: vec![RouteHop {
2333                                 pubkey: nodes[1].node.get_our_node_id(),
2334                                 node_features: nodes[1].node.node_features(),
2335                                 short_channel_id: chan_1_id,
2336                                 channel_features: nodes[1].node.channel_features(),
2337                                 fee_msat: amt_msat / 4,
2338                                 cltv_expiry_delta: 100,
2339                                 maybe_announced_channel: true,
2340                         }], blinded_tail: None },
2341                 ],
2342                 route_params: Some(route_params.clone()),
2343         };
2344         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
2345         let mut payment_params = route_params.payment_params.clone();
2346         payment_params.previously_failed_channels.push(chan_2_id);
2347         nodes[0].router.expect_find_route(
2348                 RouteParameters::from_payment_params_and_value(payment_params, amt_msat / 2),
2349                 Ok(retry_1_route));
2350         let mut payment_params = route_params.payment_params.clone();
2351         payment_params.previously_failed_channels.push(chan_3_id);
2352         nodes[0].router.expect_find_route(
2353                 RouteParameters::from_payment_params_and_value(payment_params, amt_msat / 4),
2354                 Ok(retry_2_route));
2355
2356         // Send a payment that will partially fail on send, then partially fail on retry, then succeed.
2357         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2358                 PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap();
2359         let payment_failed_events = nodes[0].node.get_and_clear_pending_events();
2360         assert_eq!(payment_failed_events.len(), 2);
2361         match payment_failed_events[0] {
2362                 Event::PaymentPathFailed { .. } => {},
2363                 _ => panic!("Unexpected event"),
2364         }
2365         match payment_failed_events[1] {
2366                 Event::PaymentPathFailed { .. } => {},
2367                 _ => panic!("Unexpected event"),
2368         }
2369
2370         // Pass the first part of the payment along the path.
2371         check_added_monitors!(nodes[0], 1); // only one HTLC actually made it out
2372         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2373
2374         // Only one HTLC/channel update actually made it out
2375         assert_eq!(msg_events.len(), 1);
2376         let mut payment_event = SendEvent::from_event(msg_events.remove(0));
2377
2378         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2379         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
2380         check_added_monitors!(nodes[1], 1);
2381         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2382
2383         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2384         check_added_monitors!(nodes[0], 1);
2385         let as_second_htlc_updates = SendEvent::from_node(&nodes[0]);
2386
2387         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2388         check_added_monitors!(nodes[0], 1);
2389         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2390
2391         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2392         check_added_monitors!(nodes[1], 1);
2393
2394         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[0]);
2395         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[1]);
2396         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.commitment_msg);
2397         check_added_monitors!(nodes[1], 1);
2398         let (bs_second_raa, bs_second_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2399
2400         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2401         check_added_monitors!(nodes[0], 1);
2402
2403         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_cs);
2404         check_added_monitors!(nodes[0], 1);
2405         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2406
2407         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2408         check_added_monitors!(nodes[1], 1);
2409
2410         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
2411         nodes[1].node.process_pending_htlc_forwards();
2412         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, amt_msat);
2413         nodes[1].node.claim_funds(payment_preimage);
2414         expect_payment_claimed!(nodes[1], payment_hash, amt_msat);
2415         let bs_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2416         assert_eq!(bs_claim_update.update_fulfill_htlcs.len(), 1);
2417
2418         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_claim_update.update_fulfill_htlcs[0]);
2419         expect_payment_sent(&nodes[0], payment_preimage, None, false, false);
2420         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_claim_update.commitment_signed);
2421         check_added_monitors!(nodes[0], 1);
2422         let (as_third_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2423
2424         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2425         check_added_monitors!(nodes[1], 4);
2426         let bs_second_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2427
2428         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2429         check_added_monitors!(nodes[1], 1);
2430         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2431
2432         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2433         check_added_monitors!(nodes[0], 1);
2434         expect_payment_path_successful!(nodes[0]);
2435
2436         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[0]);
2437         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[1]);
2438         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.commitment_signed);
2439         check_added_monitors!(nodes[0], 1);
2440         let (as_fourth_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2441
2442         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_fourth_raa);
2443         check_added_monitors!(nodes[1], 1);
2444
2445         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2446         check_added_monitors!(nodes[1], 1);
2447         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2448
2449         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2450         check_added_monitors!(nodes[0], 1);
2451         let events = nodes[0].node.get_and_clear_pending_events();
2452         assert_eq!(events.len(), 2);
2453         if let Event::PaymentPathSuccessful { .. } = events[0] {} else { panic!(); }
2454         if let Event::PaymentPathSuccessful { .. } = events[1] {} else { panic!(); }
2455 }
2456
2457 #[test]
2458 fn auto_retry_zero_attempts_send_error() {
2459         let chanmon_cfgs = create_chanmon_cfgs(2);
2460         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2461         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2462         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2463
2464         // Open a single channel that does not have sufficient liquidity for the payment we want to
2465         // send.
2466         let chan_id  = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 989_000_000).0.contents.short_channel_id;
2467
2468         // Marshall data to send the payment
2469         let amt_msat = 10_000_000;
2470         let (_, payment_hash, payment_secret) = get_payment_preimage_hash(&nodes[1], Some(amt_msat), None);
2471         #[cfg(feature = "std")]
2472         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2473         #[cfg(not(feature = "std"))]
2474         let payment_expiry_secs = 60 * 60;
2475         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2476         invoice_features.set_variable_length_onion_required();
2477         invoice_features.set_payment_secret_required();
2478         invoice_features.set_basic_mpp_optional();
2479         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2480                 .with_expiry_time(payment_expiry_secs as u64)
2481                 .with_bolt11_features(invoice_features).unwrap();
2482         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2483
2484         // Override the route search to return a route, rather than failing at the route-finding step.
2485         let send_route = Route {
2486                 paths: vec![
2487                         Path { hops: vec![RouteHop {
2488                                 pubkey: nodes[1].node.get_our_node_id(),
2489                                 node_features: nodes[1].node.node_features(),
2490                                 short_channel_id: chan_id,
2491                                 channel_features: nodes[1].node.channel_features(),
2492                                 fee_msat: amt_msat,
2493                                 cltv_expiry_delta: 100,
2494                                 maybe_announced_channel: true,
2495                         }], blinded_tail: None },
2496                 ],
2497                 route_params: Some(route_params.clone()),
2498         };
2499         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
2500
2501         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2502                 PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
2503         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
2504         let events = nodes[0].node.get_and_clear_pending_events();
2505         assert_eq!(events.len(), 2);
2506         if let Event::PaymentPathFailed { .. } = events[0] { } else { panic!(); }
2507         if let Event::PaymentFailed { .. } = events[1] { } else { panic!(); }
2508         check_added_monitors!(nodes[0], 0);
2509 }
2510
2511 #[test]
2512 fn fails_paying_after_rejected_by_payee() {
2513         let chanmon_cfgs = create_chanmon_cfgs(2);
2514         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2515         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2516         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2517
2518         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2519
2520         // Marshall data to send the payment
2521         let amt_msat = 20_000;
2522         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2523         #[cfg(feature = "std")]
2524         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2525         #[cfg(not(feature = "std"))]
2526         let payment_expiry_secs = 60 * 60;
2527         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2528         invoice_features.set_variable_length_onion_required();
2529         invoice_features.set_payment_secret_required();
2530         invoice_features.set_basic_mpp_optional();
2531         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2532                 .with_expiry_time(payment_expiry_secs as u64)
2533                 .with_bolt11_features(invoice_features).unwrap();
2534         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2535
2536         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2537                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2538         check_added_monitors!(nodes[0], 1);
2539         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
2540         assert_eq!(events.len(), 1);
2541         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
2542         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2543         check_added_monitors!(nodes[1], 0);
2544         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
2545         expect_pending_htlcs_forwardable!(nodes[1]);
2546         expect_payment_claimable!(&nodes[1], payment_hash, payment_secret, amt_msat);
2547
2548         nodes[1].node.fail_htlc_backwards(&payment_hash);
2549         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash }]);
2550         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
2551 }
2552
2553 #[test]
2554 fn retry_multi_path_single_failed_payment() {
2555         // Tests that we can/will retry after a single path of an MPP payment failed immediately
2556         let chanmon_cfgs = create_chanmon_cfgs(2);
2557         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2558         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2559         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2560
2561         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2562         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2563
2564         let amt_msat = 100_010_000;
2565
2566         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2567         #[cfg(feature = "std")]
2568         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2569         #[cfg(not(feature = "std"))]
2570         let payment_expiry_secs = 60 * 60;
2571         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2572         invoice_features.set_variable_length_onion_required();
2573         invoice_features.set_payment_secret_required();
2574         invoice_features.set_basic_mpp_optional();
2575         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2576                 .with_expiry_time(payment_expiry_secs as u64)
2577                 .with_bolt11_features(invoice_features).unwrap();
2578         let route_params = RouteParameters::from_payment_params_and_value(
2579                 payment_params.clone(), amt_msat);
2580
2581         let chans = nodes[0].node.list_usable_channels();
2582         let mut route = Route {
2583                 paths: vec![
2584                         Path { hops: vec![RouteHop {
2585                                 pubkey: nodes[1].node.get_our_node_id(),
2586                                 node_features: nodes[1].node.node_features(),
2587                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2588                                 channel_features: nodes[1].node.channel_features(),
2589                                 fee_msat: 10_000,
2590                                 cltv_expiry_delta: 100,
2591                                 maybe_announced_channel: true,
2592                         }], blinded_tail: None },
2593                         Path { hops: vec![RouteHop {
2594                                 pubkey: nodes[1].node.get_our_node_id(),
2595                                 node_features: nodes[1].node.node_features(),
2596                                 short_channel_id: chans[1].short_channel_id.unwrap(),
2597                                 channel_features: nodes[1].node.channel_features(),
2598                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2599                                 cltv_expiry_delta: 100,
2600                                 maybe_announced_channel: true,
2601                         }], blinded_tail: None },
2602                 ],
2603                 route_params: Some(route_params.clone()),
2604         };
2605         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2606         // On retry, split the payment across both channels.
2607         route.paths[0].hops[0].fee_msat = 50_000_001;
2608         route.paths[1].hops[0].fee_msat = 50_000_000;
2609         let mut pay_params = route.route_params.clone().unwrap().payment_params;
2610         pay_params.previously_failed_channels.push(chans[1].short_channel_id.unwrap());
2611         nodes[0].router.expect_find_route(
2612                 // Note that the second request here requests the amount we originally failed to send,
2613                 // not the amount remaining on the full payment, which should be changed.
2614                 RouteParameters::from_payment_params_and_value(pay_params, 100_000_001),
2615                 Ok(route.clone()));
2616
2617         {
2618                 let scorer = chanmon_cfgs[0].scorer.read().unwrap();
2619                 // The initial send attempt, 2 paths
2620                 scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 10_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2621                 scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 100_000_001, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2622                 // The retry, 2 paths. Ensure that the in-flight HTLC amount is factored in.
2623                 scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_001, inflight_htlc_msat: 10_000, effective_capacity: EffectiveCapacity::Unknown });
2624                 scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2625         }
2626
2627         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2628                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2629         let events = nodes[0].node.get_and_clear_pending_events();
2630         assert_eq!(events.len(), 1);
2631         match events[0] {
2632                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false,
2633                         failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }},
2634                         short_channel_id: Some(expected_scid), .. } =>
2635                 {
2636                         assert_eq!(payment_hash, ev_payment_hash);
2637                         assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
2638                 },
2639                 _ => panic!("Unexpected event"),
2640         }
2641         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2642         assert_eq!(htlc_msgs.len(), 2);
2643         check_added_monitors!(nodes[0], 2);
2644 }
2645
2646 #[test]
2647 fn immediate_retry_on_failure() {
2648         // Tests that we can/will retry immediately after a failure
2649         let chanmon_cfgs = create_chanmon_cfgs(2);
2650         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2651         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2652         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2653
2654         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2655         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2656
2657         let amt_msat = 100_000_001;
2658         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2659         #[cfg(feature = "std")]
2660         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2661         #[cfg(not(feature = "std"))]
2662         let payment_expiry_secs = 60 * 60;
2663         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2664         invoice_features.set_variable_length_onion_required();
2665         invoice_features.set_payment_secret_required();
2666         invoice_features.set_basic_mpp_optional();
2667         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2668                 .with_expiry_time(payment_expiry_secs as u64)
2669                 .with_bolt11_features(invoice_features).unwrap();
2670         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2671
2672         let chans = nodes[0].node.list_usable_channels();
2673         let mut route = Route {
2674                 paths: vec![
2675                         Path { hops: vec![RouteHop {
2676                                 pubkey: nodes[1].node.get_our_node_id(),
2677                                 node_features: nodes[1].node.node_features(),
2678                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2679                                 channel_features: nodes[1].node.channel_features(),
2680                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2681                                 cltv_expiry_delta: 100,
2682                                 maybe_announced_channel: true,
2683                         }], blinded_tail: None },
2684                 ],
2685                 route_params: Some(RouteParameters::from_payment_params_and_value(
2686                         PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV),
2687                         100_000_001)),
2688         };
2689         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2690         // On retry, split the payment across both channels.
2691         route.paths.push(route.paths[0].clone());
2692         route.paths[0].hops[0].short_channel_id = chans[1].short_channel_id.unwrap();
2693         route.paths[0].hops[0].fee_msat = 50_000_000;
2694         route.paths[1].hops[0].fee_msat = 50_000_001;
2695         let mut pay_params = route_params.payment_params.clone();
2696         pay_params.previously_failed_channels.push(chans[0].short_channel_id.unwrap());
2697         nodes[0].router.expect_find_route(
2698                 RouteParameters::from_payment_params_and_value(pay_params, amt_msat),
2699                 Ok(route.clone()));
2700
2701         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2702                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2703         let events = nodes[0].node.get_and_clear_pending_events();
2704         assert_eq!(events.len(), 1);
2705         match events[0] {
2706                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false,
2707                         failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }},
2708                         short_channel_id: Some(expected_scid), .. } =>
2709                 {
2710                         assert_eq!(payment_hash, ev_payment_hash);
2711                         assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
2712                 },
2713                 _ => panic!("Unexpected event"),
2714         }
2715         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2716         assert_eq!(htlc_msgs.len(), 2);
2717         check_added_monitors!(nodes[0], 2);
2718 }
2719
2720 #[test]
2721 fn no_extra_retries_on_back_to_back_fail() {
2722         // In a previous release, we had a race where we may exceed the payment retry count if we
2723         // get two failures in a row with the second indicating that all paths had failed (this field,
2724         // `all_paths_failed`, has since been removed).
2725         // Generally, when we give up trying to retry a payment, we don't know for sure what the
2726         // current state of the ChannelManager event queue is. Specifically, we cannot be sure that
2727         // there are not multiple additional `PaymentPathFailed` or even `PaymentSent` events
2728         // pending which we will see later. Thus, when we previously removed the retry tracking map
2729         // entry after a `all_paths_failed` `PaymentPathFailed` event, we may have dropped the
2730         // retry entry even though more events for the same payment were still pending. This led to
2731         // us retrying a payment again even though we'd already given up on it.
2732         //
2733         // We now have a separate event - `PaymentFailed` which indicates no HTLCs remain and which
2734         // is used to remove the payment retry counter entries instead. This tests for the specific
2735         // excess-retry case while also testing `PaymentFailed` generation.
2736
2737         let chanmon_cfgs = create_chanmon_cfgs(3);
2738         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2739         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2740         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2741
2742         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2743         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2744
2745         let amt_msat = 200_000_000;
2746         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2747         #[cfg(feature = "std")]
2748         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2749         #[cfg(not(feature = "std"))]
2750         let payment_expiry_secs = 60 * 60;
2751         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2752         invoice_features.set_variable_length_onion_required();
2753         invoice_features.set_payment_secret_required();
2754         invoice_features.set_basic_mpp_optional();
2755         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2756                 .with_expiry_time(payment_expiry_secs as u64)
2757                 .with_bolt11_features(invoice_features).unwrap();
2758         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2759
2760         let mut route = Route {
2761                 paths: vec![
2762                         Path { hops: vec![RouteHop {
2763                                 pubkey: nodes[1].node.get_our_node_id(),
2764                                 node_features: nodes[1].node.node_features(),
2765                                 short_channel_id: chan_1_scid,
2766                                 channel_features: nodes[1].node.channel_features(),
2767                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2768                                 cltv_expiry_delta: 100,
2769                                 maybe_announced_channel: true,
2770                         }, RouteHop {
2771                                 pubkey: nodes[2].node.get_our_node_id(),
2772                                 node_features: nodes[2].node.node_features(),
2773                                 short_channel_id: chan_2_scid,
2774                                 channel_features: nodes[2].node.channel_features(),
2775                                 fee_msat: 100_000_000,
2776                                 cltv_expiry_delta: 100,
2777                                 maybe_announced_channel: true,
2778                         }], blinded_tail: None },
2779                         Path { hops: vec![RouteHop {
2780                                 pubkey: nodes[1].node.get_our_node_id(),
2781                                 node_features: nodes[1].node.node_features(),
2782                                 short_channel_id: chan_1_scid,
2783                                 channel_features: nodes[1].node.channel_features(),
2784                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2785                                 cltv_expiry_delta: 100,
2786                                 maybe_announced_channel: true,
2787                         }, RouteHop {
2788                                 pubkey: nodes[2].node.get_our_node_id(),
2789                                 node_features: nodes[2].node.node_features(),
2790                                 short_channel_id: chan_2_scid,
2791                                 channel_features: nodes[2].node.channel_features(),
2792                                 fee_msat: 100_000_000,
2793                                 cltv_expiry_delta: 100,
2794                                 maybe_announced_channel: true,
2795                         }], blinded_tail: None }
2796                 ],
2797                 route_params: Some(RouteParameters::from_payment_params_and_value(
2798                         PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV),
2799                         100_000_000)),
2800         };
2801         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2802         let mut second_payment_params = route_params.payment_params.clone();
2803         second_payment_params.previously_failed_channels = vec![chan_2_scid, chan_2_scid];
2804         // On retry, we'll only return one path
2805         route.paths.remove(1);
2806         route.paths[0].hops[1].fee_msat = amt_msat;
2807         nodes[0].router.expect_find_route(
2808                 RouteParameters::from_payment_params_and_value(second_payment_params, amt_msat),
2809                 Ok(route.clone()));
2810
2811         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2812                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2813         let htlc_updates = SendEvent::from_node(&nodes[0]);
2814         check_added_monitors!(nodes[0], 1);
2815         assert_eq!(htlc_updates.msgs.len(), 1);
2816
2817         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2818         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2819         check_added_monitors!(nodes[1], 1);
2820         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2821
2822         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2823         check_added_monitors!(nodes[0], 1);
2824         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2825
2826         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2827         check_added_monitors!(nodes[0], 1);
2828         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2829
2830         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2831         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2832         check_added_monitors!(nodes[1], 1);
2833         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2834
2835         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2836         check_added_monitors!(nodes[1], 1);
2837         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2838
2839         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2840         check_added_monitors!(nodes[0], 1);
2841
2842         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2843         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2844         check_added_monitors!(nodes[0], 1);
2845         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2846
2847         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2848         check_added_monitors!(nodes[1], 1);
2849         let bs_second_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2850
2851         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2852         check_added_monitors!(nodes[1], 1);
2853         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2854
2855         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.update_fail_htlcs[0]);
2856         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.commitment_signed);
2857         check_added_monitors!(nodes[0], 1);
2858
2859         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2860         check_added_monitors!(nodes[0], 1);
2861         let (as_third_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2862
2863         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2864         check_added_monitors!(nodes[1], 1);
2865         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2866         check_added_monitors!(nodes[1], 1);
2867         let bs_fourth_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2868
2869         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_fourth_raa);
2870         check_added_monitors!(nodes[0], 1);
2871
2872         // At this point A has sent two HTLCs which both failed due to lack of fee. It now has two
2873         // pending `PaymentPathFailed` events, one with `all_paths_failed` unset, and the second
2874         // with it set.
2875         //
2876         // Previously, we retried payments in an event consumer, which would retry each
2877         // `PaymentPathFailed` individually. In that setup, we had retried the payment in response to
2878         // the first `PaymentPathFailed`, then seen the second `PaymentPathFailed` with
2879         // `all_paths_failed` set and assumed the payment was completely failed. We ultimately fixed it
2880         // by adding the `PaymentFailed` event.
2881         //
2882         // Because we now retry payments as a batch, we simply return a single-path route in the
2883         // second, batched, request, have that fail, ensure the payment was abandoned.
2884         let mut events = nodes[0].node.get_and_clear_pending_events();
2885         assert_eq!(events.len(), 3);
2886         match events[0] {
2887                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2888                         assert_eq!(payment_hash, ev_payment_hash);
2889                         assert_eq!(payment_failed_permanently, false);
2890                 },
2891                 _ => panic!("Unexpected event"),
2892         }
2893         match events[1] {
2894                 Event::PendingHTLCsForwardable { .. } => {},
2895                 _ => panic!("Unexpected event"),
2896         }
2897         match events[2] {
2898                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2899                         assert_eq!(payment_hash, ev_payment_hash);
2900                         assert_eq!(payment_failed_permanently, false);
2901                 },
2902                 _ => panic!("Unexpected event"),
2903         }
2904
2905         nodes[0].node.process_pending_htlc_forwards();
2906         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
2907         check_added_monitors!(nodes[0], 1);
2908
2909         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
2910         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
2911         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2912         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2913         commitment_signed_dance!(nodes[0], nodes[1], &bs_fail_update.commitment_signed, false, true);
2914
2915         let mut events = nodes[0].node.get_and_clear_pending_events();
2916         assert_eq!(events.len(), 2);
2917         match events[0] {
2918                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2919                         assert_eq!(payment_hash, ev_payment_hash);
2920                         assert_eq!(payment_failed_permanently, false);
2921                 },
2922                 _ => panic!("Unexpected event"),
2923         }
2924         match events[1] {
2925                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2926                         assert_eq!(payment_hash, *ev_payment_hash);
2927                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2928                         assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
2929                 },
2930                 _ => panic!("Unexpected event"),
2931         }
2932 }
2933
2934 #[test]
2935 fn test_simple_partial_retry() {
2936         // In the first version of the in-`ChannelManager` payment retries, retries were sent for the
2937         // full amount of the payment, rather than only the missing amount. Here we simply test for
2938         // this by sending a payment with two parts, failing one, and retrying the second. Note that
2939         // `TestRouter` will check that the `RouteParameters` (which contain the amount) matches the
2940         // request.
2941         let chanmon_cfgs = create_chanmon_cfgs(3);
2942         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2943         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2944         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2945
2946         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2947         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2948
2949         let amt_msat = 200_000_000;
2950         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
2951         #[cfg(feature = "std")]
2952         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2953         #[cfg(not(feature = "std"))]
2954         let payment_expiry_secs = 60 * 60;
2955         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2956         invoice_features.set_variable_length_onion_required();
2957         invoice_features.set_payment_secret_required();
2958         invoice_features.set_basic_mpp_optional();
2959         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2960                 .with_expiry_time(payment_expiry_secs as u64)
2961                 .with_bolt11_features(invoice_features).unwrap();
2962         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2963
2964         let mut route = Route {
2965                 paths: vec![
2966                         Path { hops: vec![RouteHop {
2967                                 pubkey: nodes[1].node.get_our_node_id(),
2968                                 node_features: nodes[1].node.node_features(),
2969                                 short_channel_id: chan_1_scid,
2970                                 channel_features: nodes[1].node.channel_features(),
2971                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2972                                 cltv_expiry_delta: 100,
2973                                 maybe_announced_channel: true,
2974                         }, RouteHop {
2975                                 pubkey: nodes[2].node.get_our_node_id(),
2976                                 node_features: nodes[2].node.node_features(),
2977                                 short_channel_id: chan_2_scid,
2978                                 channel_features: nodes[2].node.channel_features(),
2979                                 fee_msat: 100_000_000,
2980                                 cltv_expiry_delta: 100,
2981                                 maybe_announced_channel: true,
2982                         }], blinded_tail: None },
2983                         Path { hops: vec![RouteHop {
2984                                 pubkey: nodes[1].node.get_our_node_id(),
2985                                 node_features: nodes[1].node.node_features(),
2986                                 short_channel_id: chan_1_scid,
2987                                 channel_features: nodes[1].node.channel_features(),
2988                                 fee_msat: 100_000,
2989                                 cltv_expiry_delta: 100,
2990                                 maybe_announced_channel: true,
2991                         }, RouteHop {
2992                                 pubkey: nodes[2].node.get_our_node_id(),
2993                                 node_features: nodes[2].node.node_features(),
2994                                 short_channel_id: chan_2_scid,
2995                                 channel_features: nodes[2].node.channel_features(),
2996                                 fee_msat: 100_000_000,
2997                                 cltv_expiry_delta: 100,
2998                                 maybe_announced_channel: true,
2999                         }], blinded_tail: None }
3000                 ],
3001                 route_params: Some(RouteParameters::from_payment_params_and_value(
3002                         PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV),
3003                         100_000_000)),
3004         };
3005         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3006         let mut second_payment_params = route_params.payment_params.clone();
3007         second_payment_params.previously_failed_channels = vec![chan_2_scid];
3008         // On retry, we'll only be asked for one path (or 100k sats)
3009         route.paths.remove(0);
3010         nodes[0].router.expect_find_route(
3011                 RouteParameters::from_payment_params_and_value(second_payment_params, amt_msat / 2),
3012                 Ok(route.clone()));
3013
3014         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3015                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
3016         let htlc_updates = SendEvent::from_node(&nodes[0]);
3017         check_added_monitors!(nodes[0], 1);
3018         assert_eq!(htlc_updates.msgs.len(), 1);
3019
3020         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
3021         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
3022         check_added_monitors!(nodes[1], 1);
3023         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3024
3025         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
3026         check_added_monitors!(nodes[0], 1);
3027         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
3028
3029         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
3030         check_added_monitors!(nodes[0], 1);
3031         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
3032
3033         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
3034         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
3035         check_added_monitors!(nodes[1], 1);
3036         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
3037
3038         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
3039         check_added_monitors!(nodes[1], 1);
3040         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3041
3042         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
3043         check_added_monitors!(nodes[0], 1);
3044
3045         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
3046         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
3047         check_added_monitors!(nodes[0], 1);
3048         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
3049
3050         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
3051         check_added_monitors!(nodes[1], 1);
3052
3053         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
3054         check_added_monitors!(nodes[1], 1);
3055
3056         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
3057
3058         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
3059         check_added_monitors!(nodes[0], 1);
3060
3061         let mut events = nodes[0].node.get_and_clear_pending_events();
3062         assert_eq!(events.len(), 2);
3063         match events[0] {
3064                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
3065                         assert_eq!(payment_hash, ev_payment_hash);
3066                         assert_eq!(payment_failed_permanently, false);
3067                 },
3068                 _ => panic!("Unexpected event"),
3069         }
3070         match events[1] {
3071                 Event::PendingHTLCsForwardable { .. } => {},
3072                 _ => panic!("Unexpected event"),
3073         }
3074
3075         nodes[0].node.process_pending_htlc_forwards();
3076         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
3077         check_added_monitors!(nodes[0], 1);
3078
3079         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
3080         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
3081
3082         expect_pending_htlcs_forwardable!(nodes[1]);
3083         check_added_monitors!(nodes[1], 1);
3084
3085         let bs_forward_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
3086         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[0]);
3087         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[1]);
3088         commitment_signed_dance!(nodes[2], nodes[1], &bs_forward_update.commitment_signed, false);
3089
3090         expect_pending_htlcs_forwardable!(nodes[2]);
3091         expect_payment_claimable!(nodes[2], payment_hash, payment_secret, amt_msat);
3092 }
3093
3094 #[test]
3095 #[cfg(feature = "std")]
3096 fn test_threaded_payment_retries() {
3097         // In the first version of the in-`ChannelManager` payment retries, retries weren't limited to
3098         // a single thread and would happily let multiple threads run retries at the same time. Because
3099         // retries are done by first calculating the amount we need to retry, then dropping the
3100         // relevant lock, then actually sending, we would happily let multiple threads retry the same
3101         // amount at the same time, overpaying our original HTLC!
3102         let chanmon_cfgs = create_chanmon_cfgs(4);
3103         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3104         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3105         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3106
3107         // There is one mitigating guardrail when retrying payments - we can never over-pay by more
3108         // than 10% of the original value. Thus, we want all our retries to be below that. In order to
3109         // keep things simple, we route one HTLC for 0.1% of the payment over channel 1 and the rest
3110         // out over channel 3+4. This will let us ignore 99% of the payment value and deal with only
3111         // our channel.
3112         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
3113         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 10_000_000, 0);
3114         let chan_3_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 10_000_000, 0).0.contents.short_channel_id;
3115         let chan_4_scid = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 10_000_000, 0).0.contents.short_channel_id;
3116
3117         let amt_msat = 100_000_000;
3118         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
3119         #[cfg(feature = "std")]
3120         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
3121         #[cfg(not(feature = "std"))]
3122         let payment_expiry_secs = 60 * 60;
3123         let mut invoice_features = Bolt11InvoiceFeatures::empty();
3124         invoice_features.set_variable_length_onion_required();
3125         invoice_features.set_payment_secret_required();
3126         invoice_features.set_basic_mpp_optional();
3127         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
3128                 .with_expiry_time(payment_expiry_secs as u64)
3129                 .with_bolt11_features(invoice_features).unwrap();
3130         let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
3131
3132         let mut route = Route {
3133                 paths: vec![
3134                         Path { hops: vec![RouteHop {
3135                                 pubkey: nodes[1].node.get_our_node_id(),
3136                                 node_features: nodes[1].node.node_features(),
3137                                 short_channel_id: chan_1_scid,
3138                                 channel_features: nodes[1].node.channel_features(),
3139                                 fee_msat: 0,
3140                                 cltv_expiry_delta: 100,
3141                                 maybe_announced_channel: true,
3142                         }, RouteHop {
3143                                 pubkey: nodes[3].node.get_our_node_id(),
3144                                 node_features: nodes[2].node.node_features(),
3145                                 short_channel_id: 42, // Set a random SCID which nodes[1] will fail as unknown
3146                                 channel_features: nodes[2].node.channel_features(),
3147                                 fee_msat: amt_msat / 1000,
3148                                 cltv_expiry_delta: 100,
3149                                 maybe_announced_channel: true,
3150                         }], blinded_tail: None },
3151                         Path { hops: vec![RouteHop {
3152                                 pubkey: nodes[2].node.get_our_node_id(),
3153                                 node_features: nodes[2].node.node_features(),
3154                                 short_channel_id: chan_3_scid,
3155                                 channel_features: nodes[2].node.channel_features(),
3156                                 fee_msat: 100_000,
3157                                 cltv_expiry_delta: 100,
3158                                 maybe_announced_channel: true,
3159                         }, RouteHop {
3160                                 pubkey: nodes[3].node.get_our_node_id(),
3161                                 node_features: nodes[3].node.node_features(),
3162                                 short_channel_id: chan_4_scid,
3163                                 channel_features: nodes[3].node.channel_features(),
3164                                 fee_msat: amt_msat - amt_msat / 1000,
3165                                 cltv_expiry_delta: 100,
3166                                 maybe_announced_channel: true,
3167                         }], blinded_tail: None }
3168                 ],
3169                 route_params: Some(RouteParameters::from_payment_params_and_value(
3170                         PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV),
3171                         amt_msat - amt_msat / 1000)),
3172         };
3173         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3174
3175         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3176                 PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(0xdeadbeef)).unwrap();
3177         check_added_monitors!(nodes[0], 2);
3178         let mut send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
3179         assert_eq!(send_msg_events.len(), 2);
3180         send_msg_events.retain(|msg|
3181                 if let MessageSendEvent::UpdateHTLCs { node_id, .. } = msg {
3182                         // Drop the commitment update for nodes[2], we can just let that one sit pending
3183                         // forever.
3184                         *node_id == nodes[1].node.get_our_node_id()
3185                 } else { panic!(); }
3186         );
3187
3188         // from here on out, the retry `RouteParameters` amount will be amt/1000
3189         route_params.final_value_msat /= 1000;
3190         route.paths.pop();
3191
3192         let end_time = Instant::now() + Duration::from_secs(1);
3193         macro_rules! thread_body { () => { {
3194                 // We really want std::thread::scope, but its not stable until 1.63. Until then, we get unsafe.
3195                 let node_ref = NodePtr::from_node(&nodes[0]);
3196                 move || {
3197                         let node_a = unsafe { &*node_ref.0 };
3198                         while Instant::now() < end_time {
3199                                 node_a.node.get_and_clear_pending_events(); // wipe the PendingHTLCsForwardable
3200                                 // Ignore if we have any pending events, just always pretend we just got a
3201                                 // PendingHTLCsForwardable
3202                                 node_a.node.process_pending_htlc_forwards();
3203                         }
3204                 }
3205         } } }
3206         let mut threads = Vec::new();
3207         for _ in 0..16 { threads.push(std::thread::spawn(thread_body!())); }
3208
3209         // Back in the main thread, poll pending messages and make sure that we never have more than
3210         // one HTLC pending at a time. Note that the commitment_signed_dance will fail horribly if
3211         // there are HTLC messages shoved in while its running. This allows us to test that we never
3212         // generate an additional update_add_htlc until we've fully failed the first.
3213         let mut previously_failed_channels = Vec::new();
3214         loop {
3215                 assert_eq!(send_msg_events.len(), 1);
3216                 let send_event = SendEvent::from_event(send_msg_events.pop().unwrap());
3217                 assert_eq!(send_event.msgs.len(), 1);
3218
3219                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
3220                 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
3221
3222                 // Note that we only push one route into `expect_find_route` at a time, because that's all
3223                 // the retries (should) need. If the bug is reintroduced "real" routes may be selected, but
3224                 // we should still ultimately fail for the same reason - because we're trying to send too
3225                 // many HTLCs at once.
3226                 let mut new_route_params = route_params.clone();
3227                 previously_failed_channels.push(route.paths[0].hops[1].short_channel_id);
3228                 new_route_params.payment_params.previously_failed_channels = previously_failed_channels.clone();
3229                 route.paths[0].hops[1].short_channel_id += 1;
3230                 nodes[0].router.expect_find_route(new_route_params, Ok(route.clone()));
3231
3232                 let bs_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3233                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_updates.update_fail_htlcs[0]);
3234                 // The "normal" commitment_signed_dance delivers the final RAA and then calls
3235                 // `check_added_monitors` to ensure only the one RAA-generated monitor update was created.
3236                 // This races with our other threads which may generate an add-HTLCs commitment update via
3237                 // `process_pending_htlc_forwards`. Instead, we defer the monitor update check until after
3238                 // *we've* called `process_pending_htlc_forwards` when its guaranteed to have two updates.
3239                 let last_raa = commitment_signed_dance!(nodes[0], nodes[1], bs_fail_updates.commitment_signed, false, true, false, true);
3240                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &last_raa);
3241
3242                 let cur_time = Instant::now();
3243                 if cur_time > end_time {
3244                         for thread in threads.drain(..) { thread.join().unwrap(); }
3245                 }
3246
3247                 // Make sure we have some events to handle when we go around...
3248                 nodes[0].node.get_and_clear_pending_events(); // wipe the PendingHTLCsForwardable
3249                 nodes[0].node.process_pending_htlc_forwards();
3250                 send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
3251                 check_added_monitors!(nodes[0], 2);
3252
3253                 if cur_time > end_time {
3254                         break;
3255                 }
3256         }
3257 }
3258
3259 fn do_no_missing_sent_on_reload(persist_manager_with_payment: bool, at_midpoint: bool) {
3260         // Test that if we reload in the middle of an HTLC claim commitment signed dance we'll still
3261         // receive the PaymentSent event even if the ChannelManager had no idea about the payment when
3262         // it was last persisted.
3263         let chanmon_cfgs = create_chanmon_cfgs(2);
3264         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
3265         let (persister_a, persister_b, persister_c);
3266         let (chain_monitor_a, chain_monitor_b, chain_monitor_c);
3267         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
3268         let (nodes_0_deserialized, nodes_0_deserialized_b, nodes_0_deserialized_c);
3269         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
3270
3271         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
3272
3273         let mut nodes_0_serialized = Vec::new();
3274         if !persist_manager_with_payment {
3275                 nodes_0_serialized = nodes[0].node.encode();
3276         }
3277
3278         let (our_payment_preimage, our_payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 1_000_000);
3279
3280         if persist_manager_with_payment {
3281                 nodes_0_serialized = nodes[0].node.encode();
3282         }
3283
3284         nodes[1].node.claim_funds(our_payment_preimage);
3285         check_added_monitors!(nodes[1], 1);
3286         expect_payment_claimed!(nodes[1], our_payment_hash, 1_000_000);
3287
3288         if at_midpoint {
3289                 let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3290                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3291                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
3292                 check_added_monitors!(nodes[0], 1);
3293         } else {
3294                 let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3295                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
3296                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fulfill_updates.commitment_signed, false);
3297                 // Ignore the PaymentSent event which is now pending on nodes[0] - if we were to handle it we'd
3298                 // be expected to ignore the eventual conflicting PaymentFailed, but by not looking at it we
3299                 // expect to get the PaymentSent again later.
3300                 check_added_monitors(&nodes[0], 0);
3301         }
3302
3303         // The ChannelMonitor should always be the latest version, as we're required to persist it
3304         // during the commitment signed handling.
3305         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3306         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister_a, chain_monitor_a, nodes_0_deserialized);
3307
3308         let events = nodes[0].node.get_and_clear_pending_events();
3309         assert_eq!(events.len(), 2);
3310         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[0] {} else { panic!(); }
3311         if let Event::PaymentSent { payment_preimage, .. } = events[1] { assert_eq!(payment_preimage, our_payment_preimage); } else { panic!(); }
3312         // Note that we don't get a PaymentPathSuccessful here as we leave the HTLC pending to avoid
3313         // the double-claim that would otherwise appear at the end of this test.
3314         nodes[0].node.timer_tick_occurred();
3315         let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3316         assert_eq!(as_broadcasted_txn.len(), 1);
3317
3318         // Ensure that, even after some time, if we restart we still include *something* in the current
3319         // `ChannelManager` which prevents a `PaymentFailed` when we restart even if pending resolved
3320         // payments have since been timed out thanks to `IDEMPOTENCY_TIMEOUT_TICKS`.
3321         // A naive implementation of the fix here would wipe the pending payments set, causing a
3322         // failure event when we restart.
3323         for _ in 0..(IDEMPOTENCY_TIMEOUT_TICKS * 2) { nodes[0].node.timer_tick_occurred(); }
3324
3325         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3326         reload_node!(nodes[0], test_default_channel_config(), &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister_b, chain_monitor_b, nodes_0_deserialized_b);
3327         let events = nodes[0].node.get_and_clear_pending_events();
3328         assert!(events.is_empty());
3329
3330         // Ensure that we don't generate any further events even after the channel-closing commitment
3331         // transaction is confirmed on-chain.
3332         confirm_transaction(&nodes[0], &as_broadcasted_txn[0]);
3333         for _ in 0..(IDEMPOTENCY_TIMEOUT_TICKS * 2) { nodes[0].node.timer_tick_occurred(); }
3334
3335         let events = nodes[0].node.get_and_clear_pending_events();
3336         assert!(events.is_empty());
3337
3338         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3339         reload_node!(nodes[0], test_default_channel_config(), &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister_c, chain_monitor_c, nodes_0_deserialized_c);
3340         let events = nodes[0].node.get_and_clear_pending_events();
3341         assert!(events.is_empty());
3342         check_added_monitors(&nodes[0], 1);
3343 }
3344
3345 #[test]
3346 fn no_missing_sent_on_midpoint_reload() {
3347         do_no_missing_sent_on_reload(false, true);
3348         do_no_missing_sent_on_reload(true, true);
3349 }
3350
3351 #[test]
3352 fn no_missing_sent_on_reload() {
3353         do_no_missing_sent_on_reload(false, false);
3354         do_no_missing_sent_on_reload(true, false);
3355 }
3356
3357 fn do_claim_from_closed_chan(fail_payment: bool) {
3358         // Previously, LDK would refuse to claim a payment if a channel on which the payment was
3359         // received had been closed between when the HTLC was received and when we went to claim it.
3360         // This makes sense in the payment case - why pay an on-chain fee to claim the HTLC when
3361         // presumably the sender may retry later. Long ago it also reduced total code in the claim
3362         // pipeline.
3363         //
3364         // However, this doesn't make sense if you're trying to do an atomic swap or some other
3365         // protocol that requires atomicity with some other action - if your money got claimed
3366         // elsewhere you need to be able to claim the HTLC in lightning no matter what. Further, this
3367         // is an over-optimization - there should be a very, very low likelihood that a channel closes
3368         // between when we receive the last HTLC for a payment and the user goes to claim the payment.
3369         // Since we now have code to handle this anyway we should allow it.
3370
3371         // Build 4 nodes and send an MPP payment across two paths. By building a route manually set the
3372         // CLTVs on the paths to different value resulting in a different claim deadline.
3373         let chanmon_cfgs = create_chanmon_cfgs(4);
3374         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3375         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3376         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3377
3378         create_announced_chan_between_nodes(&nodes, 0, 1);
3379         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
3380         let chan_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
3381         create_announced_chan_between_nodes(&nodes, 2, 3);
3382
3383         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3]);
3384         let mut route_params = RouteParameters::from_payment_params_and_value(
3385                 PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3386                         .with_bolt11_features(nodes[1].node.invoice_features()).unwrap(),
3387                 10_000_000);
3388         let mut route = nodes[0].router.find_route(&nodes[0].node.get_our_node_id(), &route_params,
3389                 None, nodes[0].node.compute_inflight_htlcs()).unwrap();
3390         // Make sure the route is ordered as the B->D path before C->D
3391         route.paths.sort_by(|a, _| if a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
3392                 std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater });
3393
3394         // Note that we add an extra 1 in the send pipeline to compensate for any blocks found while
3395         // the HTLC is being relayed.
3396         route.paths[0].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 8;
3397         route.paths[1].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 12;
3398         let final_cltv = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 8 + 1;
3399
3400         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3401         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3402                 PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(1)).unwrap();
3403         check_added_monitors(&nodes[0], 2);
3404         let mut send_msgs = nodes[0].node.get_and_clear_pending_msg_events();
3405         send_msgs.sort_by(|a, _| {
3406                 let a_node_id =
3407                         if let MessageSendEvent::UpdateHTLCs { node_id, .. } = a { node_id } else { panic!() };
3408                 let node_b_id = nodes[1].node.get_our_node_id();
3409                 if *a_node_id == node_b_id { std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater }
3410         });
3411
3412         assert_eq!(send_msgs.len(), 2);
3413         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 10_000_000,
3414                 payment_hash, Some(payment_secret), send_msgs.remove(0), false, None);
3415         let receive_event = pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 10_000_000,
3416                 payment_hash, Some(payment_secret), send_msgs.remove(0), true, None);
3417
3418         match receive_event.unwrap() {
3419                 Event::PaymentClaimable { claim_deadline, .. } => {
3420                         assert_eq!(claim_deadline.unwrap(), final_cltv - HTLC_FAIL_BACK_BUFFER);
3421                 },
3422                 _ => panic!(),
3423         }
3424
3425         // Ensure that the claim_deadline is correct, with the payment failing at exactly the given
3426         // height.
3427         connect_blocks(&nodes[3], final_cltv - HTLC_FAIL_BACK_BUFFER - nodes[3].best_block_info().1
3428                 - if fail_payment { 0 } else { 2 });
3429         if fail_payment {
3430                 // We fail the HTLC on the A->B->D path first as it expires 4 blocks earlier. We go ahead
3431                 // and expire both immediately, though, by connecting another 4 blocks.
3432                 let reason = HTLCDestination::FailedPayment { payment_hash };
3433                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason.clone()]);
3434                 connect_blocks(&nodes[3], 4);
3435                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason]);
3436                 pass_failed_payment_back(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
3437         } else {
3438                 nodes[1].node.force_close_broadcasting_latest_txn(&chan_bd, &nodes[3].node.get_our_node_id()).unwrap();
3439                 check_closed_event!(&nodes[1], 1, ClosureReason::HolderForceClosed, false,
3440                         [nodes[3].node.get_our_node_id()], 1000000);
3441                 check_closed_broadcast(&nodes[1], 1, true);
3442                 let bs_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3443                 assert_eq!(bs_tx.len(), 1);
3444
3445                 mine_transaction(&nodes[3], &bs_tx[0]);
3446                 check_added_monitors(&nodes[3], 1);
3447                 check_closed_broadcast(&nodes[3], 1, true);
3448                 check_closed_event!(&nodes[3], 1, ClosureReason::CommitmentTxConfirmed, false,
3449                         [nodes[1].node.get_our_node_id()], 1000000);
3450
3451                 nodes[3].node.claim_funds(payment_preimage);
3452                 check_added_monitors(&nodes[3], 2);
3453                 expect_payment_claimed!(nodes[3], payment_hash, 10_000_000);
3454
3455                 let ds_tx = nodes[3].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3456                 assert_eq!(ds_tx.len(), 1);
3457                 check_spends!(&ds_tx[0], &bs_tx[0]);
3458
3459                 mine_transactions(&nodes[1], &[&bs_tx[0], &ds_tx[0]]);
3460                 check_added_monitors(&nodes[1], 1);
3461                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[3], Some(1000), false, true);
3462
3463                 let bs_claims = nodes[1].node.get_and_clear_pending_msg_events();
3464                 check_added_monitors(&nodes[1], 1);
3465                 assert_eq!(bs_claims.len(), 1);
3466                 if let MessageSendEvent::UpdateHTLCs { updates, .. } = &bs_claims[0] {
3467                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3468                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
3469                 } else { panic!(); }
3470
3471                 expect_payment_sent!(nodes[0], payment_preimage);
3472
3473                 let ds_claim_msgs = nodes[3].node.get_and_clear_pending_msg_events();
3474                 assert_eq!(ds_claim_msgs.len(), 1);
3475                 let cs_claim_msgs = if let MessageSendEvent::UpdateHTLCs { updates, .. } = &ds_claim_msgs[0] {
3476                         nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3477                         let cs_claim_msgs = nodes[2].node.get_and_clear_pending_msg_events();
3478                         check_added_monitors(&nodes[2], 1);
3479                         commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
3480                         expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
3481                         cs_claim_msgs
3482                 } else { panic!(); };
3483
3484                 assert_eq!(cs_claim_msgs.len(), 1);
3485                 if let MessageSendEvent::UpdateHTLCs { updates, .. } = &cs_claim_msgs[0] {
3486                         nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3487                         commitment_signed_dance!(nodes[0], nodes[2], updates.commitment_signed, false, true);
3488                 } else { panic!(); }
3489
3490                 expect_payment_path_successful!(nodes[0]);
3491         }
3492 }
3493
3494 #[test]
3495 fn claim_from_closed_chan() {
3496         do_claim_from_closed_chan(true);
3497         do_claim_from_closed_chan(false);
3498 }
3499
3500 #[test]
3501 fn test_custom_tlvs_basic() {
3502         do_test_custom_tlvs(false, false, false);
3503         do_test_custom_tlvs(true, false, false);
3504 }
3505
3506 #[test]
3507 fn test_custom_tlvs_explicit_claim() {
3508         // Test that when receiving even custom TLVs the user must explicitly accept in case they
3509         // are unknown.
3510         do_test_custom_tlvs(false, true, false);
3511         do_test_custom_tlvs(false, true, true);
3512 }
3513
3514 fn do_test_custom_tlvs(spontaneous: bool, even_tlvs: bool, known_tlvs: bool) {
3515         let chanmon_cfgs = create_chanmon_cfgs(2);
3516         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
3517         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None; 2]);
3518         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
3519
3520         create_announced_chan_between_nodes(&nodes, 0, 1);
3521
3522         let amt_msat = 100_000;
3523         let (mut route, our_payment_hash, our_payment_preimage, our_payment_secret) = get_route_and_payment_hash!(&nodes[0], &nodes[1], amt_msat);
3524         let payment_id = PaymentId(our_payment_hash.0);
3525         let custom_tlvs = vec![
3526                 (if even_tlvs { 5482373482 } else { 5482373483 }, vec![1, 2, 3, 4]),
3527                 (5482373487, vec![0x42u8; 16]),
3528         ];
3529         let onion_fields = RecipientOnionFields {
3530                 payment_secret: if spontaneous { None } else { Some(our_payment_secret) },
3531                 payment_metadata: None,
3532                 custom_tlvs: custom_tlvs.clone()
3533         };
3534         if spontaneous {
3535                 nodes[0].node.send_spontaneous_payment(&route, Some(our_payment_preimage), onion_fields, payment_id).unwrap();
3536         } else {
3537                 nodes[0].node.send_payment_with_route(&route, our_payment_hash, onion_fields, payment_id).unwrap();
3538         }
3539         check_added_monitors(&nodes[0], 1);
3540
3541         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
3542         let ev = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
3543         let mut payment_event = SendEvent::from_event(ev);
3544
3545         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
3546         check_added_monitors!(&nodes[1], 0);
3547         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
3548         expect_pending_htlcs_forwardable!(nodes[1]);
3549
3550         let events = nodes[1].node.get_and_clear_pending_events();
3551         assert_eq!(events.len(), 1);
3552         match events[0] {
3553                 Event::PaymentClaimable { ref onion_fields, .. } => {
3554                         assert_eq!(onion_fields.clone().unwrap().custom_tlvs().clone(), custom_tlvs);
3555                 },
3556                 _ => panic!("Unexpected event"),
3557         }
3558
3559         match (known_tlvs, even_tlvs) {
3560                 (true, _) => {
3561                         nodes[1].node.claim_funds_with_known_custom_tlvs(our_payment_preimage);
3562                         let expected_total_fee_msat = pass_claimed_payment_along_route(&nodes[0], &[&[&nodes[1]]], &[0; 1], false, our_payment_preimage);
3563                         expect_payment_sent!(&nodes[0], our_payment_preimage, Some(expected_total_fee_msat));
3564                 },
3565                 (false, false) => {
3566                         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
3567                 },
3568                 (false, true) => {
3569                         nodes[1].node.claim_funds(our_payment_preimage);
3570                         let expected_destinations = vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }];
3571                         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], expected_destinations);
3572                         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, our_payment_hash, PaymentFailureReason::RecipientRejected);
3573                 }
3574         }
3575 }
3576
3577 #[test]
3578 fn test_retry_custom_tlvs() {
3579         // Test that custom TLVs are successfully sent on retries
3580         let chanmon_cfgs = create_chanmon_cfgs(3);
3581         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
3582         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
3583         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
3584
3585         create_announced_chan_between_nodes(&nodes, 0, 1);
3586         let (chan_2_update, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 2, 1);
3587
3588         // Rebalance
3589         send_payment(&nodes[2], &vec!(&nodes[1])[..], 1_500_000);
3590
3591         let amt_msat = 1_000_000;
3592         let (route, payment_hash, payment_preimage, payment_secret) =
3593                 get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
3594
3595         // Initiate the payment
3596         let payment_id = PaymentId(payment_hash.0);
3597         let mut route_params = route.route_params.clone().unwrap();
3598
3599         let custom_tlvs = vec![((1 << 16) + 1, vec![0x42u8; 16])];
3600         let onion_fields = RecipientOnionFields::secret_only(payment_secret);
3601         let onion_fields = onion_fields.with_custom_tlvs(custom_tlvs.clone()).unwrap();
3602
3603         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3604         nodes[0].node.send_payment(payment_hash, onion_fields,
3605                 payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
3606         check_added_monitors!(nodes[0], 1); // one monitor per path
3607
3608         // Add the HTLC along the first hop.
3609         let htlc_updates = get_htlc_update_msgs(&nodes[0], &nodes[1].node.get_our_node_id());
3610         let msgs::CommitmentUpdate { update_add_htlcs, commitment_signed, .. } = htlc_updates;
3611         assert_eq!(update_add_htlcs.len(), 1);
3612         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_htlcs[0]);
3613         commitment_signed_dance!(nodes[1], nodes[0], commitment_signed, false);
3614
3615         // Attempt to forward the payment and complete the path's failure.
3616         expect_pending_htlcs_forwardable!(&nodes[1]);
3617         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1],
3618                 vec![HTLCDestination::NextHopChannel {
3619                         node_id: Some(nodes[2].node.get_our_node_id()),
3620                         channel_id: chan_2_id
3621                 }]);
3622         check_added_monitors!(nodes[1], 1);
3623
3624         let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3625         let msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. } = htlc_updates;
3626         assert_eq!(update_fail_htlcs.len(), 1);
3627         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
3628         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
3629
3630         let mut events = nodes[0].node.get_and_clear_pending_events();
3631         match events[1] {
3632                 Event::PendingHTLCsForwardable { .. } => {},
3633                 _ => panic!("Unexpected event")
3634         }
3635         events.remove(1);
3636         expect_payment_failed_conditions_event(events, payment_hash, false,
3637                 PaymentFailedConditions::new().mpp_parts_remain());
3638
3639         // Rebalance the channel so the retry of the payment can succeed.
3640         send_payment(&nodes[2], &vec!(&nodes[1])[..], 1_500_000);
3641
3642         // Retry the payment and make sure it succeeds
3643         route_params.payment_params.previously_failed_channels.push(chan_2_update.contents.short_channel_id);
3644         nodes[0].router.expect_find_route(route_params, Ok(route));
3645         nodes[0].node.process_pending_htlc_forwards();
3646         check_added_monitors!(nodes[0], 1);
3647         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
3648         assert_eq!(events.len(), 1);
3649         let payment_claimable = pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000,
3650                 payment_hash, Some(payment_secret), events.pop().unwrap(), true, None).unwrap();
3651         match payment_claimable {
3652                 Event::PaymentClaimable { onion_fields, .. } => {
3653                         assert_eq!(onion_fields.unwrap().custom_tlvs(), &custom_tlvs);
3654                 },
3655                 _ => panic!("Unexpected event"),
3656         };
3657         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
3658 }
3659
3660 #[test]
3661 fn test_custom_tlvs_consistency() {
3662         let even_type_1 = 1 << 16;
3663         let odd_type_1  = (1 << 16)+ 1;
3664         let even_type_2 = (1 << 16) + 2;
3665         let odd_type_2  = (1 << 16) + 3;
3666         let value_1 = || vec![1, 2, 3, 4];
3667         let differing_value_1 = || vec![1, 2, 3, 5];
3668         let value_2 = || vec![42u8; 16];
3669
3670         // Drop missing odd tlvs
3671         do_test_custom_tlvs_consistency(
3672                 vec![(odd_type_1, value_1()), (odd_type_2, value_2())],
3673                 vec![(odd_type_1, value_1())],
3674                 Some(vec![(odd_type_1, value_1())]),
3675         );
3676         // Drop non-matching odd tlvs
3677         do_test_custom_tlvs_consistency(
3678                 vec![(odd_type_1, value_1()), (odd_type_2, value_2())],
3679                 vec![(odd_type_1, differing_value_1()), (odd_type_2, value_2())],
3680                 Some(vec![(odd_type_2, value_2())]),
3681         );
3682         // Fail missing even tlvs
3683         do_test_custom_tlvs_consistency(
3684                 vec![(odd_type_1, value_1()), (even_type_2, value_2())],
3685                 vec![(odd_type_1, value_1())],
3686                 None,
3687         );
3688         // Fail non-matching even tlvs
3689         do_test_custom_tlvs_consistency(
3690                 vec![(even_type_1, value_1()), (odd_type_2, value_2())],
3691                 vec![(even_type_1, differing_value_1()), (odd_type_2, value_2())],
3692                 None,
3693         );
3694 }
3695
3696 fn do_test_custom_tlvs_consistency(first_tlvs: Vec<(u64, Vec<u8>)>, second_tlvs: Vec<(u64, Vec<u8>)>,
3697         expected_receive_tlvs: Option<Vec<(u64, Vec<u8>)>>) {
3698
3699         let chanmon_cfgs = create_chanmon_cfgs(4);
3700         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3701         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3702         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3703
3704         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
3705         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0);
3706         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0);
3707         let chan_2_3 = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0);
3708
3709         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3710                 .with_bolt11_features(nodes[3].node.invoice_features()).unwrap();
3711         let mut route = get_route!(nodes[0], payment_params, 15_000_000).unwrap();
3712         assert_eq!(route.paths.len(), 2);
3713         route.paths.sort_by(|path_a, _| {
3714                 // Sort the path so that the path through nodes[1] comes first
3715                 if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
3716                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
3717         });
3718
3719         let (our_payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(&nodes[3]);
3720         let payment_id = PaymentId([42; 32]);
3721         let amt_msat = 15_000_000;
3722
3723         // Send first part
3724         let onion_fields = RecipientOnionFields {
3725                 payment_secret: Some(our_payment_secret),
3726                 payment_metadata: None,
3727                 custom_tlvs: first_tlvs
3728         };
3729         let session_privs = nodes[0].node.test_add_new_pending_payment(our_payment_hash,
3730                         onion_fields.clone(), payment_id, &route).unwrap();
3731         let cur_height = nodes[0].best_block_info().1;
3732         nodes[0].node.test_send_payment_along_path(&route.paths[0], &our_payment_hash,
3733                 onion_fields.clone(), amt_msat, cur_height, payment_id,
3734                 &None, session_privs[0]).unwrap();
3735         check_added_monitors!(nodes[0], 1);
3736
3737         {
3738                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
3739                 assert_eq!(events.len(), 1);
3740                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], amt_msat, our_payment_hash,
3741                         Some(our_payment_secret), events.pop().unwrap(), false, None);
3742         }
3743         assert!(nodes[3].node.get_and_clear_pending_events().is_empty());
3744
3745         // Send second part
3746         let onion_fields = RecipientOnionFields {
3747                 payment_secret: Some(our_payment_secret),
3748                 payment_metadata: None,
3749                 custom_tlvs: second_tlvs
3750         };
3751         nodes[0].node.test_send_payment_along_path(&route.paths[1], &our_payment_hash,
3752                 onion_fields.clone(), amt_msat, cur_height, payment_id, &None, session_privs[1]).unwrap();
3753         check_added_monitors!(nodes[0], 1);
3754
3755         {
3756                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
3757                 assert_eq!(events.len(), 1);
3758                 let payment_event = SendEvent::from_event(events.pop().unwrap());
3759
3760                 nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
3761                 commitment_signed_dance!(nodes[2], nodes[0], payment_event.commitment_msg, false);
3762
3763                 expect_pending_htlcs_forwardable!(nodes[2]);
3764                 check_added_monitors!(nodes[2], 1);
3765
3766                 let mut events = nodes[2].node.get_and_clear_pending_msg_events();
3767                 assert_eq!(events.len(), 1);
3768                 let payment_event = SendEvent::from_event(events.pop().unwrap());
3769
3770                 nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
3771                 check_added_monitors!(nodes[3], 0);
3772                 commitment_signed_dance!(nodes[3], nodes[2], payment_event.commitment_msg, true, true);
3773         }
3774         expect_pending_htlcs_forwardable_ignore!(nodes[3]);
3775         nodes[3].node.process_pending_htlc_forwards();
3776
3777         if let Some(expected_tlvs) = expected_receive_tlvs {
3778                 // Claim and match expected
3779                 let events = nodes[3].node.get_and_clear_pending_events();
3780                 assert_eq!(events.len(), 1);
3781                 match events[0] {
3782                         Event::PaymentClaimable { ref onion_fields, .. } => {
3783                                 assert_eq!(onion_fields.clone().unwrap().custom_tlvs, expected_tlvs);
3784                         },
3785                         _ => panic!("Unexpected event"),
3786                 }
3787
3788                 do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]],
3789                         false, our_payment_preimage);
3790                 expect_payment_sent(&nodes[0], our_payment_preimage, Some(Some(2000)), true, true);
3791         } else {
3792                 // Expect fail back
3793                 let expected_destinations = vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }];
3794                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], expected_destinations);
3795                 check_added_monitors!(nodes[3], 1);
3796
3797                 let fail_updates_1 = get_htlc_update_msgs!(nodes[3], nodes[2].node.get_our_node_id());
3798                 nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &fail_updates_1.update_fail_htlcs[0]);
3799                 commitment_signed_dance!(nodes[2], nodes[3], fail_updates_1.commitment_signed, false);
3800
3801                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![
3802                         HTLCDestination::NextHopChannel {
3803                                 node_id: Some(nodes[3].node.get_our_node_id()),
3804                                 channel_id: chan_2_3.2
3805                         }]);
3806                 check_added_monitors!(nodes[2], 1);
3807
3808                 let fail_updates_2 = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
3809                 nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &fail_updates_2.update_fail_htlcs[0]);
3810                 commitment_signed_dance!(nodes[0], nodes[2], fail_updates_2.commitment_signed, false);
3811
3812                 expect_payment_failed_conditions(&nodes[0], our_payment_hash, true,
3813                         PaymentFailedConditions::new().mpp_parts_remain());
3814         }
3815 }
3816
3817 fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) {
3818         // Check that a payment metadata received on one HTLC that doesn't match the one received on
3819         // another results in the HTLC being rejected.
3820         //
3821         // We first set up a diamond shaped network, allowing us to split a payment into two HTLCs, the
3822         // first of which we'll deliver and the second of which we'll fail and then re-send with
3823         // modified payment metadata, which will in turn result in it being failed by the recipient.
3824         let chanmon_cfgs = create_chanmon_cfgs(4);
3825         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3826         let persister;
3827         let new_chain_monitor;
3828
3829         let mut config = test_default_channel_config();
3830         config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 50;
3831         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, Some(config), Some(config), Some(config)]);
3832         let nodes_0_deserialized;
3833
3834         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3835
3836         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
3837         let chan_id_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
3838         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
3839         let chan_id_cd = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
3840
3841         // Pay more than half of each channel's max, requiring MPP
3842         let amt_msat = 750_000_000;
3843         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3], Some(amt_msat));
3844         let payment_id = PaymentId(payment_hash.0);
3845         let payment_metadata = vec![44, 49, 52, 142];
3846
3847         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3848                 .with_bolt11_features(nodes[1].node.invoice_features()).unwrap();
3849         let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
3850
3851         // Send the MPP payment, delivering the updated commitment state to nodes[1].
3852         nodes[0].node.send_payment(payment_hash, RecipientOnionFields {
3853                         payment_secret: Some(payment_secret), payment_metadata: Some(payment_metadata), custom_tlvs: vec![],
3854                 }, payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
3855         check_added_monitors!(nodes[0], 2);
3856
3857         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
3858         assert_eq!(send_events.len(), 2);
3859         let first_send = SendEvent::from_event(send_events.pop().unwrap());
3860         let second_send = SendEvent::from_event(send_events.pop().unwrap());
3861
3862         let (b_recv_ev, c_recv_ev) = if first_send.node_id == nodes[1].node.get_our_node_id() {
3863                 (&first_send, &second_send)
3864         } else {
3865                 (&second_send, &first_send)
3866         };
3867         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &b_recv_ev.msgs[0]);
3868         commitment_signed_dance!(nodes[1], nodes[0], b_recv_ev.commitment_msg, false, true);
3869
3870         expect_pending_htlcs_forwardable!(nodes[1]);
3871         check_added_monitors(&nodes[1], 1);
3872         let b_forward_ev = SendEvent::from_node(&nodes[1]);
3873         nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &b_forward_ev.msgs[0]);
3874         commitment_signed_dance!(nodes[3], nodes[1], b_forward_ev.commitment_msg, false, true);
3875
3876         expect_pending_htlcs_forwardable!(nodes[3]);
3877
3878         // Before delivering the second MPP HTLC to nodes[2], disconnect nodes[2] and nodes[3], which
3879         // will result in nodes[2] failing the HTLC back.
3880         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
3881         nodes[3].node.peer_disconnected(&nodes[2].node.get_our_node_id());
3882
3883         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &c_recv_ev.msgs[0]);
3884         commitment_signed_dance!(nodes[2], nodes[0], c_recv_ev.commitment_msg, false, true);
3885
3886         let cs_fail = get_htlc_update_msgs(&nodes[2], &nodes[0].node.get_our_node_id());
3887         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail.update_fail_htlcs[0]);
3888         commitment_signed_dance!(nodes[0], nodes[2], cs_fail.commitment_signed, false, true);
3889
3890         let payment_fail_retryable_evs = nodes[0].node.get_and_clear_pending_events();
3891         assert_eq!(payment_fail_retryable_evs.len(), 2);
3892         if let Event::PaymentPathFailed { .. } = payment_fail_retryable_evs[0] {} else { panic!(); }
3893         if let Event::PendingHTLCsForwardable { .. } = payment_fail_retryable_evs[1] {} else { panic!(); }
3894
3895         // Before we allow the HTLC to be retried, optionally change the payment_metadata we have
3896         // stored for our payment.
3897         if do_modify {
3898                 nodes[0].node.test_set_payment_metadata(payment_id, Some(Vec::new()));
3899         }
3900
3901         // Optionally reload nodes[3] to check that the payment_metadata is properly serialized with
3902         // the payment state.
3903         if do_reload {
3904                 let mon_bd = get_monitor!(nodes[3], chan_id_bd).encode();
3905                 let mon_cd = get_monitor!(nodes[3], chan_id_cd).encode();
3906                 reload_node!(nodes[3], config, &nodes[3].node.encode(), &[&mon_bd, &mon_cd],
3907                         persister, new_chain_monitor, nodes_0_deserialized);
3908                 nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
3909                 reconnect_nodes(ReconnectArgs::new(&nodes[1], &nodes[3]));
3910         }
3911         let mut reconnect_args = ReconnectArgs::new(&nodes[2], &nodes[3]);
3912         reconnect_args.send_channel_ready = (true, true);
3913         reconnect_nodes(reconnect_args);
3914
3915         // Create a new channel between C and D as A will refuse to retry on the existing one because
3916         // it just failed.
3917         let chan_id_cd_2 = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
3918
3919         // Now retry the failed HTLC.
3920         nodes[0].node.process_pending_htlc_forwards();
3921         check_added_monitors(&nodes[0], 1);
3922         let as_resend = SendEvent::from_node(&nodes[0]);
3923         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_resend.msgs[0]);
3924         commitment_signed_dance!(nodes[2], nodes[0], as_resend.commitment_msg, false, true);
3925
3926         expect_pending_htlcs_forwardable!(nodes[2]);
3927         check_added_monitors(&nodes[2], 1);
3928         let cs_forward = SendEvent::from_node(&nodes[2]);
3929         nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &cs_forward.msgs[0]);
3930         commitment_signed_dance!(nodes[3], nodes[2], cs_forward.commitment_msg, false, true);
3931
3932         // Finally, check that nodes[3] does the correct thing - either accepting the payment or, if
3933         // the payment metadata was modified, failing only the one modified HTLC and retaining the
3934         // other.
3935         if do_modify {
3936                 expect_pending_htlcs_forwardable_ignore!(nodes[3]);
3937                 nodes[3].node.process_pending_htlc_forwards();
3938                 expect_pending_htlcs_forwardable_conditions(nodes[3].node.get_and_clear_pending_events(),
3939                         &[HTLCDestination::FailedPayment {payment_hash}]);
3940                 nodes[3].node.process_pending_htlc_forwards();
3941
3942                 check_added_monitors(&nodes[3], 1);
3943                 let ds_fail = get_htlc_update_msgs(&nodes[3], &nodes[2].node.get_our_node_id());
3944
3945                 nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &ds_fail.update_fail_htlcs[0]);
3946                 commitment_signed_dance!(nodes[2], nodes[3], ds_fail.commitment_signed, false, true);
3947                 expect_pending_htlcs_forwardable_conditions(nodes[2].node.get_and_clear_pending_events(),
3948                         &[HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_id_cd_2 }]);
3949         } else {
3950                 expect_pending_htlcs_forwardable!(nodes[3]);
3951                 expect_payment_claimable!(nodes[3], payment_hash, payment_secret, amt_msat);
3952                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
3953         }
3954 }
3955
3956 #[test]
3957 fn test_payment_metadata_consistency() {
3958         do_test_payment_metadata_consistency(true, true);
3959         do_test_payment_metadata_consistency(true, false);
3960         do_test_payment_metadata_consistency(false, true);
3961         do_test_payment_metadata_consistency(false, false);
3962 }