_ Git - rust-lightning/blob - lightning/src/ln/payment_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Tests that test the payment retry logic in ChannelManager, including various edge-cases around
  11 //! serialization ordering between ChannelManager/ChannelMonitors and ensuring we can still retry
  12 //! payments thereafter.
  13
  14 use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Listen, Watch};
  15 use crate::chain::channelmonitor::{ANTI_REORG_DELAY, HTLC_FAIL_BACK_BUFFER, LATENCY_GRACE_PERIOD_BLOCKS};
  16 use crate::sign::EntropySource;
  17 use crate::chain::transaction::OutPoint;
  18 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentFailureReason, PaymentPurpose};
  19 use crate::ln::channel::{EXPIRE_PREV_CONFIG_TICKS, commit_tx_fee_msat, get_holder_selected_channel_reserve_satoshis, ANCHOR_OUTPUT_VALUE_SATOSHI};
  20 use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, RecentPaymentDetails, RecipientOnionFields, HTLCForwardInfo, PendingHTLCRouting, PendingAddHTLCInfo};
  21 use crate::ln::features::{Bolt11InvoiceFeatures, ChannelTypeFeatures};
  22 use crate::ln::{msgs, ChannelId, PaymentHash, PaymentSecret, PaymentPreimage};
  23 use crate::ln::msgs::ChannelMessageHandler;
  24 use crate::ln::onion_utils;
  25 use crate::ln::outbound_payment::{IDEMPOTENCY_TIMEOUT_TICKS, Retry};
  26 use crate::routing::gossip::{EffectiveCapacity, RoutingFees};
  27 use crate::routing::router::{get_route, Path, PaymentParameters, Route, Router, RouteHint, RouteHintHop, RouteHop, RouteParameters, find_route};
  28 use crate::routing::scoring::ChannelUsage;
  29 use crate::util::config::UserConfig;
  30 use crate::util::test_utils;
  31 use crate::util::errors::APIError;
  32 use crate::util::ser::Writeable;
  33 use crate::util::string::UntrustedString;
  34
  35 use bitcoin::hashes::Hash;
  36 use bitcoin::hashes::sha256::Hash as Sha256;
  37 use bitcoin::network::constants::Network;
  38 use bitcoin::secp256k1::{Secp256k1, SecretKey};
  39
  40 use crate::prelude::*;
  41
  42 use crate::ln::functional_test_utils;
  43 use crate::ln::functional_test_utils::*;
  44 use crate::routing::gossip::NodeId;
  45 #[cfg(feature = "std")]
  46 use std::time::{SystemTime, Instant, Duration};
  47 #[cfg(not(feature = "no-std"))]
  48 use crate::util::time::tests::SinceEpoch;
  49
  50 #[test]
  51 fn mpp_failure() {
  52         let chanmon_cfgs = create_chanmon_cfgs(4);
  53         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
  54         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
  55         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
  56
  57         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
  58         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
  59         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
  60         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3).0.contents.short_channel_id;
  61
  62         let (mut route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
  63         let path = route.paths[0].clone();
  64         route.paths.push(path);
  65         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
  66         route.paths[0].hops[0].short_channel_id = chan_1_id;
  67         route.paths[0].hops[1].short_channel_id = chan_3_id;
  68         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
  69         route.paths[1].hops[0].short_channel_id = chan_2_id;
  70         route.paths[1].hops[1].short_channel_id = chan_4_id;
  71         send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
  72         fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
  73 }
  74
  75 #[test]
  76 fn mpp_retry() {
  77         let chanmon_cfgs = create_chanmon_cfgs(4);
  78         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
  79         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
  80         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
  81
  82         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
  83         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
  84         let (chan_3_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
  85         let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes(&nodes, 3, 2);
  86         // Rebalance
  87         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
  88
  89         let amt_msat = 1_000_000;
  90         let max_total_routing_fee_msat = 50_000;
  91         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
  92                 .with_bolt11_features(nodes[3].node.bolt11_invoice_features()).unwrap();
  93         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(
  94                 nodes[0], nodes[3], payment_params, amt_msat, Some(max_total_routing_fee_msat));
  95         let path = route.paths[0].clone();
  96         route.paths.push(path);
  97         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
  98         route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
  99         route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
 100         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
 101         route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
 102         route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
 103
 104         // Initiate the MPP payment.
 105         let payment_id = PaymentId(payment_hash.0);
 106         let mut route_params = route.route_params.clone().unwrap();
 107
 108         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
 109         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 110                 payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
 111         check_added_monitors!(nodes[0], 2); // one monitor per path
 112         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 113         assert_eq!(events.len(), 2);
 114
 115         // Pass half of the payment along the success path.
 116         let success_path_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 117         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), success_path_msgs, false, None);
 118
 119         // Add the HTLC along the first hop.
 120         let fail_path_msgs_1 = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 121         let send_event = SendEvent::from_event(fail_path_msgs_1);
 122         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 123         commitment_signed_dance!(nodes[2], nodes[0], &send_event.commitment_msg, false);
 124
 125         // Attempt to forward the payment and complete the 2nd path's failure.
 126         expect_pending_htlcs_forwardable!(&nodes[2]);
 127         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id }]);
 128         let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 129         assert!(htlc_updates.update_add_htlcs.is_empty());
 130         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 131         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 132         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 133         check_added_monitors!(nodes[2], 1);
 134         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 135         commitment_signed_dance!(nodes[0], nodes[2], htlc_updates.commitment_signed, false);
 136         let mut events = nodes[0].node.get_and_clear_pending_events();
 137         match events[1] {
 138                 Event::PendingHTLCsForwardable { .. } => {},
 139                 _ => panic!("Unexpected event")
 140         }
 141         events.remove(1);
 142         expect_payment_failed_conditions_event(events, payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 143
 144         // Rebalance the channel so the second half of the payment can succeed.
 145         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 146
 147         // Retry the second half of the payment and make sure it succeeds.
 148         route.paths.remove(0);
 149         route_params.final_value_msat = 1_000_000;
 150         route_params.payment_params.previously_failed_channels.push(chan_4_update.contents.short_channel_id);
 151         // Check the remaining max total routing fee for the second attempt is 50_000 - 1_000 msat fee
 152         // used by the first path
 153         route_params.max_total_routing_fee_msat = Some(max_total_routing_fee_msat - 1_000);
 154         route.route_params = Some(route_params.clone());
 155         nodes[0].router.expect_find_route(route_params, Ok(route));
 156         nodes[0].node.process_pending_htlc_forwards();
 157         check_added_monitors!(nodes[0], 1);
 158         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 159         assert_eq!(events.len(), 1);
 160         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 161         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 162 }
 163
 164 #[test]
 165 fn mpp_retry_overpay() {
 166         // We create an MPP scenario with two paths in which we need to overpay to reach
 167         // htlc_minimum_msat. We then fail the overpaid path and check that on retry our
 168         // max_total_routing_fee_msat only accounts for the path's fees, but not for the fees overpaid
 169         // in the first attempt.
 170         let chanmon_cfgs = create_chanmon_cfgs(4);
 171         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 172         let mut user_config = test_default_channel_config();
 173         user_config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 100;
 174         let mut limited_config_1 = user_config.clone();
 175         limited_config_1.channel_handshake_config.our_htlc_minimum_msat = 35_000_000;
 176         let mut limited_config_2 = user_config.clone();
 177         limited_config_2.channel_handshake_config.our_htlc_minimum_msat = 34_500_000;
 178         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs,
 179                 &[Some(user_config), Some(limited_config_1), Some(limited_config_2), Some(user_config)]);
 180         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 181
 182         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 40_000, 0);
 183         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 40_000, 0);
 184         let (_chan_3_update, _, _, _) = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 40_000, 0);
 185         let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes_with_value(&nodes, 3, 2, 40_000, 0);
 186
 187         let amt_msat = 70_000_000;
 188         let max_total_routing_fee_msat = Some(1_000_000);
 189
 190         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
 191                 .with_bolt11_features(nodes[3].node.bolt11_invoice_features()).unwrap();
 192         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(
 193                 nodes[0], nodes[3], payment_params, amt_msat, max_total_routing_fee_msat);
 194
 195         // Check we overpay on the second path which we're about to fail.
 196         assert_eq!(chan_1_update.contents.fee_proportional_millionths, 0);
 197         let overpaid_amount_1 = route.paths[0].fee_msat() as u32 - chan_1_update.contents.fee_base_msat;
 198         assert_eq!(overpaid_amount_1, 0);
 199
 200         assert_eq!(chan_2_update.contents.fee_proportional_millionths, 0);
 201         let overpaid_amount_2 = route.paths[1].fee_msat() as u32 - chan_2_update.contents.fee_base_msat;
 202
 203         let total_overpaid_amount = overpaid_amount_1 + overpaid_amount_2;
 204
 205         // Initiate the payment.
 206         let payment_id = PaymentId(payment_hash.0);
 207         let mut route_params = route.route_params.clone().unwrap();
 208
 209         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
 210         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 211                 payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
 212         check_added_monitors!(nodes[0], 2); // one monitor per path
 213         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 214         assert_eq!(events.len(), 2);
 215
 216         // Pass half of the payment along the success path.
 217         let success_path_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 218         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], amt_msat, payment_hash,
 219                 Some(payment_secret), success_path_msgs, false, None);
 220
 221         // Add the HTLC along the first hop.
 222         let fail_path_msgs_1 = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 223         let send_event = SendEvent::from_event(fail_path_msgs_1);
 224         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 225         commitment_signed_dance!(nodes[2], nodes[0], &send_event.commitment_msg, false);
 226
 227         // Attempt to forward the payment and complete the 2nd path's failure.
 228         expect_pending_htlcs_forwardable!(&nodes[2]);
 229         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2],
 230                 vec![HTLCDestination::NextHopChannel {
 231                         node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id
 232                 }]
 233         );
 234         let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 235         assert!(htlc_updates.update_add_htlcs.is_empty());
 236         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 237         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 238         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 239         check_added_monitors!(nodes[2], 1);
 240         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(),
 241                 &htlc_updates.update_fail_htlcs[0]);
 242         commitment_signed_dance!(nodes[0], nodes[2], htlc_updates.commitment_signed, false);
 243         let mut events = nodes[0].node.get_and_clear_pending_events();
 244         match events[1] {
 245                 Event::PendingHTLCsForwardable { .. } => {},
 246                 _ => panic!("Unexpected event")
 247         }
 248         events.remove(1);
 249         expect_payment_failed_conditions_event(events, payment_hash, false,
 250                 PaymentFailedConditions::new().mpp_parts_remain());
 251
 252         // Rebalance the channel so the second half of the payment can succeed.
 253         send_payment(&nodes[3], &vec!(&nodes[2])[..], 38_000_000);
 254
 255         // Retry the second half of the payment and make sure it succeeds.
 256         let first_path_value = route.paths[0].final_value_msat();
 257         assert_eq!(first_path_value, 36_000_000);
 258
 259         route.paths.remove(0);
 260         route_params.final_value_msat -= first_path_value;
 261         route_params.payment_params.previously_failed_channels.push(chan_4_update.contents.short_channel_id);
 262         // Check the remaining max total routing fee for the second attempt accounts only for 1_000 msat
 263         // base fee, but not for overpaid value of the first try.
 264         route_params.max_total_routing_fee_msat.as_mut().map(|m| *m -= 1000);
 265
 266         route.route_params = Some(route_params.clone());
 267         nodes[0].router.expect_find_route(route_params, Ok(route));
 268         nodes[0].node.process_pending_htlc_forwards();
 269
 270         check_added_monitors!(nodes[0], 1);
 271         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 272         assert_eq!(events.len(), 1);
 273         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], amt_msat, payment_hash,
 274                 Some(payment_secret), events.pop().unwrap(), true, None);
 275
 276         // Can't use claim_payment_along_route as it doesn't support overpayment, so we break out the
 277         // individual steps here.
 278         let extra_fees = vec![0, total_overpaid_amount];
 279         let expected_total_fee_msat = do_claim_payment_along_route_with_extra_penultimate_hop_fees(
 280                 &nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], &extra_fees[..], false,
 281                 payment_preimage);
 282         expect_payment_sent!(&nodes[0], payment_preimage, Some(expected_total_fee_msat));
 283 }
 284
 285 fn do_mpp_receive_timeout(send_partial_mpp: bool) {
 286         let chanmon_cfgs = create_chanmon_cfgs(4);
 287         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 288         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 289         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 290
 291         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 292         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 293         let (chan_3_update, _, chan_3_id, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 294         let (chan_4_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 295
 296         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 100_000);
 297         let path = route.paths[0].clone();
 298         route.paths.push(path);
 299         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
 300         route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
 301         route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
 302         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
 303         route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
 304         route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
 305
 306         // Initiate the MPP payment.
 307         nodes[0].node.send_payment_with_route(&route, payment_hash,
 308                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 309         check_added_monitors!(nodes[0], 2); // one monitor per path
 310         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 311         assert_eq!(events.len(), 2);
 312
 313         // Pass half of the payment along the first path.
 314         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 315         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_1_msgs, false, None);
 316
 317         if send_partial_mpp {
 318                 // Time out the partial MPP
 319                 for _ in 0..MPP_TIMEOUT_TICKS {
 320                         nodes[3].node.timer_tick_occurred();
 321                 }
 322
 323                 // Failed HTLC from node 3 -> 1
 324                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 325                 let htlc_fail_updates_3_1 = get_htlc_update_msgs!(nodes[3], nodes[1].node.get_our_node_id());
 326                 assert_eq!(htlc_fail_updates_3_1.update_fail_htlcs.len(), 1);
 327                 nodes[1].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &htlc_fail_updates_3_1.update_fail_htlcs[0]);
 328                 check_added_monitors!(nodes[3], 1);
 329                 commitment_signed_dance!(nodes[1], nodes[3], htlc_fail_updates_3_1.commitment_signed, false);
 330
 331                 // Failed HTLC from node 1 -> 0
 332                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_3_id }]);
 333                 let htlc_fail_updates_1_0 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 334                 assert_eq!(htlc_fail_updates_1_0.update_fail_htlcs.len(), 1);
 335                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates_1_0.update_fail_htlcs[0]);
 336                 check_added_monitors!(nodes[1], 1);
 337                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates_1_0.commitment_signed, false);
 338
 339                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain().expected_htlc_error_data(23, &[][..]));
 340         } else {
 341                 // Pass half of the payment along the second path.
 342                 let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 343                 pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_2_msgs, true, None);
 344
 345                 // Even after MPP_TIMEOUT_TICKS we should not timeout the MPP if we have all the parts
 346                 for _ in 0..MPP_TIMEOUT_TICKS {
 347                         nodes[3].node.timer_tick_occurred();
 348                 }
 349
 350                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 351         }
 352 }
 353
 354 #[test]
 355 fn mpp_receive_timeout() {
 356         do_mpp_receive_timeout(true);
 357         do_mpp_receive_timeout(false);
 358 }
 359
 360 #[test]
 361 fn test_keysend_payments() {
 362         do_test_keysend_payments(false, false);
 363         do_test_keysend_payments(false, true);
 364         do_test_keysend_payments(true, false);
 365         do_test_keysend_payments(true, true);
 366 }
 367
 368 fn do_test_keysend_payments(public_node: bool, with_retry: bool) {
 369         let chanmon_cfgs = create_chanmon_cfgs(2);
 370         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 371         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 372         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 373
 374         if public_node {
 375                 create_announced_chan_between_nodes(&nodes, 0, 1);
 376         } else {
 377                 create_chan_between_nodes(&nodes[0], &nodes[1]);
 378         }
 379         let payer_pubkey = nodes[0].node.get_our_node_id();
 380         let payee_pubkey = nodes[1].node.get_our_node_id();
 381         let route_params = RouteParameters::from_payment_params_and_value(
 382                 PaymentParameters::for_keysend(payee_pubkey, 40, false), 10000);
 383
 384         let network_graph = nodes[0].network_graph;
 385         let channels = nodes[0].node.list_usable_channels();
 386         let first_hops = channels.iter().collect::<Vec<_>>();
 387         let first_hops = if public_node { None } else { Some(first_hops.as_slice()) };
 388
 389         let scorer = test_utils::TestScorer::new();
 390         let random_seed_bytes = chanmon_cfgs[1].keys_manager.get_secure_random_bytes();
 391         let route = find_route(
 392                 &payer_pubkey, &route_params, &network_graph, first_hops,
 393                 nodes[0].logger, &scorer, &Default::default(), &random_seed_bytes
 394         ).unwrap();
 395
 396         {
 397                 let test_preimage = PaymentPreimage([42; 32]);
 398                 if with_retry {
 399                         nodes[0].node.send_spontaneous_payment_with_retry(Some(test_preimage),
 400                                 RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0),
 401                                 route_params, Retry::Attempts(1)).unwrap()
 402                 } else {
 403                         nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage),
 404                                 RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0)).unwrap()
 405                 };
 406         }
 407         check_added_monitors!(nodes[0], 1);
 408         let send_event = SendEvent::from_node(&nodes[0]);
 409         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 410         do_commitment_signed_dance(&nodes[1], &nodes[0], &send_event.commitment_msg, false, false);
 411         expect_pending_htlcs_forwardable!(nodes[1]);
 412         // Previously, a refactor caused us to stop including the payment preimage in the onion which
 413         // is sent as a part of keysend payments. Thus, to be extra careful here, we scope the preimage
 414         // above to demonstrate that we have no way to get the preimage at this point except by
 415         // extracting it from the onion nodes[1] received.
 416         let event = nodes[1].node.get_and_clear_pending_events();
 417         assert_eq!(event.len(), 1);
 418         if let Event::PaymentClaimable { purpose: PaymentPurpose::SpontaneousPayment(preimage), .. } = event[0] {
 419                 claim_payment(&nodes[0], &[&nodes[1]], preimage);
 420         } else { panic!(); }
 421 }
 422
 423 #[test]
 424 fn test_mpp_keysend() {
 425         let mut mpp_keysend_config = test_default_channel_config();
 426         mpp_keysend_config.accept_mpp_keysend = true;
 427         let chanmon_cfgs = create_chanmon_cfgs(4);
 428         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 429         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, Some(mpp_keysend_config)]);
 430         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 431
 432         create_announced_chan_between_nodes(&nodes, 0, 1);
 433         create_announced_chan_between_nodes(&nodes, 0, 2);
 434         create_announced_chan_between_nodes(&nodes, 1, 3);
 435         create_announced_chan_between_nodes(&nodes, 2, 3);
 436         let network_graph = nodes[0].network_graph;
 437
 438         let payer_pubkey = nodes[0].node.get_our_node_id();
 439         let payee_pubkey = nodes[3].node.get_our_node_id();
 440         let recv_value = 15_000_000;
 441         let route_params = RouteParameters::from_payment_params_and_value(
 442                 PaymentParameters::for_keysend(payee_pubkey, 40, true), recv_value);
 443         let scorer = test_utils::TestScorer::new();
 444         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
 445         let route = find_route(&payer_pubkey, &route_params, &network_graph, None, nodes[0].logger,
 446                 &scorer, &Default::default(), &random_seed_bytes).unwrap();
 447
 448         let payment_preimage = PaymentPreimage([42; 32]);
 449         let payment_secret = PaymentSecret(payment_preimage.0);
 450         let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage),
 451                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_preimage.0)).unwrap();
 452         check_added_monitors!(nodes[0], 2);
 453
 454         let expected_route: &[&[&Node]] = &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]];
 455         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 456         assert_eq!(events.len(), 2);
 457
 458         let ev = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 459         pass_along_path(&nodes[0], expected_route[0], recv_value, payment_hash.clone(),
 460                 Some(payment_secret), ev.clone(), false, Some(payment_preimage));
 461
 462         let ev = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 463         pass_along_path(&nodes[0], expected_route[1], recv_value, payment_hash.clone(),
 464                 Some(payment_secret), ev.clone(), true, Some(payment_preimage));
 465         claim_payment_along_route(&nodes[0], expected_route, false, payment_preimage);
 466 }
 467
 468 #[test]
 469 fn test_reject_mpp_keysend_htlc() {
 470         // This test enforces that we reject MPP keysend HTLCs if our config states we don't support
 471         // MPP keysend. When receiving a payment, if we don't support MPP keysend we'll reject the
 472         // payment if it's keysend and has a payment secret, never reaching our payment validation
 473         // logic. To check that we enforce rejecting MPP keysends in our payment logic, here we send
 474         // keysend payments without payment secrets, then modify them by adding payment secrets in the
 475         // final node in between receiving the HTLCs and actually processing them.
 476         let mut reject_mpp_keysend_cfg = test_default_channel_config();
 477         reject_mpp_keysend_cfg.accept_mpp_keysend = false;
 478
 479         let chanmon_cfgs = create_chanmon_cfgs(4);
 480         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 481         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, Some(reject_mpp_keysend_cfg)]);
 482         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 483         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
 484         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
 485         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
 486         let (update_a, _, chan_4_channel_id, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 487         let chan_4_id = update_a.contents.short_channel_id;
 488         let amount = 40_000;
 489         let (mut route, payment_hash, payment_preimage, _) = get_route_and_payment_hash!(nodes[0], nodes[3], amount);
 490
 491         // Pay along nodes[1]
 492         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
 493         route.paths[0].hops[0].short_channel_id = chan_1_id;
 494         route.paths[0].hops[1].short_channel_id = chan_3_id;
 495
 496         let payment_id_0 = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 497         nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), RecipientOnionFields::spontaneous_empty(), payment_id_0).unwrap();
 498         check_added_monitors!(nodes[0], 1);
 499
 500         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 501         let update_add_0 = update_0.update_add_htlcs[0].clone();
 502         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_0);
 503         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
 504         expect_pending_htlcs_forwardable!(nodes[1]);
 505
 506         check_added_monitors!(&nodes[1], 1);
 507         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[3].node.get_our_node_id());
 508         let update_add_1 = update_1.update_add_htlcs[0].clone();
 509         nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &update_add_1);
 510         commitment_signed_dance!(nodes[3], nodes[1], update_1.commitment_signed, false, true);
 511
 512         assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
 513         for (_, pending_forwards) in nodes[3].node.forward_htlcs.lock().unwrap().iter_mut() {
 514                 for f in pending_forwards.iter_mut() {
 515                         match f {
 516                                 &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) => {
 517                                         match forward_info.routing {
 518                                                 PendingHTLCRouting::ReceiveKeysend { ref mut payment_data, .. } => {
 519                                                         *payment_data = Some(msgs::FinalOnionHopData {
 520                                                                 payment_secret: PaymentSecret([42; 32]),
 521                                                                 total_msat: amount * 2,
 522                                                         });
 523                                                 },
 524                                                 _ => panic!("Expected PendingHTLCRouting::ReceiveKeysend"),
 525                                         }
 526                                 },
 527                                 _ => {},
 528                         }
 529                 }
 530         }
 531         expect_pending_htlcs_forwardable!(nodes[3]);
 532
 533         // Pay along nodes[2]
 534         route.paths[0].hops[0].pubkey = nodes[2].node.get_our_node_id();
 535         route.paths[0].hops[0].short_channel_id = chan_2_id;
 536         route.paths[0].hops[1].short_channel_id = chan_4_id;
 537
 538         let payment_id_1 = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 539         nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), RecipientOnionFields::spontaneous_empty(), payment_id_1).unwrap();
 540         check_added_monitors!(nodes[0], 1);
 541
 542         let update_2 = get_htlc_update_msgs!(nodes[0], nodes[2].node.get_our_node_id());
 543         let update_add_2 = update_2.update_add_htlcs[0].clone();
 544         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_2);
 545         commitment_signed_dance!(nodes[2], nodes[0], &update_2.commitment_signed, false, true);
 546         expect_pending_htlcs_forwardable!(nodes[2]);
 547
 548         check_added_monitors!(&nodes[2], 1);
 549         let update_3 = get_htlc_update_msgs!(nodes[2], nodes[3].node.get_our_node_id());
 550         let update_add_3 = update_3.update_add_htlcs[0].clone();
 551         nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &update_add_3);
 552         commitment_signed_dance!(nodes[3], nodes[2], update_3.commitment_signed, false, true);
 553
 554         assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
 555         for (_, pending_forwards) in nodes[3].node.forward_htlcs.lock().unwrap().iter_mut() {
 556                 for f in pending_forwards.iter_mut() {
 557                         match f {
 558                                 &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) => {
 559                                         match forward_info.routing {
 560                                                 PendingHTLCRouting::ReceiveKeysend { ref mut payment_data, .. } => {
 561                                                         *payment_data = Some(msgs::FinalOnionHopData {
 562                                                                 payment_secret: PaymentSecret([42; 32]),
 563                                                                 total_msat: amount * 2,
 564                                                         });
 565                                                 },
 566                                                 _ => panic!("Expected PendingHTLCRouting::ReceiveKeysend"),
 567                                         }
 568                                 },
 569                                 _ => {},
 570                         }
 571                 }
 572         }
 573         expect_pending_htlcs_forwardable!(nodes[3]);
 574         check_added_monitors!(nodes[3], 1);
 575
 576         // Fail back along nodes[2]
 577         let update_fail_0 = get_htlc_update_msgs!(&nodes[3], &nodes[2].node.get_our_node_id());
 578         nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &update_fail_0.update_fail_htlcs[0]);
 579         commitment_signed_dance!(nodes[2], nodes[3], update_fail_0.commitment_signed, false);
 580         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_channel_id }]);
 581         check_added_monitors!(nodes[2], 1);
 582
 583         let update_fail_1 = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 584         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_1.update_fail_htlcs[0]);
 585         commitment_signed_dance!(nodes[0], nodes[2], update_fail_1.commitment_signed, false);
 586
 587         expect_payment_failed_conditions(&nodes[0], payment_hash, true, PaymentFailedConditions::new());
 588         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 589 }
 590
 591
 592 #[test]
 593 fn no_pending_leak_on_initial_send_failure() {
 594         // In an earlier version of our payment tracking, we'd have a retry entry even when the initial
 595         // HTLC for payment failed to send due to local channel errors (e.g. peer disconnected). In this
 596         // case, the user wouldn't have a PaymentId to retry the payment with, but we'd think we have a
 597         // pending payment forever and never time it out.
 598         // Here we test exactly that - retrying a payment when a peer was disconnected on the first
 599         // try, and then check that no pending payment is being tracked.
 600         let chanmon_cfgs = create_chanmon_cfgs(2);
 601         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 602         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 603         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 604
 605         create_announced_chan_between_nodes(&nodes, 0, 1);
 606
 607         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 608
 609         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 610         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 611
 612         unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash,
 613                         RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
 614                 ), true, APIError::ChannelUnavailable { ref err },
 615                 assert_eq!(err, "Peer for first hop currently disconnected"));
 616
 617         assert!(!nodes[0].node.has_pending_payments());
 618 }
 619
 620 fn do_retry_with_no_persist(confirm_before_reload: bool) {
 621         // If we send a pending payment and `send_payment` returns success, we should always either
 622         // return a payment failure event or a payment success event, and on failure the payment should
 623         // be retryable.
 624         //
 625         // In order to do so when the ChannelManager isn't immediately persisted (which is normal - its
 626         // always persisted asynchronously), the ChannelManager has to reload some payment data from
 627         // ChannelMonitor(s) in some cases. This tests that reloading.
 628         //
 629         // `confirm_before_reload` confirms the channel-closing commitment transaction on-chain prior
 630         // to reloading the ChannelManager, increasing test coverage in ChannelMonitor HTLC tracking
 631         // which has separate codepaths for "commitment transaction already confirmed" and not.
 632         let chanmon_cfgs = create_chanmon_cfgs(3);
 633         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 634         let persister;
 635         let new_chain_monitor;
 636         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 637         let nodes_0_deserialized;
 638         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 639
 640         let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1);
 641         let (_, _, chan_id_2, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
 642
 643         // Serialize the ChannelManager prior to sending payments
 644         let nodes_0_serialized = nodes[0].node.encode();
 645
 646         // Send two payments - one which will get to nodes[2] and will be claimed, one which we'll time
 647         // out and retry.
 648         let amt_msat = 1_000_000;
 649         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
 650         let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000);
 651         let route_params = route.route_params.unwrap().clone();
 652         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 653                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
 654         check_added_monitors!(nodes[0], 1);
 655
 656         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 657         assert_eq!(events.len(), 1);
 658         let payment_event = SendEvent::from_event(events.pop().unwrap());
 659         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 660
 661         // We relay the payment to nodes[1] while its disconnected from nodes[2], causing the payment
 662         // to be returned immediately to nodes[0], without having nodes[2] fail the inbound payment
 663         // which would prevent retry.
 664         nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id());
 665         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 666
 667         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 668         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true);
 669         // nodes[1] now immediately fails the HTLC as the next-hop channel is disconnected
 670         let _ = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 671
 672         reconnect_nodes(ReconnectArgs::new(&nodes[1], &nodes[2]));
 673
 674         let as_commitment_tx = get_local_commitment_txn!(nodes[0], chan_id)[0].clone();
 675         if confirm_before_reload {
 676                 mine_transaction(&nodes[0], &as_commitment_tx);
 677                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 678         }
 679
 680         // The ChannelMonitor should always be the latest version, as we're required to persist it
 681         // during the `commitment_signed_dance!()`.
 682         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 683         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 684
 685         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 686         // force-close the channel.
 687         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[1].node.get_our_node_id()], 100000);
 688         assert!(nodes[0].node.list_channels().is_empty());
 689         assert!(nodes[0].node.has_pending_payments());
 690         nodes[0].node.timer_tick_occurred();
 691         if !confirm_before_reload {
 692                 let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 693                 assert_eq!(as_broadcasted_txn.len(), 1);
 694                 assert_eq!(as_broadcasted_txn[0].txid(), as_commitment_tx.txid());
 695         } else {
 696                 assert!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
 697         }
 698         check_added_monitors!(nodes[0], 1);
 699
 700         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 701         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 702                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 703         }, true).unwrap();
 704         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 705
 706         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 707         // error, as the channel has hit the chain.
 708         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 709                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 710         }, false).unwrap();
 711         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 712         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 713         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 714         assert_eq!(as_err.len(), 2);
 715         match as_err[1] {
 716                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 717                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 718                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 719                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}",
 720                                 &nodes[1].node.get_our_node_id())) }, [nodes[0].node.get_our_node_id()], 100000);
 721                         check_added_monitors!(nodes[1], 1);
 722                         assert_eq!(nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 723                 },
 724                 _ => panic!("Unexpected event"),
 725         }
 726         check_closed_broadcast!(nodes[1], false);
 727
 728         // Now claim the first payment, which should allow nodes[1] to claim the payment on-chain when
 729         // we close in a moment.
 730         nodes[2].node.claim_funds(payment_preimage_1);
 731         check_added_monitors!(nodes[2], 1);
 732         expect_payment_claimed!(nodes[2], payment_hash_1, 1_000_000);
 733
 734         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 735         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 736         check_added_monitors!(nodes[1], 1);
 737         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 738         expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], None, true, false);
 739
 740         if confirm_before_reload {
 741                 let best_block = nodes[0].blocks.lock().unwrap().last().unwrap().clone();
 742                 nodes[0].node.best_block_updated(&best_block.0.header, best_block.1);
 743         }
 744
 745         // Create a new channel on which to retry the payment before we fail the payment via the
 746         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 747         // connecting several blocks while creating the channel (implying time has passed).
 748         create_announced_chan_between_nodes(&nodes, 0, 1);
 749         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 750
 751         mine_transaction(&nodes[1], &as_commitment_tx);
 752         let bs_htlc_claim_txn = {
 753                 let mut txn = nodes[1].tx_broadcaster.unique_txn_broadcast();
 754                 assert_eq!(txn.len(), 2);
 755                 check_spends!(txn[0], funding_tx);
 756                 check_spends!(txn[1], as_commitment_tx);
 757                 txn.pop().unwrap()
 758         };
 759
 760         if !confirm_before_reload {
 761                 mine_transaction(&nodes[0], &as_commitment_tx);
 762                 let txn = nodes[0].tx_broadcaster.unique_txn_broadcast();
 763                 assert_eq!(txn.len(), 1);
 764                 assert_eq!(txn[0].txid(), as_commitment_tx.txid());
 765         }
 766         mine_transaction(&nodes[0], &bs_htlc_claim_txn);
 767         expect_payment_sent(&nodes[0], payment_preimage_1, None, true, false);
 768         connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
 769         let (first_htlc_timeout_tx, second_htlc_timeout_tx) = {
 770                 let mut txn = nodes[0].tx_broadcaster.unique_txn_broadcast();
 771                 assert_eq!(txn.len(), 2);
 772                 (txn.remove(0), txn.remove(0))
 773         };
 774         check_spends!(first_htlc_timeout_tx, as_commitment_tx);
 775         check_spends!(second_htlc_timeout_tx, as_commitment_tx);
 776         if first_htlc_timeout_tx.input[0].previous_output == bs_htlc_claim_txn.input[0].previous_output {
 777                 confirm_transaction(&nodes[0], &second_htlc_timeout_tx);
 778         } else {
 779                 confirm_transaction(&nodes[0], &first_htlc_timeout_tx);
 780         }
 781         nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 782         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new());
 783
 784         // Finally, retry the payment (which was reloaded from the ChannelMonitor when nodes[0] was
 785         // reloaded) via a route over the new channel, which work without issue and eventually be
 786         // received and claimed at the recipient just like any other payment.
 787         let (mut new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 788
 789         // Update the fee on the middle hop to ensure PaymentSent events have the correct (retried) fee
 790         // and not the original fee. We also update node[1]'s relevant config as
 791         // do_claim_payment_along_route expects us to never overpay.
 792         {
 793                 let per_peer_state = nodes[1].node.per_peer_state.read().unwrap();
 794                 let mut peer_state = per_peer_state.get(&nodes[2].node.get_our_node_id())
 795                         .unwrap().lock().unwrap();
 796                 let mut channel = peer_state.channel_by_id.get_mut(&chan_id_2).unwrap();
 797                 let mut new_config = channel.context().config();
 798                 new_config.forwarding_fee_base_msat += 100_000;
 799                 channel.context_mut().update_config(&new_config);
 800                 new_route.paths[0].hops[0].fee_msat += 100_000;
 801         }
 802
 803         // Force expiration of the channel's previous config.
 804         for _ in 0..EXPIRE_PREV_CONFIG_TICKS {
 805                 nodes[1].node.timer_tick_occurred();
 806         }
 807
 808         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash, // Shouldn't be allowed to retry a fulfilled payment
 809                 RecipientOnionFields::secret_only(payment_secret), payment_id_1).is_err());
 810         nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 811                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 812         check_added_monitors!(nodes[0], 1);
 813         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 814         assert_eq!(events.len(), 1);
 815         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 816         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
 817         expect_payment_sent!(nodes[0], payment_preimage, Some(new_route.paths[0].hops[0].fee_msat));
 818 }
 819
 820 #[test]
 821 fn retry_with_no_persist() {
 822         do_retry_with_no_persist(true);
 823         do_retry_with_no_persist(false);
 824 }
 825
 826 fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 827         // Test that an off-chain completed payment is not retryable on restart. This was previously
 828         // broken for dust payments, but we test for both dust and non-dust payments.
 829         //
 830         // `use_dust` switches to using a dust HTLC, which results in the HTLC not having an on-chain
 831         // output at all.
 832         let chanmon_cfgs = create_chanmon_cfgs(3);
 833         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 834
 835         let mut manually_accept_config = test_default_channel_config();
 836         manually_accept_config.manually_accept_inbound_channels = true;
 837
 838         let first_persister;
 839         let first_new_chain_monitor;
 840         let second_persister;
 841         let second_new_chain_monitor;
 842         let third_persister;
 843         let third_new_chain_monitor;
 844
 845         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(manually_accept_config), None]);
 846         let first_nodes_0_deserialized;
 847         let second_nodes_0_deserialized;
 848         let third_nodes_0_deserialized;
 849
 850         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 851
 852         // Because we set nodes[1] to manually accept channels, just open a 0-conf channel.
 853         let (funding_tx, chan_id) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 854         confirm_transaction(&nodes[0], &funding_tx);
 855         confirm_transaction(&nodes[1], &funding_tx);
 856         // Ignore the announcement_signatures messages
 857         nodes[0].node.get_and_clear_pending_msg_events();
 858         nodes[1].node.get_and_clear_pending_msg_events();
 859         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 860
 861         // Serialize the ChannelManager prior to sending payments
 862         let mut nodes_0_serialized = nodes[0].node.encode();
 863
 864         let route = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 }).0;
 865         let (payment_preimage, payment_hash, payment_secret, payment_id) = send_along_route(&nodes[0], route, &[&nodes[1], &nodes[2]], if use_dust { 1_000 } else { 1_000_000 });
 866
 867         // The ChannelMonitor should always be the latest version, as we're required to persist it
 868         // during the `commitment_signed_dance!()`.
 869         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 870
 871         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized], first_persister, first_new_chain_monitor, first_nodes_0_deserialized);
 872         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 873
 874         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 875         // force-close the channel.
 876         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[1].node.get_our_node_id()], 100000);
 877         nodes[0].node.timer_tick_occurred();
 878         assert!(nodes[0].node.list_channels().is_empty());
 879         assert!(nodes[0].node.has_pending_payments());
 880         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 881         check_added_monitors!(nodes[0], 1);
 882
 883         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 884                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 885         }, true).unwrap();
 886         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 887
 888         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 889         // error, as the channel has hit the chain.
 890         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 891                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 892         }, false).unwrap();
 893         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 894         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 895         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 896         assert_eq!(as_err.len(), 2);
 897         let bs_commitment_tx;
 898         match as_err[1] {
 899                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 900                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 901                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 902                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) }
 903                                 , [nodes[0].node.get_our_node_id()], 100000);
 904                         check_added_monitors!(nodes[1], 1);
 905                         bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 906                 },
 907                 _ => panic!("Unexpected event"),
 908         }
 909         check_closed_broadcast!(nodes[1], false);
 910
 911         // Now fail back the payment from nodes[2] to nodes[1]. This doesn't really matter as the
 912         // previous hop channel is already on-chain, but it makes nodes[2] willing to see additional
 913         // incoming HTLCs with the same payment hash later.
 914         nodes[2].node.fail_htlc_backwards(&payment_hash);
 915         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
 916         check_added_monitors!(nodes[2], 1);
 917
 918         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 919         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fail_htlcs[0]);
 920         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 921         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1],
 922                 [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 923
 924         // Connect the HTLC-Timeout transaction, timing out the HTLC on both nodes (but not confirming
 925         // the HTLC-Timeout transaction beyond 1 conf). For dust HTLCs, the HTLC is considered resolved
 926         // after the commitment transaction, so always connect the commitment transaction.
 927         mine_transaction(&nodes[0], &bs_commitment_tx[0]);
 928         if nodes[0].connect_style.borrow().updates_best_block_first() {
 929                 let _ = nodes[0].tx_broadcaster.txn_broadcast();
 930         }
 931         mine_transaction(&nodes[1], &bs_commitment_tx[0]);
 932         if !use_dust {
 933                 connect_blocks(&nodes[0], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
 934                 connect_blocks(&nodes[1], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
 935                 let as_htlc_timeout = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 936                 assert_eq!(as_htlc_timeout.len(), 1);
 937                 check_spends!(as_htlc_timeout[0], bs_commitment_tx[0]);
 938
 939                 mine_transaction(&nodes[0], &as_htlc_timeout[0]);
 940                 mine_transaction(&nodes[1], &as_htlc_timeout[0]);
 941         }
 942         if nodes[0].connect_style.borrow().updates_best_block_first() {
 943                 let _ = nodes[0].tx_broadcaster.txn_broadcast();
 944         }
 945
 946         // Create a new channel on which to retry the payment before we fail the payment via the
 947         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 948         // connecting several blocks while creating the channel (implying time has passed).
 949         // We do this with a zero-conf channel to avoid connecting blocks as a side-effect.
 950         let (_, chan_id_3) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 951         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 952
 953         // If we attempt to retry prior to the HTLC-Timeout (or commitment transaction, for dust HTLCs)
 954         // confirming, we will fail as it's considered still-pending...
 955         let (new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 });
 956         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 957                 Err(PaymentSendFailure::DuplicatePayment) => {},
 958                 _ => panic!("Unexpected error")
 959         }
 960         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 961
 962         // After ANTI_REORG_DELAY confirmations, the HTLC should be failed and we can try the payment
 963         // again. We serialize the node first as we'll then test retrying the HTLC after a restart
 964         // (which should also still work).
 965         connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
 966         connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 967         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new());
 968
 969         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 970         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 971         nodes_0_serialized = nodes[0].node.encode();
 972
 973         // After the payment failed, we're free to send it again.
 974         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 975                 RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
 976         assert!(!nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 977
 978         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], second_persister, second_new_chain_monitor, second_nodes_0_deserialized);
 979         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 980
 981         nodes[0].node.test_process_background_events();
 982         check_added_monitors(&nodes[0], 1);
 983
 984         let mut reconnect_args = ReconnectArgs::new(&nodes[0], &nodes[1]);
 985         reconnect_args.send_channel_ready = (true, true);
 986         reconnect_nodes(reconnect_args);
 987
 988         // Now resend the payment, delivering the HTLC and actually claiming it this time. This ensures
 989         // the payment is not (spuriously) listed as still pending.
 990         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 991                 RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
 992         check_added_monitors!(nodes[0], 1);
 993         pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], if use_dust { 1_000 } else { 1_000_000 }, payment_hash, payment_secret);
 994         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
 995
 996         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 997                 Err(PaymentSendFailure::DuplicatePayment) => {},
 998                 _ => panic!("Unexpected error")
 999         }
1000         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1001
1002         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
1003         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
1004         nodes_0_serialized = nodes[0].node.encode();
1005
1006         // Check that after reload we can send the payment again (though we shouldn't, since it was
1007         // claimed previously).
1008         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], third_persister, third_new_chain_monitor, third_nodes_0_deserialized);
1009         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
1010
1011         nodes[0].node.test_process_background_events();
1012         check_added_monitors(&nodes[0], 1);
1013
1014         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1015
1016         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
1017                 Err(PaymentSendFailure::DuplicatePayment) => {},
1018                 _ => panic!("Unexpected error")
1019         }
1020         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1021 }
1022
1023 #[test]
1024 fn test_completed_payment_not_retryable_on_reload() {
1025         do_test_completed_payment_not_retryable_on_reload(true);
1026         do_test_completed_payment_not_retryable_on_reload(false);
1027 }
1028
1029
1030 fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, confirm_commitment_tx: bool, payment_timeout: bool) {
1031         // When a Channel is closed, any outbound HTLCs which were relayed through it are simply
1032         // dropped when the Channel is. From there, the ChannelManager relies on the ChannelMonitor
1033         // having a copy of the relevant fail-/claim-back data and processes the HTLC fail/claim when
1034         // the ChannelMonitor tells it to.
1035         //
1036         // If, due to an on-chain event, an HTLC is failed/claimed, we should avoid providing the
1037         // ChannelManager the HTLC event until after the monitor is re-persisted. This should prevent a
1038         // duplicate HTLC fail/claim (e.g. via a PaymentPathFailed event).
1039         let chanmon_cfgs = create_chanmon_cfgs(2);
1040         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1041         let persister;
1042         let new_chain_monitor;
1043         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1044         let nodes_0_deserialized;
1045         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1046
1047         let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1);
1048
1049         // Route a payment, but force-close the channel before the HTLC fulfill message arrives at
1050         // nodes[0].
1051         let (payment_preimage, payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 10_000_000);
1052         nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap();
1053         check_closed_broadcast!(nodes[0], true);
1054         check_added_monitors!(nodes[0], 1);
1055         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 100000);
1056
1057         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1058         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
1059
1060         // Connect blocks until the CLTV timeout is up so that we get an HTLC-Timeout transaction
1061         connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
1062         let (commitment_tx, htlc_timeout_tx) = {
1063                 let mut txn = nodes[0].tx_broadcaster.unique_txn_broadcast();
1064                 assert_eq!(txn.len(), 2);
1065                 check_spends!(txn[0], funding_tx);
1066                 check_spends!(txn[1], txn[0]);
1067                 (txn.remove(0), txn.remove(0))
1068         };
1069
1070         nodes[1].node.claim_funds(payment_preimage);
1071         check_added_monitors!(nodes[1], 1);
1072         expect_payment_claimed!(nodes[1], payment_hash, 10_000_000);
1073
1074         mine_transaction(&nodes[1], &commitment_tx);
1075         check_closed_broadcast!(nodes[1], true);
1076         check_added_monitors!(nodes[1], 1);
1077         check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed, [nodes[0].node.get_our_node_id()], 100000);
1078         let htlc_success_tx = {
1079                 let mut txn = nodes[1].tx_broadcaster.txn_broadcast();
1080                 assert_eq!(txn.len(), 1);
1081                 check_spends!(txn[0], commitment_tx);
1082                 txn.pop().unwrap()
1083         };
1084
1085         mine_transaction(&nodes[0], &commitment_tx);
1086
1087         if confirm_commitment_tx {
1088                 connect_blocks(&nodes[0], BREAKDOWN_TIMEOUT as u32 - 1);
1089         }
1090
1091         let claim_block = create_dummy_block(nodes[0].best_block_hash(), 42, if payment_timeout { vec![htlc_timeout_tx] } else { vec![htlc_success_tx] });
1092
1093         if payment_timeout {
1094                 assert!(confirm_commitment_tx); // Otherwise we're spending below our CSV!
1095                 connect_block(&nodes[0], &claim_block);
1096                 connect_blocks(&nodes[0], ANTI_REORG_DELAY - 2);
1097         }
1098
1099         // Now connect the HTLC claim transaction with the ChainMonitor-generated ChannelMonitor update
1100         // returning InProgress. This should cause the claim event to never make its way to the
1101         // ChannelManager.
1102         chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap().clear();
1103         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
1104
1105         if payment_timeout {
1106                 connect_blocks(&nodes[0], 1);
1107         } else {
1108                 connect_block(&nodes[0], &claim_block);
1109         }
1110
1111         let funding_txo = OutPoint { txid: funding_tx.txid(), index: 0 };
1112         let mon_updates: Vec<_> = chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap()
1113                 .get_mut(&funding_txo).unwrap().drain().collect();
1114         // If we are using chain::Confirm instead of chain::Listen, we will get the same update twice.
1115         // If we're testing connection idempotency we may get substantially more.
1116         assert!(mon_updates.len() >= 1);
1117         assert!(nodes[0].chain_monitor.release_pending_monitor_events().is_empty());
1118         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1119
1120         // If we persist the ChannelManager here, we should get the PaymentSent event after
1121         // deserialization.
1122         let mut chan_manager_serialized = Vec::new();
1123         if !persist_manager_post_event {
1124                 chan_manager_serialized = nodes[0].node.encode();
1125         }
1126
1127         // Now persist the ChannelMonitor and inform the ChainMonitor that we're done, generating the
1128         // payment sent event.
1129         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1130         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
1131         for update in mon_updates {
1132                 nodes[0].chain_monitor.chain_monitor.channel_monitor_updated(funding_txo, update).unwrap();
1133         }
1134         if payment_timeout {
1135                 expect_payment_failed!(nodes[0], payment_hash, false);
1136         } else {
1137                 expect_payment_sent(&nodes[0], payment_preimage, None, true, false);
1138         }
1139
1140         // If we persist the ChannelManager after we get the PaymentSent event, we shouldn't get it
1141         // twice.
1142         if persist_manager_post_event {
1143                 chan_manager_serialized = nodes[0].node.encode();
1144         }
1145
1146         // Now reload nodes[0]...
1147         reload_node!(nodes[0], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
1148
1149         if persist_manager_post_event {
1150                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1151         } else if payment_timeout {
1152                 expect_payment_failed!(nodes[0], payment_hash, false);
1153         } else {
1154                 expect_payment_sent(&nodes[0], payment_preimage, None, true, false);
1155         }
1156
1157         // Note that if we re-connect the block which exposed nodes[0] to the payment preimage (but
1158         // which the current ChannelMonitor has not seen), the ChannelManager's de-duplication of
1159         // payment events should kick in, leaving us with no pending events here.
1160         let height = nodes[0].blocks.lock().unwrap().len() as u32 - 1;
1161         nodes[0].chain_monitor.chain_monitor.block_connected(&claim_block, height);
1162         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1163         check_added_monitors(&nodes[0], 1);
1164 }
1165
1166 #[test]
1167 fn test_dup_htlc_onchain_fails_on_reload() {
1168         do_test_dup_htlc_onchain_fails_on_reload(true, true, true);
1169         do_test_dup_htlc_onchain_fails_on_reload(true, true, false);
1170         do_test_dup_htlc_onchain_fails_on_reload(true, false, false);
1171         do_test_dup_htlc_onchain_fails_on_reload(false, true, true);
1172         do_test_dup_htlc_onchain_fails_on_reload(false, true, false);
1173         do_test_dup_htlc_onchain_fails_on_reload(false, false, false);
1174 }
1175
1176 #[test]
1177 fn test_fulfill_restart_failure() {
1178         // When we receive an update_fulfill_htlc message, we immediately consider the HTLC fully
1179         // fulfilled. At this point, the peer can reconnect and decide to either fulfill the HTLC
1180         // again, or fail it, giving us free money.
1181         //
1182         // Of course probably they won't fail it and give us free money, but because we have code to
1183         // handle it, we should test the logic for it anyway. We do that here.
1184         let chanmon_cfgs = create_chanmon_cfgs(2);
1185         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1186         let persister;
1187         let new_chain_monitor;
1188         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1189         let nodes_1_deserialized;
1190         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1191
1192         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1193         let (payment_preimage, payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 100_000);
1194
1195         // The simplest way to get a failure after a fulfill is to reload nodes[1] from a state
1196         // pre-fulfill, which we do by serializing it here.
1197         let chan_manager_serialized = nodes[1].node.encode();
1198         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id).encode();
1199
1200         nodes[1].node.claim_funds(payment_preimage);
1201         check_added_monitors!(nodes[1], 1);
1202         expect_payment_claimed!(nodes[1], payment_hash, 100_000);
1203
1204         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1205         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
1206         expect_payment_sent(&nodes[0], payment_preimage, None, false, false);
1207
1208         // Now reload nodes[1]...
1209         reload_node!(nodes[1], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1210
1211         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1212         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1213
1214         nodes[1].node.fail_htlc_backwards(&payment_hash);
1215         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]);
1216         check_added_monitors!(nodes[1], 1);
1217         let htlc_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1218         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates.update_fail_htlcs[0]);
1219         commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates.commitment_signed, false);
1220         // nodes[0] shouldn't generate any events here, while it just got a payment failure completion
1221         // it had already considered the payment fulfilled, and now they just got free money.
1222         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1223 }
1224
1225 #[test]
1226 fn get_ldk_payment_preimage() {
1227         // Ensure that `ChannelManager::get_payment_preimage` can successfully be used to claim a payment.
1228         let chanmon_cfgs = create_chanmon_cfgs(2);
1229         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1230         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1231         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1232         create_announced_chan_between_nodes(&nodes, 0, 1);
1233
1234         let amt_msat = 60_000;
1235         let expiry_secs = 60 * 60;
1236         let (payment_hash, payment_secret) = nodes[1].node.create_inbound_payment(Some(amt_msat), expiry_secs, None).unwrap();
1237
1238         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
1239                 .with_bolt11_features(nodes[1].node.bolt11_invoice_features()).unwrap();
1240         let scorer = test_utils::TestScorer::new();
1241         let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet);
1242         let random_seed_bytes = keys_manager.get_secure_random_bytes();
1243         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
1244         let route = get_route( &nodes[0].node.get_our_node_id(), &route_params,
1245                 &nodes[0].network_graph.read_only(),
1246                 Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()), nodes[0].logger,
1247                 &scorer, &Default::default(), &random_seed_bytes).unwrap();
1248         nodes[0].node.send_payment_with_route(&route, payment_hash,
1249                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
1250         check_added_monitors!(nodes[0], 1);
1251
1252         // Make sure to use `get_payment_preimage`
1253         let payment_preimage = nodes[1].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1254         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1255         assert_eq!(events.len(), 1);
1256         pass_along_path(&nodes[0], &[&nodes[1]], amt_msat, payment_hash, Some(payment_secret), events.pop().unwrap(), true, Some(payment_preimage));
1257         claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, payment_preimage);
1258 }
1259
1260 #[test]
1261 fn sent_probe_is_probe_of_sending_node() {
1262         let chanmon_cfgs = create_chanmon_cfgs(3);
1263         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1264         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1265         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1266
1267         create_announced_chan_between_nodes(&nodes, 0, 1);
1268         create_announced_chan_between_nodes(&nodes, 1, 2);
1269
1270         // First check we refuse to build a single-hop probe
1271         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[1], 100_000);
1272         assert!(nodes[0].node.send_probe(route.paths[0].clone()).is_err());
1273
1274         // Then build an actual two-hop probing path
1275         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
1276
1277         match nodes[0].node.send_probe(route.paths[0].clone()) {
1278                 Ok((payment_hash, payment_id)) => {
1279                         assert!(nodes[0].node.payment_is_probe(&payment_hash, &payment_id));
1280                         assert!(!nodes[1].node.payment_is_probe(&payment_hash, &payment_id));
1281                         assert!(!nodes[2].node.payment_is_probe(&payment_hash, &payment_id));
1282                 },
1283                 _ => panic!(),
1284         }
1285
1286         get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1287         check_added_monitors!(nodes[0], 1);
1288 }
1289
1290 #[test]
1291 fn successful_probe_yields_event() {
1292         let chanmon_cfgs = create_chanmon_cfgs(3);
1293         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1294         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1295         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1296
1297         create_announced_chan_between_nodes(&nodes, 0, 1);
1298         create_announced_chan_between_nodes(&nodes, 1, 2);
1299
1300         let recv_value = 100_000;
1301         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], recv_value);
1302
1303         let res = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1304
1305         let expected_route: &[&[&Node]] = &[&[&nodes[1], &nodes[2]]];
1306
1307         send_probe_along_route(&nodes[0], expected_route);
1308
1309         expect_probe_successful_events(&nodes[0], vec![res]);
1310
1311         assert!(!nodes[0].node.has_pending_payments());
1312 }
1313
1314 #[test]
1315 fn failed_probe_yields_event() {
1316         let chanmon_cfgs = create_chanmon_cfgs(3);
1317         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1318         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1319         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1320
1321         create_announced_chan_between_nodes(&nodes, 0, 1);
1322         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 100000, 90000000);
1323
1324         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1325
1326         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], payment_params, 9_998_000);
1327
1328         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1329
1330         // node[0] -- update_add_htlcs -> node[1]
1331         check_added_monitors!(nodes[0], 1);
1332         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1333         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1334         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1335         check_added_monitors!(nodes[1], 0);
1336         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1337         expect_pending_htlcs_forwardable!(nodes[1]);
1338
1339         // node[0] <- update_fail_htlcs -- node[1]
1340         check_added_monitors!(nodes[1], 1);
1341         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1342         // Skip the PendingHTLCsForwardable event
1343         let _events = nodes[1].node.get_and_clear_pending_events();
1344         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1345         check_added_monitors!(nodes[0], 0);
1346         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1347
1348         let mut events = nodes[0].node.get_and_clear_pending_events();
1349         assert_eq!(events.len(), 1);
1350         match events.drain(..).next().unwrap() {
1351                 crate::events::Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1352                         assert_eq!(payment_id, ev_pid);
1353                         assert_eq!(payment_hash, ev_ph);
1354                 },
1355                 _ => panic!(),
1356         };
1357         assert!(!nodes[0].node.has_pending_payments());
1358 }
1359
1360 #[test]
1361 fn onchain_failed_probe_yields_event() {
1362         // Tests that an attempt to probe over a channel that is eventaully closed results in a failure
1363         // event.
1364         let chanmon_cfgs = create_chanmon_cfgs(3);
1365         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1366         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1367         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1368
1369         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1370         create_announced_chan_between_nodes(&nodes, 1, 2);
1371
1372         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1373
1374         // Send a dust HTLC, which will be treated as if it timed out once the channel hits the chain.
1375         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], payment_params, 1_000);
1376         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1377
1378         // node[0] -- update_add_htlcs -> node[1]
1379         check_added_monitors!(nodes[0], 1);
1380         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1381         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1382         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1383         check_added_monitors!(nodes[1], 0);
1384         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1385         expect_pending_htlcs_forwardable!(nodes[1]);
1386
1387         check_added_monitors!(nodes[1], 1);
1388         let _ = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1389
1390         // Don't bother forwarding the HTLC onwards and just confirm the force-close transaction on
1391         // Node A, which after 6 confirmations should result in a probe failure event.
1392         let bs_txn = get_local_commitment_txn!(nodes[1], chan_id);
1393         confirm_transaction(&nodes[0], &bs_txn[0]);
1394         check_closed_broadcast!(&nodes[0], true);
1395         check_added_monitors!(nodes[0], 1);
1396
1397         let mut events = nodes[0].node.get_and_clear_pending_events();
1398         assert_eq!(events.len(), 2);
1399         let mut found_probe_failed = false;
1400         for event in events.drain(..) {
1401                 match event {
1402                         Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1403                                 assert_eq!(payment_id, ev_pid);
1404                                 assert_eq!(payment_hash, ev_ph);
1405                                 found_probe_failed = true;
1406                         },
1407                         Event::ChannelClosed { .. } => {},
1408                         _ => panic!(),
1409                 }
1410         }
1411         assert!(found_probe_failed);
1412         assert!(!nodes[0].node.has_pending_payments());
1413 }
1414
1415 #[test]
1416 fn preflight_probes_yield_event_skip_private_hop() {
1417         let chanmon_cfgs = create_chanmon_cfgs(5);
1418         let node_cfgs = create_node_cfgs(5, &chanmon_cfgs);
1419
1420         // We alleviate the HTLC max-in-flight limit, as otherwise we'd always be limited through that.
1421         let mut no_htlc_limit_config = test_default_channel_config();
1422         no_htlc_limit_config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 100;
1423
1424         let user_configs = std::iter::repeat(no_htlc_limit_config).take(5).map(|c| Some(c)).collect::<Vec<Option<UserConfig>>>();
1425         let node_chanmgrs = create_node_chanmgrs(5, &node_cfgs, &user_configs);
1426         let nodes = create_network(5, &node_cfgs, &node_chanmgrs);
1427
1428         // Setup channel topology:
1429         //            N0 -(1M:0)- N1 -(1M:0)- N2 -(70k:0)- N3 -(50k:0)- N4
1430
1431         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
1432         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 1_000_000, 0);
1433         create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 70_000, 0);
1434         create_unannounced_chan_between_nodes_with_value(&nodes, 3, 4, 50_000, 0);
1435
1436         let mut invoice_features = Bolt11InvoiceFeatures::empty();
1437         invoice_features.set_basic_mpp_optional();
1438
1439         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
1440                 .with_bolt11_features(invoice_features).unwrap();
1441
1442         let recv_value = 50_000_000;
1443         let route_params = RouteParameters::from_payment_params_and_value(payment_params, recv_value);
1444         let res = nodes[0].node.send_preflight_probes(route_params, None).unwrap();
1445
1446         let expected_route: &[&[&Node]] = &[&[&nodes[1], &nodes[2], &nodes[3]]];
1447
1448         assert_eq!(res.len(), expected_route.len());
1449
1450         send_probe_along_route(&nodes[0], expected_route);
1451
1452         expect_probe_successful_events(&nodes[0], res.clone());
1453
1454         assert!(!nodes[0].node.has_pending_payments());
1455 }
1456
1457 #[test]
1458 fn preflight_probes_yield_event() {
1459         let chanmon_cfgs = create_chanmon_cfgs(4);
1460         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
1461
1462         // We alleviate the HTLC max-in-flight limit, as otherwise we'd always be limited through that.
1463         let mut no_htlc_limit_config = test_default_channel_config();
1464         no_htlc_limit_config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 100;
1465
1466         let user_configs = std::iter::repeat(no_htlc_limit_config).take(4).map(|c| Some(c)).collect::<Vec<Option<UserConfig>>>();
1467         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &user_configs);
1468         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
1469
1470         // Setup channel topology:
1471         //                    (1M:0)- N1 -(30k:0)
1472         //                   /                  \
1473         //                 N0                    N4
1474         //                   \                  /
1475         //                    (1M:0)- N2 -(70k:0)
1476         //
1477         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
1478         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
1479         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 30_000, 0);
1480         create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 70_000, 0);
1481
1482         let mut invoice_features = Bolt11InvoiceFeatures::empty();
1483         invoice_features.set_basic_mpp_optional();
1484
1485         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
1486                 .with_bolt11_features(invoice_features).unwrap();
1487
1488         let recv_value = 50_000_000;
1489         let route_params = RouteParameters::from_payment_params_and_value(payment_params, recv_value);
1490         let res = nodes[0].node.send_preflight_probes(route_params, None).unwrap();
1491
1492         let expected_route: &[&[&Node]] = &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]];
1493
1494         assert_eq!(res.len(), expected_route.len());
1495
1496         send_probe_along_route(&nodes[0], expected_route);
1497
1498         expect_probe_successful_events(&nodes[0], res.clone());
1499
1500         assert!(!nodes[0].node.has_pending_payments());
1501 }
1502
1503 #[test]
1504 fn preflight_probes_yield_event_and_skip() {
1505         let chanmon_cfgs = create_chanmon_cfgs(5);
1506         let node_cfgs = create_node_cfgs(5, &chanmon_cfgs);
1507
1508         // We alleviate the HTLC max-in-flight limit, as otherwise we'd always be limited through that.
1509         let mut no_htlc_limit_config = test_default_channel_config();
1510         no_htlc_limit_config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 100;
1511
1512         let user_configs = std::iter::repeat(no_htlc_limit_config).take(5).map(|c| Some(c)).collect::<Vec<Option<UserConfig>>>();
1513         let node_chanmgrs = create_node_chanmgrs(5, &node_cfgs, &user_configs);
1514         let nodes = create_network(5, &node_cfgs, &node_chanmgrs);
1515
1516         // Setup channel topology:
1517         //                    (30k:0)- N2 -(1M:0)
1518         //                   /                  \
1519         //  N0 -(100k:0)-> N1                    N4
1520         //                   \                  /
1521         //                    (70k:0)- N3 -(1M:0)
1522         //
1523         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
1524         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 30_000, 0);
1525         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 70_000, 0);
1526         create_announced_chan_between_nodes_with_value(&nodes, 2, 4, 1_000_000, 0);
1527         create_announced_chan_between_nodes_with_value(&nodes, 3, 4, 1_000_000, 0);
1528
1529         let mut invoice_features = Bolt11InvoiceFeatures::empty();
1530         invoice_features.set_basic_mpp_optional();
1531
1532         let payment_params = PaymentParameters::from_node_id(nodes[4].node.get_our_node_id(), TEST_FINAL_CLTV)
1533                 .with_bolt11_features(invoice_features).unwrap();
1534
1535         let recv_value = 80_000_000;
1536         let route_params = RouteParameters::from_payment_params_and_value(payment_params, recv_value);
1537         let res = nodes[0].node.send_preflight_probes(route_params, None).unwrap();
1538
1539         let expected_route : &[&[&Node]] = &[&[&nodes[1], &nodes[2], &nodes[4]]];
1540
1541         // We check that only one probe was sent, the other one was skipped due to limited liquidity.
1542         assert_eq!(res.len(), 1);
1543
1544         send_probe_along_route(&nodes[0], expected_route);
1545
1546         expect_probe_successful_events(&nodes[0], res.clone());
1547
1548         assert!(!nodes[0].node.has_pending_payments());
1549 }
1550
1551 #[test]
1552 fn claimed_send_payment_idempotent() {
1553         // Tests that `send_payment` (and friends) are (reasonably) idempotent.
1554         let chanmon_cfgs = create_chanmon_cfgs(2);
1555         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1556         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1557         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1558
1559         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1560
1561         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1562         let (first_payment_preimage, _, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1563
1564         macro_rules! check_send_rejected {
1565                 () => {
1566                         // If we try to resend a new payment with a different payment_hash but with the same
1567                         // payment_id, it should be rejected.
1568                         let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1569                                 RecipientOnionFields::secret_only(second_payment_secret), payment_id);
1570                         match send_result {
1571                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1572                                 _ => panic!("Unexpected send result: {:?}", send_result),
1573                         }
1574
1575                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1576                         // also be rejected.
1577                         let send_result = nodes[0].node.send_spontaneous_payment(
1578                                 &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
1579                         match send_result {
1580                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1581                                 _ => panic!("Unexpected send result: {:?}", send_result),
1582                         }
1583                 }
1584         }
1585
1586         check_send_rejected!();
1587
1588         // Claim the payment backwards, but note that the PaymentSent event is still pending and has
1589         // not been seen by the user. At this point, from the user perspective nothing has changed, so
1590         // we must remain just as idempotent as we were before.
1591         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, first_payment_preimage);
1592
1593         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1594                 nodes[0].node.timer_tick_occurred();
1595         }
1596
1597         check_send_rejected!();
1598
1599         // Once the user sees and handles the `PaymentSent` event, we expect them to no longer call
1600         // `send_payment`, and our idempotency guarantees are off - they should have atomically marked
1601         // the payment complete. However, they could have called `send_payment` while the event was
1602         // being processed, leading to a race in our idempotency guarantees. Thus, even immediately
1603         // after the event is handled a duplicate payment should sitll be rejected.
1604         expect_payment_sent!(&nodes[0], first_payment_preimage, Some(0));
1605         check_send_rejected!();
1606
1607         // If relatively little time has passed, a duplicate payment should still fail.
1608         nodes[0].node.timer_tick_occurred();
1609         check_send_rejected!();
1610
1611         // However, after some time has passed (at least more than the one timer tick above), a
1612         // duplicate payment should go through, as ChannelManager should no longer have any remaining
1613         // references to the old payment data.
1614         for _ in 0..IDEMPOTENCY_TIMEOUT_TICKS {
1615                 nodes[0].node.timer_tick_occurred();
1616         }
1617
1618         nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1619                 RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
1620         check_added_monitors!(nodes[0], 1);
1621         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1622         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1623 }
1624
1625 #[test]
1626 fn abandoned_send_payment_idempotent() {
1627         // Tests that `send_payment` (and friends) allow duplicate PaymentIds immediately after
1628         // abandon_payment.
1629         let chanmon_cfgs = create_chanmon_cfgs(2);
1630         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1631         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1632         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1633
1634         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1635
1636         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1637         let (_, first_payment_hash, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1638
1639         macro_rules! check_send_rejected {
1640                 () => {
1641                         // If we try to resend a new payment with a different payment_hash but with the same
1642                         // payment_id, it should be rejected.
1643                         let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1644                                 RecipientOnionFields::secret_only(second_payment_secret), payment_id);
1645                         match send_result {
1646                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1647                                 _ => panic!("Unexpected send result: {:?}", send_result),
1648                         }
1649
1650                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1651                         // also be rejected.
1652                         let send_result = nodes[0].node.send_spontaneous_payment(
1653                                 &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
1654                         match send_result {
1655                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1656                                 _ => panic!("Unexpected send result: {:?}", send_result),
1657                         }
1658                 }
1659         }
1660
1661         check_send_rejected!();
1662
1663         nodes[1].node.fail_htlc_backwards(&first_payment_hash);
1664         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash: first_payment_hash }]);
1665
1666         // Until we abandon the payment upon path failure, no matter how many timer ticks pass, we still cannot reuse the
1667         // PaymentId.
1668         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1669                 nodes[0].node.timer_tick_occurred();
1670         }
1671         check_send_rejected!();
1672
1673         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash, PaymentFailureReason::RecipientRejected);
1674
1675         // However, we can reuse the PaymentId immediately after we `abandon_payment` upon passing the
1676         // failed payment back.
1677         nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1678                 RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
1679         check_added_monitors!(nodes[0], 1);
1680         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1681         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1682 }
1683
1684 #[derive(PartialEq)]
1685 enum InterceptTest {
1686         Forward,
1687         Fail,
1688         Timeout,
1689 }
1690
1691 #[test]
1692 fn test_trivial_inflight_htlc_tracking(){
1693         // In this test, we test three scenarios:
1694         // (1) Sending + claiming a payment successfully should return `None` when querying InFlightHtlcs
1695         // (2) Sending a payment without claiming it should return the payment's value (500000) when querying InFlightHtlcs
1696         // (3) After we claim the payment sent in (2), InFlightHtlcs should return `None` for the query.
1697         let chanmon_cfgs = create_chanmon_cfgs(3);
1698         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1699         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1700         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1701
1702         let (_, _, chan_1_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
1703         let (_, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
1704
1705         // Send and claim the payment. Inflight HTLCs should be empty.
1706         let (_, payment_hash, _, payment_id) = send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 500000);
1707         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1708         {
1709                 let mut node_0_per_peer_lock;
1710                 let mut node_0_peer_state_lock;
1711                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1712
1713                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1714                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1715                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1716                         channel_1.context().get_short_channel_id().unwrap()
1717                 );
1718                 assert_eq!(chan_1_used_liquidity, None);
1719         }
1720         {
1721                 let mut node_1_per_peer_lock;
1722                 let mut node_1_peer_state_lock;
1723                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1724
1725                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1726                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1727                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1728                         channel_2.context().get_short_channel_id().unwrap()
1729                 );
1730
1731                 assert_eq!(chan_2_used_liquidity, None);
1732         }
1733         let pending_payments = nodes[0].node.list_recent_payments();
1734         assert_eq!(pending_payments.len(), 1);
1735         assert_eq!(pending_payments[0], RecentPaymentDetails::Fulfilled { payment_hash: Some(payment_hash), payment_id });
1736
1737         // Remove fulfilled payment
1738         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1739                 nodes[0].node.timer_tick_occurred();
1740         }
1741
1742         // Send the payment, but do not claim it. Our inflight HTLCs should contain the pending payment.
1743         let (payment_preimage, payment_hash,  _, payment_id) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 500000);
1744         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1745         {
1746                 let mut node_0_per_peer_lock;
1747                 let mut node_0_peer_state_lock;
1748                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1749
1750                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1751                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1752                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1753                         channel_1.context().get_short_channel_id().unwrap()
1754                 );
1755                 // First hop accounts for expected 1000 msat fee
1756                 assert_eq!(chan_1_used_liquidity, Some(501000));
1757         }
1758         {
1759                 let mut node_1_per_peer_lock;
1760                 let mut node_1_peer_state_lock;
1761                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1762
1763                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1764                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1765                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1766                         channel_2.context().get_short_channel_id().unwrap()
1767                 );
1768
1769                 assert_eq!(chan_2_used_liquidity, Some(500000));
1770         }
1771         let pending_payments = nodes[0].node.list_recent_payments();
1772         assert_eq!(pending_payments.len(), 1);
1773         assert_eq!(pending_payments[0], RecentPaymentDetails::Pending { payment_id, payment_hash, total_msat: 500000 });
1774
1775         // Now, let's claim the payment. This should result in the used liquidity to return `None`.
1776         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
1777
1778         // Remove fulfilled payment
1779         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1780                 nodes[0].node.timer_tick_occurred();
1781         }
1782
1783         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1784         {
1785                 let mut node_0_per_peer_lock;
1786                 let mut node_0_peer_state_lock;
1787                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1788
1789                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1790                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1791                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1792                         channel_1.context().get_short_channel_id().unwrap()
1793                 );
1794                 assert_eq!(chan_1_used_liquidity, None);
1795         }
1796         {
1797                 let mut node_1_per_peer_lock;
1798                 let mut node_1_peer_state_lock;
1799                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1800
1801                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1802                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1803                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1804                         channel_2.context().get_short_channel_id().unwrap()
1805                 );
1806                 assert_eq!(chan_2_used_liquidity, None);
1807         }
1808
1809         let pending_payments = nodes[0].node.list_recent_payments();
1810         assert_eq!(pending_payments.len(), 0);
1811 }
1812
1813 #[test]
1814 fn test_holding_cell_inflight_htlcs() {
1815         let chanmon_cfgs = create_chanmon_cfgs(2);
1816         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1817         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1818         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1819         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1820
1821         let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
1822         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1823
1824         // Queue up two payments - one will be delivered right away, one immediately goes into the
1825         // holding cell as nodes[0] is AwaitingRAA.
1826         {
1827                 nodes[0].node.send_payment_with_route(&route, payment_hash_1,
1828                         RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
1829                 check_added_monitors!(nodes[0], 1);
1830                 nodes[0].node.send_payment_with_route(&route, payment_hash_2,
1831                         RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
1832                 check_added_monitors!(nodes[0], 0);
1833         }
1834
1835         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1836
1837         {
1838                 let mut node_0_per_peer_lock;
1839                 let mut node_0_peer_state_lock;
1840                 let channel =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, channel_id);
1841
1842                 let used_liquidity = inflight_htlcs.used_liquidity_msat(
1843                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1844                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1845                         channel.context().get_short_channel_id().unwrap()
1846                 );
1847
1848                 assert_eq!(used_liquidity, Some(2000000));
1849         }
1850
1851         // Clear pending events so test doesn't throw a "Had excess message on node..." error
1852         nodes[0].node.get_and_clear_pending_msg_events();
1853 }
1854
1855 #[test]
1856 fn intercepted_payment() {
1857         // Test that detecting an intercept scid on payment forward will signal LDK to generate an
1858         // intercept event, which the LSP can then use to either (a) open a JIT channel to forward the
1859         // payment or (b) fail the payment.
1860         do_test_intercepted_payment(InterceptTest::Forward);
1861         do_test_intercepted_payment(InterceptTest::Fail);
1862         // Make sure that intercepted payments will be automatically failed back if too many blocks pass.
1863         do_test_intercepted_payment(InterceptTest::Timeout);
1864 }
1865
1866 fn do_test_intercepted_payment(test: InterceptTest) {
1867         let chanmon_cfgs = create_chanmon_cfgs(3);
1868         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1869
1870         let mut zero_conf_chan_config = test_default_channel_config();
1871         zero_conf_chan_config.manually_accept_inbound_channels = true;
1872         let mut intercept_forwards_config = test_default_channel_config();
1873         intercept_forwards_config.accept_intercept_htlcs = true;
1874         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(zero_conf_chan_config)]);
1875
1876         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1877         let scorer = test_utils::TestScorer::new();
1878         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
1879
1880         let _ = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1881
1882         let amt_msat = 100_000;
1883         let intercept_scid = nodes[1].node.get_intercept_scid();
1884         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1885                 .with_route_hints(vec![
1886                         RouteHint(vec![RouteHintHop {
1887                                 src_node_id: nodes[1].node.get_our_node_id(),
1888                                 short_channel_id: intercept_scid,
1889                                 fees: RoutingFees {
1890                                         base_msat: 1000,
1891                                         proportional_millionths: 0,
1892                                 },
1893                                 cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
1894                                 htlc_minimum_msat: None,
1895                                 htlc_maximum_msat: None,
1896                         }])
1897                 ]).unwrap()
1898                 .with_bolt11_features(nodes[2].node.bolt11_invoice_features()).unwrap();
1899         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
1900         let route = get_route(
1901                 &nodes[0].node.get_our_node_id(), &route_params, &nodes[0].network_graph.read_only(), None,
1902                 nodes[0].logger, &scorer, &Default::default(), &random_seed_bytes
1903         ).unwrap();
1904
1905         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
1906         nodes[0].node.send_payment_with_route(&route, payment_hash,
1907                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
1908         let payment_event = {
1909                 {
1910                         let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
1911                         assert_eq!(added_monitors.len(), 1);
1912                         added_monitors.clear();
1913                 }
1914                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1915                 assert_eq!(events.len(), 1);
1916                 SendEvent::from_event(events.remove(0))
1917         };
1918         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1919         commitment_signed_dance!(nodes[1], nodes[0], &payment_event.commitment_msg, false, true);
1920
1921         // Check that we generate the PaymentIntercepted event when an intercept forward is detected.
1922         let events = nodes[1].node.get_and_clear_pending_events();
1923         assert_eq!(events.len(), 1);
1924         let (intercept_id, expected_outbound_amount_msat) = match events[0] {
1925                 crate::events::Event::HTLCIntercepted {
1926                         intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, inbound_amount_msat, requested_next_hop_scid: short_channel_id
1927                 } => {
1928                         assert_eq!(pmt_hash, payment_hash);
1929                         assert_eq!(inbound_amount_msat, route.get_total_amount() + route.get_total_fees());
1930                         assert_eq!(short_channel_id, intercept_scid);
1931                         (intercept_id, expected_outbound_amount_msat)
1932                 },
1933                 _ => panic!()
1934         };
1935
1936         // Check for unknown channel id error.
1937         let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &ChannelId::from_bytes([42; 32]), nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1938         assert_eq!(unknown_chan_id_err , APIError::ChannelUnavailable  {
1939                 err: format!("Channel with id {} not found for the passed counterparty node_id {}",
1940                         log_bytes!([42; 32]), nodes[2].node.get_our_node_id()) });
1941
1942         if test == InterceptTest::Fail {
1943                 // Ensure we can fail the intercepted payment back.
1944                 nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap();
1945                 expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::UnknownNextHop { requested_forward_scid: intercept_scid }]);
1946                 nodes[1].node.process_pending_htlc_forwards();
1947                 let update_fail = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1948                 check_added_monitors!(&nodes[1], 1);
1949                 assert!(update_fail.update_fail_htlcs.len() == 1);
1950                 let fail_msg = update_fail.update_fail_htlcs[0].clone();
1951                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1952                 commitment_signed_dance!(nodes[0], nodes[1], update_fail.commitment_signed, false);
1953
1954                 // Ensure the payment fails with the expected error.
1955                 let fail_conditions = PaymentFailedConditions::new()
1956                         .blamed_scid(intercept_scid)
1957                         .blamed_chan_closed(true)
1958                         .expected_htlc_error_data(0x4000 | 10, &[]);
1959                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
1960         } else if test == InterceptTest::Forward {
1961                 // Check that we'll fail as expected when sending to a channel that isn't in `ChannelReady` yet.
1962                 let temp_chan_id = nodes[1].node.create_channel(nodes[2].node.get_our_node_id(), 100_000, 0, 42, None, None).unwrap();
1963                 let unusable_chan_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &temp_chan_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1964                 assert_eq!(unusable_chan_err , APIError::ChannelUnavailable {
1965                         err: format!("Channel with id {} for the passed counterparty node_id {} is still opening.",
1966                                 temp_chan_id, nodes[2].node.get_our_node_id()) });
1967                 assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
1968
1969                 // Open the just-in-time channel so the payment can then be forwarded.
1970                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1971
1972                 // Finally, forward the intercepted payment through and claim it.
1973                 nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap();
1974                 expect_pending_htlcs_forwardable!(nodes[1]);
1975
1976                 let payment_event = {
1977                         {
1978                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
1979                                 assert_eq!(added_monitors.len(), 1);
1980                                 added_monitors.clear();
1981                         }
1982                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
1983                         assert_eq!(events.len(), 1);
1984                         SendEvent::from_event(events.remove(0))
1985                 };
1986                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1987                 commitment_signed_dance!(nodes[2], nodes[1], &payment_event.commitment_msg, false, true);
1988                 expect_pending_htlcs_forwardable!(nodes[2]);
1989
1990                 let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1991                 expect_payment_claimable!(&nodes[2], payment_hash, payment_secret, amt_msat, Some(payment_preimage), nodes[2].node.get_our_node_id());
1992                 do_claim_payment_along_route(&nodes[0], &vec!(&vec!(&nodes[1], &nodes[2])[..]), false, payment_preimage);
1993                 let events = nodes[0].node.get_and_clear_pending_events();
1994                 assert_eq!(events.len(), 2);
1995                 match events[0] {
1996                         Event::PaymentSent { payment_preimage: ref ev_preimage, payment_hash: ref ev_hash, ref fee_paid_msat, .. } => {
1997                                 assert_eq!(payment_preimage, *ev_preimage);
1998                                 assert_eq!(payment_hash, *ev_hash);
1999                                 assert_eq!(fee_paid_msat, &Some(1000));
2000                         },
2001                         _ => panic!("Unexpected event")
2002                 }
2003                 match events[1] {
2004                         Event::PaymentPathSuccessful { payment_hash: hash, .. } => {
2005                                 assert_eq!(hash, Some(payment_hash));
2006                         },
2007                         _ => panic!("Unexpected event")
2008                 }
2009                 check_added_monitors(&nodes[0], 1);
2010         } else if test == InterceptTest::Timeout {
2011                 let mut block = create_dummy_block(nodes[0].best_block_hash(), 42, Vec::new());
2012                 connect_block(&nodes[0], &block);
2013                 connect_block(&nodes[1], &block);
2014                 for _ in 0..TEST_FINAL_CLTV {
2015                         block.header.prev_blockhash = block.block_hash();
2016                         connect_block(&nodes[0], &block);
2017                         connect_block(&nodes[1], &block);
2018                 }
2019                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::InvalidForward { requested_forward_scid: intercept_scid }]);
2020                 check_added_monitors!(nodes[1], 1);
2021                 let htlc_timeout_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2022                 assert!(htlc_timeout_updates.update_add_htlcs.is_empty());
2023                 assert_eq!(htlc_timeout_updates.update_fail_htlcs.len(), 1);
2024                 assert!(htlc_timeout_updates.update_fail_malformed_htlcs.is_empty());
2025                 assert!(htlc_timeout_updates.update_fee.is_none());
2026
2027                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_timeout_updates.update_fail_htlcs[0]);
2028                 commitment_signed_dance!(nodes[0], nodes[1], htlc_timeout_updates.commitment_signed, false);
2029                 expect_payment_failed!(nodes[0], payment_hash, false, 0x2000 | 2, []);
2030
2031                 // Check for unknown intercept id error.
2032                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
2033                 let unknown_intercept_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
2034                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
2035                 let unknown_intercept_id_err = nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap_err();
2036                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
2037         }
2038 }
2039
2040 #[test]
2041 fn accept_underpaying_htlcs_config() {
2042         do_accept_underpaying_htlcs_config(1);
2043         do_accept_underpaying_htlcs_config(2);
2044         do_accept_underpaying_htlcs_config(3);
2045 }
2046
2047 fn do_accept_underpaying_htlcs_config(num_mpp_parts: usize) {
2048         let chanmon_cfgs = create_chanmon_cfgs(3);
2049         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2050         let mut intercept_forwards_config = test_default_channel_config();
2051         intercept_forwards_config.accept_intercept_htlcs = true;
2052         let mut underpay_config = test_default_channel_config();
2053         underpay_config.channel_config.accept_underpaying_htlcs = true;
2054         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(underpay_config)]);
2055         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2056
2057         let mut chan_ids = Vec::new();
2058         for _ in 0..num_mpp_parts {
2059                 let _ = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000, 0);
2060                 let channel_id = create_unannounced_chan_between_nodes_with_value(&nodes, 1, 2, 2_000_000, 0).0.channel_id;
2061                 chan_ids.push(channel_id);
2062         }
2063
2064         // Send the initial payment.
2065         let amt_msat = 900_000;
2066         let skimmed_fee_msat = 20;
2067         let mut route_hints = Vec::new();
2068         for _ in 0..num_mpp_parts {
2069                 route_hints.push(RouteHint(vec![RouteHintHop {
2070                         src_node_id: nodes[1].node.get_our_node_id(),
2071                         short_channel_id: nodes[1].node.get_intercept_scid(),
2072                         fees: RoutingFees {
2073                                 base_msat: 1000,
2074                                 proportional_millionths: 0,
2075                         },
2076                         cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
2077                         htlc_minimum_msat: None,
2078                         htlc_maximum_msat: Some(amt_msat / num_mpp_parts as u64 + 5),
2079                 }]));
2080         }
2081         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
2082                 .with_route_hints(route_hints).unwrap()
2083                 .with_bolt11_features(nodes[2].node.bolt11_invoice_features()).unwrap();
2084         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2085         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
2086         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2087                 PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
2088         check_added_monitors!(nodes[0], num_mpp_parts); // one monitor per path
2089         let mut events: Vec<SendEvent> = nodes[0].node.get_and_clear_pending_msg_events().into_iter().map(|e| SendEvent::from_event(e)).collect();
2090         assert_eq!(events.len(), num_mpp_parts);
2091
2092         // Forward the intercepted payments.
2093         for (idx, ev) in events.into_iter().enumerate() {
2094                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &ev.msgs[0]);
2095                 do_commitment_signed_dance(&nodes[1], &nodes[0], &ev.commitment_msg, false, true);
2096
2097                 let events = nodes[1].node.get_and_clear_pending_events();
2098                 assert_eq!(events.len(), 1);
2099                 let (intercept_id, expected_outbound_amt_msat) = match events[0] {
2100                         crate::events::Event::HTLCIntercepted {
2101                                 intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, ..
2102                         } => {
2103                                 assert_eq!(pmt_hash, payment_hash);
2104                                 (intercept_id, expected_outbound_amount_msat)
2105                         },
2106                         _ => panic!()
2107                 };
2108                 nodes[1].node.forward_intercepted_htlc(intercept_id, &chan_ids[idx],
2109                         nodes[2].node.get_our_node_id(), expected_outbound_amt_msat - skimmed_fee_msat).unwrap();
2110                 expect_pending_htlcs_forwardable!(nodes[1]);
2111                 let payment_event = {
2112                         {
2113                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
2114                                 assert_eq!(added_monitors.len(), 1);
2115                                 added_monitors.clear();
2116                         }
2117                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
2118                         assert_eq!(events.len(), 1);
2119                         SendEvent::from_event(events.remove(0))
2120                 };
2121                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
2122                 do_commitment_signed_dance(&nodes[2], &nodes[1], &payment_event.commitment_msg, false, true);
2123                 if idx == num_mpp_parts - 1 {
2124                         expect_pending_htlcs_forwardable!(nodes[2]);
2125                 }
2126         }
2127
2128         // Claim the payment and check that the skimmed fee is as expected.
2129         let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
2130         let events = nodes[2].node.get_and_clear_pending_events();
2131         assert_eq!(events.len(), 1);
2132         match events[0] {
2133                 crate::events::Event::PaymentClaimable {
2134                         ref payment_hash, ref purpose, amount_msat, counterparty_skimmed_fee_msat, receiver_node_id, ..
2135                 } => {
2136                         assert_eq!(payment_hash, payment_hash);
2137                         assert_eq!(amt_msat - skimmed_fee_msat * num_mpp_parts as u64, amount_msat);
2138                         assert_eq!(skimmed_fee_msat * num_mpp_parts as u64, counterparty_skimmed_fee_msat);
2139                         assert_eq!(nodes[2].node.get_our_node_id(), receiver_node_id.unwrap());
2140                         match purpose {
2141                                 crate::events::PaymentPurpose::InvoicePayment { payment_preimage: ev_payment_preimage,
2142                                         payment_secret: ev_payment_secret, .. } =>
2143                                 {
2144                                         assert_eq!(payment_preimage, ev_payment_preimage.unwrap());
2145                                         assert_eq!(payment_secret, *ev_payment_secret);
2146                                 },
2147                                 _ => panic!(),
2148                         }
2149                 },
2150                 _ => panic!("Unexpected event"),
2151         }
2152         let mut expected_paths_vecs = Vec::new();
2153         let mut expected_paths = Vec::new();
2154         for _ in 0..num_mpp_parts { expected_paths_vecs.push(vec!(&nodes[1], &nodes[2])); }
2155         for i in 0..num_mpp_parts { expected_paths.push(&expected_paths_vecs[i][..]); }
2156         let total_fee_msat = do_claim_payment_along_route_with_extra_penultimate_hop_fees(
2157                 &nodes[0], &expected_paths[..], &vec![skimmed_fee_msat as u32; num_mpp_parts][..], false,
2158                 payment_preimage);
2159         // The sender doesn't know that the penultimate hop took an extra fee.
2160         expect_payment_sent(&nodes[0], payment_preimage,
2161                 Some(Some(total_fee_msat - skimmed_fee_msat * num_mpp_parts as u64)), true, true);
2162 }
2163
2164 #[derive(PartialEq)]
2165 enum AutoRetry {
2166         Success,
2167         Spontaneous,
2168         FailAttempts,
2169         FailTimeout,
2170         FailOnRestart,
2171         FailOnRetry,
2172 }
2173
2174 #[test]
2175 fn automatic_retries() {
2176         do_automatic_retries(AutoRetry::Success);
2177         do_automatic_retries(AutoRetry::Spontaneous);
2178         do_automatic_retries(AutoRetry::FailAttempts);
2179         do_automatic_retries(AutoRetry::FailTimeout);
2180         do_automatic_retries(AutoRetry::FailOnRestart);
2181         do_automatic_retries(AutoRetry::FailOnRetry);
2182 }
2183 fn do_automatic_retries(test: AutoRetry) {
2184         // Test basic automatic payment retries in ChannelManager. See individual `test` variant comments
2185         // below.
2186         let chanmon_cfgs = create_chanmon_cfgs(3);
2187         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2188         let persister;
2189         let new_chain_monitor;
2190
2191         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2192         let node_0_deserialized;
2193
2194         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2195         let channel_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
2196         let channel_id_2 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
2197
2198         // Marshall data to send the payment
2199         #[cfg(feature = "std")]
2200         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2201         #[cfg(not(feature = "std"))]
2202         let payment_expiry_secs = 60 * 60;
2203         let amt_msat = 1000;
2204         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2205         invoice_features.set_variable_length_onion_required();
2206         invoice_features.set_payment_secret_required();
2207         invoice_features.set_basic_mpp_optional();
2208         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
2209                 .with_expiry_time(payment_expiry_secs as u64)
2210                 .with_bolt11_features(invoice_features).unwrap();
2211         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2212         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
2213
2214         macro_rules! pass_failed_attempt_with_retry_along_path {
2215                 ($failing_channel_id: expr, $expect_pending_htlcs_forwardable: expr) => {
2216                         // Send a payment attempt that fails due to lack of liquidity on the second hop
2217                         check_added_monitors!(nodes[0], 1);
2218                         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2219                         let mut update_add = update_0.update_add_htlcs[0].clone();
2220                         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
2221                         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
2222                         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
2223                         nodes[1].node.process_pending_htlc_forwards();
2224                         expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1],
2225                                 vec![HTLCDestination::NextHopChannel {
2226                                         node_id: Some(nodes[2].node.get_our_node_id()),
2227                                         channel_id: $failing_channel_id,
2228                                 }]);
2229                         nodes[1].node.process_pending_htlc_forwards();
2230                         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2231                         check_added_monitors!(&nodes[1], 1);
2232                         assert!(update_1.update_fail_htlcs.len() == 1);
2233                         let fail_msg = update_1.update_fail_htlcs[0].clone();
2234                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
2235                         commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
2236
2237                         // Ensure the attempt fails and a new PendingHTLCsForwardable event is generated for the retry
2238                         let mut events = nodes[0].node.get_and_clear_pending_events();
2239                         assert_eq!(events.len(), 2);
2240                         match events[0] {
2241                                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2242                                         assert_eq!(payment_hash, ev_payment_hash);
2243                                         assert_eq!(payment_failed_permanently, false);
2244                                 },
2245                                 _ => panic!("Unexpected event"),
2246                         }
2247                         if $expect_pending_htlcs_forwardable {
2248                                 match events[1] {
2249                                         Event::PendingHTLCsForwardable { .. } => {},
2250                                         _ => panic!("Unexpected event"),
2251                                 }
2252                         } else {
2253                                 match events[1] {
2254                                         Event::PaymentFailed { payment_hash: ev_payment_hash, .. } => {
2255                                                 assert_eq!(payment_hash, ev_payment_hash);
2256                                         },
2257                                         _ => panic!("Unexpected event"),
2258                                 }
2259                         }
2260                 }
2261         }
2262
2263         if test == AutoRetry::Success {
2264                 // Test that we can succeed on the first retry.
2265                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2266                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2267                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2268
2269                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
2270                 // attempt, since the initial second hop channel will be excluded from pathfinding
2271                 create_announced_chan_between_nodes(&nodes, 1, 2);
2272
2273                 // We retry payments in `process_pending_htlc_forwards`
2274                 nodes[0].node.process_pending_htlc_forwards();
2275                 check_added_monitors!(nodes[0], 1);
2276                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2277                 assert_eq!(msg_events.len(), 1);
2278                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, Some(payment_secret), msg_events.pop().unwrap(), true, None);
2279                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
2280         } else if test == AutoRetry::Spontaneous {
2281                 nodes[0].node.send_spontaneous_payment_with_retry(Some(payment_preimage),
2282                         RecipientOnionFields::spontaneous_empty(), PaymentId(payment_hash.0), route_params,
2283                         Retry::Attempts(1)).unwrap();
2284                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2285
2286                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
2287                 // attempt, since the initial second hop channel will be excluded from pathfinding
2288                 create_announced_chan_between_nodes(&nodes, 1, 2);
2289
2290                 // We retry payments in `process_pending_htlc_forwards`
2291                 nodes[0].node.process_pending_htlc_forwards();
2292                 check_added_monitors!(nodes[0], 1);
2293                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2294                 assert_eq!(msg_events.len(), 1);
2295                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, None, msg_events.pop().unwrap(), true, Some(payment_preimage));
2296                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
2297         } else if test == AutoRetry::FailAttempts {
2298                 // Ensure ChannelManager will not retry a payment if it has run out of payment attempts.
2299                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2300                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2301                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2302
2303                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
2304                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
2305                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
2306
2307                 // We retry payments in `process_pending_htlc_forwards`
2308                 nodes[0].node.process_pending_htlc_forwards();
2309                 pass_failed_attempt_with_retry_along_path!(channel_id_3, false);
2310
2311                 // Ensure we won't retry a second time.
2312                 nodes[0].node.process_pending_htlc_forwards();
2313                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2314                 assert_eq!(msg_events.len(), 0);
2315         } else if test == AutoRetry::FailTimeout {
2316                 #[cfg(not(feature = "no-std"))] {
2317                         // Ensure ChannelManager will not retry a payment if it times out due to Retry::Timeout.
2318                         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2319                                 PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
2320                         pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2321
2322                         // Advance the time so the second attempt fails due to timeout.
2323                         SinceEpoch::advance(Duration::from_secs(61));
2324
2325                         // Make sure we don't retry again.
2326                         nodes[0].node.process_pending_htlc_forwards();
2327                         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2328                         assert_eq!(msg_events.len(), 0);
2329
2330                         let mut events = nodes[0].node.get_and_clear_pending_events();
2331                         assert_eq!(events.len(), 1);
2332                         match events[0] {
2333                                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2334                                         assert_eq!(payment_hash, *ev_payment_hash);
2335                                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2336                                         assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
2337                                 },
2338                                 _ => panic!("Unexpected event"),
2339                         }
2340                 }
2341         } else if test == AutoRetry::FailOnRestart {
2342                 // Ensure ChannelManager will not retry a payment after restart, even if there were retry
2343                 // attempts remaining prior to restart.
2344                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2345                         PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
2346                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2347
2348                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
2349                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
2350                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
2351
2352                 // Ensure the first retry attempt fails, with 1 retry attempt remaining
2353                 nodes[0].node.process_pending_htlc_forwards();
2354                 pass_failed_attempt_with_retry_along_path!(channel_id_3, true);
2355
2356                 // Restart the node and ensure that ChannelManager does not use its remaining retry attempt
2357                 let node_encoded = nodes[0].node.encode();
2358                 let chan_1_monitor_serialized = get_monitor!(nodes[0], channel_id_1).encode();
2359                 reload_node!(nodes[0], node_encoded, &[&chan_1_monitor_serialized], persister, new_chain_monitor, node_0_deserialized);
2360
2361                 let mut events = nodes[0].node.get_and_clear_pending_events();
2362                 expect_pending_htlcs_forwardable_from_events!(nodes[0], events, true);
2363                 // Make sure we don't retry again.
2364                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2365                 assert_eq!(msg_events.len(), 0);
2366
2367                 let mut events = nodes[0].node.get_and_clear_pending_events();
2368                 assert_eq!(events.len(), 1);
2369                 match events[0] {
2370                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2371                                 assert_eq!(payment_hash, *ev_payment_hash);
2372                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2373                                 assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
2374                         },
2375                         _ => panic!("Unexpected event"),
2376                 }
2377         } else if test == AutoRetry::FailOnRetry {
2378                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2379                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2380                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2381
2382                 // We retry payments in `process_pending_htlc_forwards`. Since our channel closed, we should
2383                 // fail to find a route.
2384                 nodes[0].node.process_pending_htlc_forwards();
2385                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2386                 assert_eq!(msg_events.len(), 0);
2387
2388                 let mut events = nodes[0].node.get_and_clear_pending_events();
2389                 assert_eq!(events.len(), 1);
2390                 match events[0] {
2391                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2392                                 assert_eq!(payment_hash, *ev_payment_hash);
2393                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2394                                 assert_eq!(PaymentFailureReason::RouteNotFound, ev_reason.unwrap());
2395                         },
2396                         _ => panic!("Unexpected event"),
2397                 }
2398         }
2399 }
2400
2401 #[test]
2402 fn auto_retry_partial_failure() {
2403         // Test that we'll retry appropriately on send partial failure and retry partial failure.
2404         let chanmon_cfgs = create_chanmon_cfgs(2);
2405         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2406         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2407         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2408
2409         // Open three channels, the first has plenty of liquidity, the second and third have ~no
2410         // available liquidity, causing any outbound payments routed over it to fail immediately.
2411         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2412         let chan_2_id = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 989_000_000).0.contents.short_channel_id;
2413         let chan_3_id = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 989_000_000).0.contents.short_channel_id;
2414
2415         // Marshall data to send the payment
2416         let amt_msat = 10_000_000;
2417         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2418         #[cfg(feature = "std")]
2419         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2420         #[cfg(not(feature = "std"))]
2421         let payment_expiry_secs = 60 * 60;
2422         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2423         invoice_features.set_variable_length_onion_required();
2424         invoice_features.set_payment_secret_required();
2425         invoice_features.set_basic_mpp_optional();
2426         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2427                 .with_expiry_time(payment_expiry_secs as u64)
2428                 .with_bolt11_features(invoice_features).unwrap();
2429
2430         // Configure the initial send path
2431         let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2432         route_params.max_total_routing_fee_msat = None;
2433
2434         let send_route = Route {
2435                 paths: vec![
2436                         Path { hops: vec![RouteHop {
2437                                 pubkey: nodes[1].node.get_our_node_id(),
2438                                 node_features: nodes[1].node.node_features(),
2439                                 short_channel_id: chan_1_id,
2440                                 channel_features: nodes[1].node.channel_features(),
2441                                 fee_msat: amt_msat / 2,
2442                                 cltv_expiry_delta: 100,
2443                                 maybe_announced_channel: true,
2444                         }], blinded_tail: None },
2445                         Path { hops: vec![RouteHop {
2446                                 pubkey: nodes[1].node.get_our_node_id(),
2447                                 node_features: nodes[1].node.node_features(),
2448                                 short_channel_id: chan_2_id,
2449                                 channel_features: nodes[1].node.channel_features(),
2450                                 fee_msat: amt_msat / 2,
2451                                 cltv_expiry_delta: 100,
2452                                 maybe_announced_channel: true,
2453                         }], blinded_tail: None },
2454                 ],
2455                 route_params: Some(route_params.clone()),
2456         };
2457         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
2458
2459         // Configure the retry1 paths
2460         let mut payment_params = route_params.payment_params.clone();
2461         payment_params.previously_failed_channels.push(chan_2_id);
2462         let mut retry_1_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat / 2);
2463         retry_1_params.max_total_routing_fee_msat = None;
2464
2465         let retry_1_route = Route {
2466                 paths: vec![
2467                         Path { hops: vec![RouteHop {
2468                                 pubkey: nodes[1].node.get_our_node_id(),
2469                                 node_features: nodes[1].node.node_features(),
2470                                 short_channel_id: chan_1_id,
2471                                 channel_features: nodes[1].node.channel_features(),
2472                                 fee_msat: amt_msat / 4,
2473                                 cltv_expiry_delta: 100,
2474                                 maybe_announced_channel: true,
2475                         }], blinded_tail: None },
2476                         Path { hops: vec![RouteHop {
2477                                 pubkey: nodes[1].node.get_our_node_id(),
2478                                 node_features: nodes[1].node.node_features(),
2479                                 short_channel_id: chan_3_id,
2480                                 channel_features: nodes[1].node.channel_features(),
2481                                 fee_msat: amt_msat / 4,
2482                                 cltv_expiry_delta: 100,
2483                                 maybe_announced_channel: true,
2484                         }], blinded_tail: None },
2485                 ],
2486                 route_params: Some(retry_1_params.clone()),
2487         };
2488         nodes[0].router.expect_find_route(retry_1_params.clone(), Ok(retry_1_route));
2489
2490         // Configure the retry2 path
2491         let mut payment_params = retry_1_params.payment_params.clone();
2492         payment_params.previously_failed_channels.push(chan_3_id);
2493         let mut retry_2_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat / 4);
2494         retry_2_params.max_total_routing_fee_msat = None;
2495
2496         let retry_2_route = Route {
2497                 paths: vec![
2498                         Path { hops: vec![RouteHop {
2499                                 pubkey: nodes[1].node.get_our_node_id(),
2500                                 node_features: nodes[1].node.node_features(),
2501                                 short_channel_id: chan_1_id,
2502                                 channel_features: nodes[1].node.channel_features(),
2503                                 fee_msat: amt_msat / 4,
2504                                 cltv_expiry_delta: 100,
2505                                 maybe_announced_channel: true,
2506                         }], blinded_tail: None },
2507                 ],
2508                 route_params: Some(retry_2_params.clone()),
2509         };
2510         nodes[0].router.expect_find_route(retry_2_params, Ok(retry_2_route));
2511
2512         // Send a payment that will partially fail on send, then partially fail on retry, then succeed.
2513         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2514                 PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap();
2515         let payment_failed_events = nodes[0].node.get_and_clear_pending_events();
2516         assert_eq!(payment_failed_events.len(), 2);
2517         match payment_failed_events[0] {
2518                 Event::PaymentPathFailed { .. } => {},
2519                 _ => panic!("Unexpected event"),
2520         }
2521         match payment_failed_events[1] {
2522                 Event::PaymentPathFailed { .. } => {},
2523                 _ => panic!("Unexpected event"),
2524         }
2525
2526         // Pass the first part of the payment along the path.
2527         check_added_monitors!(nodes[0], 1); // only one HTLC actually made it out
2528         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2529
2530         // Only one HTLC/channel update actually made it out
2531         assert_eq!(msg_events.len(), 1);
2532         let mut payment_event = SendEvent::from_event(msg_events.remove(0));
2533
2534         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2535         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
2536         check_added_monitors!(nodes[1], 1);
2537         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2538
2539         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2540         check_added_monitors!(nodes[0], 1);
2541         let as_second_htlc_updates = SendEvent::from_node(&nodes[0]);
2542
2543         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2544         check_added_monitors!(nodes[0], 1);
2545         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2546
2547         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2548         check_added_monitors!(nodes[1], 1);
2549
2550         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[0]);
2551         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[1]);
2552         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.commitment_msg);
2553         check_added_monitors!(nodes[1], 1);
2554         let (bs_second_raa, bs_second_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2555
2556         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2557         check_added_monitors!(nodes[0], 1);
2558
2559         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_cs);
2560         check_added_monitors!(nodes[0], 1);
2561         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2562
2563         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2564         check_added_monitors!(nodes[1], 1);
2565
2566         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
2567         nodes[1].node.process_pending_htlc_forwards();
2568         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, amt_msat);
2569         nodes[1].node.claim_funds(payment_preimage);
2570         expect_payment_claimed!(nodes[1], payment_hash, amt_msat);
2571         let bs_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2572         assert_eq!(bs_claim_update.update_fulfill_htlcs.len(), 1);
2573
2574         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_claim_update.update_fulfill_htlcs[0]);
2575         expect_payment_sent(&nodes[0], payment_preimage, None, false, false);
2576         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_claim_update.commitment_signed);
2577         check_added_monitors!(nodes[0], 1);
2578         let (as_third_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2579
2580         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2581         check_added_monitors!(nodes[1], 4);
2582         let bs_second_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2583
2584         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2585         check_added_monitors!(nodes[1], 1);
2586         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2587
2588         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2589         check_added_monitors!(nodes[0], 1);
2590         expect_payment_path_successful!(nodes[0]);
2591
2592         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[0]);
2593         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[1]);
2594         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.commitment_signed);
2595         check_added_monitors!(nodes[0], 1);
2596         let (as_fourth_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2597
2598         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_fourth_raa);
2599         check_added_monitors!(nodes[1], 1);
2600
2601         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2602         check_added_monitors!(nodes[1], 1);
2603         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2604
2605         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2606         check_added_monitors!(nodes[0], 1);
2607         let events = nodes[0].node.get_and_clear_pending_events();
2608         assert_eq!(events.len(), 2);
2609         if let Event::PaymentPathSuccessful { .. } = events[0] {} else { panic!(); }
2610         if let Event::PaymentPathSuccessful { .. } = events[1] {} else { panic!(); }
2611 }
2612
2613 #[test]
2614 fn auto_retry_zero_attempts_send_error() {
2615         let chanmon_cfgs = create_chanmon_cfgs(2);
2616         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2617         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2618         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2619
2620         // Open a single channel that does not have sufficient liquidity for the payment we want to
2621         // send.
2622         let chan_id  = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 989_000_000).0.contents.short_channel_id;
2623
2624         // Marshall data to send the payment
2625         let amt_msat = 10_000_000;
2626         let (_, payment_hash, payment_secret) = get_payment_preimage_hash(&nodes[1], Some(amt_msat), None);
2627         #[cfg(feature = "std")]
2628         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2629         #[cfg(not(feature = "std"))]
2630         let payment_expiry_secs = 60 * 60;
2631         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2632         invoice_features.set_variable_length_onion_required();
2633         invoice_features.set_payment_secret_required();
2634         invoice_features.set_basic_mpp_optional();
2635         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2636                 .with_expiry_time(payment_expiry_secs as u64)
2637                 .with_bolt11_features(invoice_features).unwrap();
2638         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2639
2640         // Override the route search to return a route, rather than failing at the route-finding step.
2641         let send_route = Route {
2642                 paths: vec![
2643                         Path { hops: vec![RouteHop {
2644                                 pubkey: nodes[1].node.get_our_node_id(),
2645                                 node_features: nodes[1].node.node_features(),
2646                                 short_channel_id: chan_id,
2647                                 channel_features: nodes[1].node.channel_features(),
2648                                 fee_msat: amt_msat,
2649                                 cltv_expiry_delta: 100,
2650                                 maybe_announced_channel: true,
2651                         }], blinded_tail: None },
2652                 ],
2653                 route_params: Some(route_params.clone()),
2654         };
2655         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
2656
2657         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2658                 PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
2659         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
2660         let events = nodes[0].node.get_and_clear_pending_events();
2661         assert_eq!(events.len(), 2);
2662         if let Event::PaymentPathFailed { .. } = events[0] { } else { panic!(); }
2663         if let Event::PaymentFailed { .. } = events[1] { } else { panic!(); }
2664         check_added_monitors!(nodes[0], 0);
2665 }
2666
2667 #[test]
2668 fn fails_paying_after_rejected_by_payee() {
2669         let chanmon_cfgs = create_chanmon_cfgs(2);
2670         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2671         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2672         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2673
2674         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2675
2676         // Marshall data to send the payment
2677         let amt_msat = 20_000;
2678         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2679         #[cfg(feature = "std")]
2680         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2681         #[cfg(not(feature = "std"))]
2682         let payment_expiry_secs = 60 * 60;
2683         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2684         invoice_features.set_variable_length_onion_required();
2685         invoice_features.set_payment_secret_required();
2686         invoice_features.set_basic_mpp_optional();
2687         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2688                 .with_expiry_time(payment_expiry_secs as u64)
2689                 .with_bolt11_features(invoice_features).unwrap();
2690         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2691
2692         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2693                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2694         check_added_monitors!(nodes[0], 1);
2695         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
2696         assert_eq!(events.len(), 1);
2697         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
2698         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2699         check_added_monitors!(nodes[1], 0);
2700         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
2701         expect_pending_htlcs_forwardable!(nodes[1]);
2702         expect_payment_claimable!(&nodes[1], payment_hash, payment_secret, amt_msat);
2703
2704         nodes[1].node.fail_htlc_backwards(&payment_hash);
2705         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash }]);
2706         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
2707 }
2708
2709 #[test]
2710 fn retry_multi_path_single_failed_payment() {
2711         // Tests that we can/will retry after a single path of an MPP payment failed immediately
2712         let chanmon_cfgs = create_chanmon_cfgs(2);
2713         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2714         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2715         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2716
2717         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2718         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2719
2720         let amt_msat = 100_010_000;
2721
2722         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2723         #[cfg(feature = "std")]
2724         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2725         #[cfg(not(feature = "std"))]
2726         let payment_expiry_secs = 60 * 60;
2727         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2728         invoice_features.set_variable_length_onion_required();
2729         invoice_features.set_payment_secret_required();
2730         invoice_features.set_basic_mpp_optional();
2731         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2732                 .with_expiry_time(payment_expiry_secs as u64)
2733                 .with_bolt11_features(invoice_features).unwrap();
2734         let mut route_params = RouteParameters::from_payment_params_and_value(
2735                 payment_params.clone(), amt_msat);
2736         route_params.max_total_routing_fee_msat = None;
2737
2738         let chans = nodes[0].node.list_usable_channels();
2739         let mut route = Route {
2740                 paths: vec![
2741                         Path { hops: vec![RouteHop {
2742                                 pubkey: nodes[1].node.get_our_node_id(),
2743                                 node_features: nodes[1].node.node_features(),
2744                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2745                                 channel_features: nodes[1].node.channel_features(),
2746                                 fee_msat: 10_000,
2747                                 cltv_expiry_delta: 100,
2748                                 maybe_announced_channel: true,
2749                         }], blinded_tail: None },
2750                         Path { hops: vec![RouteHop {
2751                                 pubkey: nodes[1].node.get_our_node_id(),
2752                                 node_features: nodes[1].node.node_features(),
2753                                 short_channel_id: chans[1].short_channel_id.unwrap(),
2754                                 channel_features: nodes[1].node.channel_features(),
2755                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2756                                 cltv_expiry_delta: 100,
2757                                 maybe_announced_channel: true,
2758                         }], blinded_tail: None },
2759                 ],
2760                 route_params: Some(route_params.clone()),
2761         };
2762         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2763         // On retry, split the payment across both channels.
2764         route.paths[0].hops[0].fee_msat = 50_000_001;
2765         route.paths[1].hops[0].fee_msat = 50_000_000;
2766         let mut pay_params = route.route_params.clone().unwrap().payment_params;
2767         pay_params.previously_failed_channels.push(chans[1].short_channel_id.unwrap());
2768
2769         let mut retry_params = RouteParameters::from_payment_params_and_value(pay_params, 100_000_000);
2770         retry_params.max_total_routing_fee_msat = None;
2771         route.route_params = Some(retry_params.clone());
2772         nodes[0].router.expect_find_route(retry_params, Ok(route.clone()));
2773
2774         {
2775                 let scorer = chanmon_cfgs[0].scorer.read().unwrap();
2776                 // The initial send attempt, 2 paths
2777                 scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 10_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2778                 scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 100_000_001, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2779                 // The retry, 2 paths. Ensure that the in-flight HTLC amount is factored in.
2780                 scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_001, inflight_htlc_msat: 10_000, effective_capacity: EffectiveCapacity::Unknown });
2781                 scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2782         }
2783
2784         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2785                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2786         let events = nodes[0].node.get_and_clear_pending_events();
2787         assert_eq!(events.len(), 1);
2788         match events[0] {
2789                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false,
2790                         failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }},
2791                         short_channel_id: Some(expected_scid), .. } =>
2792                 {
2793                         assert_eq!(payment_hash, ev_payment_hash);
2794                         assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
2795                 },
2796                 _ => panic!("Unexpected event"),
2797         }
2798         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2799         assert_eq!(htlc_msgs.len(), 2);
2800         check_added_monitors!(nodes[0], 2);
2801 }
2802
2803 #[test]
2804 fn immediate_retry_on_failure() {
2805         // Tests that we can/will retry immediately after a failure
2806         let chanmon_cfgs = create_chanmon_cfgs(2);
2807         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2808         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2809         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2810
2811         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2812         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2813
2814         let amt_msat = 100_000_001;
2815         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2816         #[cfg(feature = "std")]
2817         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2818         #[cfg(not(feature = "std"))]
2819         let payment_expiry_secs = 60 * 60;
2820         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2821         invoice_features.set_variable_length_onion_required();
2822         invoice_features.set_payment_secret_required();
2823         invoice_features.set_basic_mpp_optional();
2824         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2825                 .with_expiry_time(payment_expiry_secs as u64)
2826                 .with_bolt11_features(invoice_features).unwrap();
2827         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2828
2829         let chans = nodes[0].node.list_usable_channels();
2830         let mut route = Route {
2831                 paths: vec![
2832                         Path { hops: vec![RouteHop {
2833                                 pubkey: nodes[1].node.get_our_node_id(),
2834                                 node_features: nodes[1].node.node_features(),
2835                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2836                                 channel_features: nodes[1].node.channel_features(),
2837                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2838                                 cltv_expiry_delta: 100,
2839                                 maybe_announced_channel: true,
2840                         }], blinded_tail: None },
2841                 ],
2842                 route_params: Some(route_params.clone()),
2843         };
2844         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2845         // On retry, split the payment across both channels.
2846         route.paths.push(route.paths[0].clone());
2847         route.paths[0].hops[0].short_channel_id = chans[1].short_channel_id.unwrap();
2848         route.paths[0].hops[0].fee_msat = 50_000_000;
2849         route.paths[1].hops[0].fee_msat = 50_000_001;
2850         let mut pay_params = route_params.payment_params.clone();
2851         pay_params.previously_failed_channels.push(chans[0].short_channel_id.unwrap());
2852         let retry_params = RouteParameters::from_payment_params_and_value(pay_params, amt_msat);
2853         route.route_params = Some(retry_params.clone());
2854         nodes[0].router.expect_find_route(retry_params, Ok(route.clone()));
2855
2856         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2857                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2858         let events = nodes[0].node.get_and_clear_pending_events();
2859         assert_eq!(events.len(), 1);
2860         match events[0] {
2861                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false,
2862                         failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }},
2863                         short_channel_id: Some(expected_scid), .. } =>
2864                 {
2865                         assert_eq!(payment_hash, ev_payment_hash);
2866                         assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
2867                 },
2868                 _ => panic!("Unexpected event"),
2869         }
2870         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2871         assert_eq!(htlc_msgs.len(), 2);
2872         check_added_monitors!(nodes[0], 2);
2873 }
2874
2875 #[test]
2876 fn no_extra_retries_on_back_to_back_fail() {
2877         // In a previous release, we had a race where we may exceed the payment retry count if we
2878         // get two failures in a row with the second indicating that all paths had failed (this field,
2879         // `all_paths_failed`, has since been removed).
2880         // Generally, when we give up trying to retry a payment, we don't know for sure what the
2881         // current state of the ChannelManager event queue is. Specifically, we cannot be sure that
2882         // there are not multiple additional `PaymentPathFailed` or even `PaymentSent` events
2883         // pending which we will see later. Thus, when we previously removed the retry tracking map
2884         // entry after a `all_paths_failed` `PaymentPathFailed` event, we may have dropped the
2885         // retry entry even though more events for the same payment were still pending. This led to
2886         // us retrying a payment again even though we'd already given up on it.
2887         //
2888         // We now have a separate event - `PaymentFailed` which indicates no HTLCs remain and which
2889         // is used to remove the payment retry counter entries instead. This tests for the specific
2890         // excess-retry case while also testing `PaymentFailed` generation.
2891
2892         let chanmon_cfgs = create_chanmon_cfgs(3);
2893         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2894         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2895         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2896
2897         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2898         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2899
2900         let amt_msat = 200_000_000;
2901         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2902         #[cfg(feature = "std")]
2903         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2904         #[cfg(not(feature = "std"))]
2905         let payment_expiry_secs = 60 * 60;
2906         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2907         invoice_features.set_variable_length_onion_required();
2908         invoice_features.set_payment_secret_required();
2909         invoice_features.set_basic_mpp_optional();
2910         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2911                 .with_expiry_time(payment_expiry_secs as u64)
2912                 .with_bolt11_features(invoice_features).unwrap();
2913         let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2914         route_params.max_total_routing_fee_msat = None;
2915
2916         let mut route = Route {
2917                 paths: vec![
2918                         Path { hops: vec![RouteHop {
2919                                 pubkey: nodes[1].node.get_our_node_id(),
2920                                 node_features: nodes[1].node.node_features(),
2921                                 short_channel_id: chan_1_scid,
2922                                 channel_features: nodes[1].node.channel_features(),
2923                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2924                                 cltv_expiry_delta: 100,
2925                                 maybe_announced_channel: true,
2926                         }, RouteHop {
2927                                 pubkey: nodes[2].node.get_our_node_id(),
2928                                 node_features: nodes[2].node.node_features(),
2929                                 short_channel_id: chan_2_scid,
2930                                 channel_features: nodes[2].node.channel_features(),
2931                                 fee_msat: 100_000_000,
2932                                 cltv_expiry_delta: 100,
2933                                 maybe_announced_channel: true,
2934                         }], blinded_tail: None },
2935                         Path { hops: vec![RouteHop {
2936                                 pubkey: nodes[1].node.get_our_node_id(),
2937                                 node_features: nodes[1].node.node_features(),
2938                                 short_channel_id: chan_1_scid,
2939                                 channel_features: nodes[1].node.channel_features(),
2940                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2941                                 cltv_expiry_delta: 100,
2942                                 maybe_announced_channel: true,
2943                         }, RouteHop {
2944                                 pubkey: nodes[2].node.get_our_node_id(),
2945                                 node_features: nodes[2].node.node_features(),
2946                                 short_channel_id: chan_2_scid,
2947                                 channel_features: nodes[2].node.channel_features(),
2948                                 fee_msat: 100_000_000,
2949                                 cltv_expiry_delta: 100,
2950                                 maybe_announced_channel: true,
2951                         }], blinded_tail: None }
2952                 ],
2953                 route_params: Some(route_params.clone()),
2954         };
2955         route.route_params.as_mut().unwrap().max_total_routing_fee_msat = None;
2956         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2957         let mut second_payment_params = route_params.payment_params.clone();
2958         second_payment_params.previously_failed_channels = vec![chan_2_scid, chan_2_scid];
2959         // On retry, we'll only return one path
2960         route.paths.remove(1);
2961         route.paths[0].hops[1].fee_msat = amt_msat;
2962         let mut retry_params = RouteParameters::from_payment_params_and_value(second_payment_params, amt_msat);
2963         retry_params.max_total_routing_fee_msat = None;
2964         route.route_params = Some(retry_params.clone());
2965         nodes[0].router.expect_find_route(retry_params, Ok(route.clone()));
2966
2967         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2968                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2969         let htlc_updates = SendEvent::from_node(&nodes[0]);
2970         check_added_monitors!(nodes[0], 1);
2971         assert_eq!(htlc_updates.msgs.len(), 1);
2972
2973         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2974         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2975         check_added_monitors!(nodes[1], 1);
2976         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2977
2978         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2979         check_added_monitors!(nodes[0], 1);
2980         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2981
2982         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2983         check_added_monitors!(nodes[0], 1);
2984         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2985
2986         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2987         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2988         check_added_monitors!(nodes[1], 1);
2989         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2990
2991         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2992         check_added_monitors!(nodes[1], 1);
2993         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2994
2995         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2996         check_added_monitors!(nodes[0], 1);
2997
2998         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2999         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
3000         check_added_monitors!(nodes[0], 1);
3001         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
3002
3003         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
3004         check_added_monitors!(nodes[1], 1);
3005         let bs_second_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3006
3007         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
3008         check_added_monitors!(nodes[1], 1);
3009         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
3010
3011         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.update_fail_htlcs[0]);
3012         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.commitment_signed);
3013         check_added_monitors!(nodes[0], 1);
3014
3015         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
3016         check_added_monitors!(nodes[0], 1);
3017         let (as_third_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
3018
3019         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
3020         check_added_monitors!(nodes[1], 1);
3021         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
3022         check_added_monitors!(nodes[1], 1);
3023         let bs_fourth_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
3024
3025         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_fourth_raa);
3026         check_added_monitors!(nodes[0], 1);
3027
3028         // At this point A has sent two HTLCs which both failed due to lack of fee. It now has two
3029         // pending `PaymentPathFailed` events, one with `all_paths_failed` unset, and the second
3030         // with it set.
3031         //
3032         // Previously, we retried payments in an event consumer, which would retry each
3033         // `PaymentPathFailed` individually. In that setup, we had retried the payment in response to
3034         // the first `PaymentPathFailed`, then seen the second `PaymentPathFailed` with
3035         // `all_paths_failed` set and assumed the payment was completely failed. We ultimately fixed it
3036         // by adding the `PaymentFailed` event.
3037         //
3038         // Because we now retry payments as a batch, we simply return a single-path route in the
3039         // second, batched, request, have that fail, ensure the payment was abandoned.
3040         let mut events = nodes[0].node.get_and_clear_pending_events();
3041         assert_eq!(events.len(), 3);
3042         match events[0] {
3043                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
3044                         assert_eq!(payment_hash, ev_payment_hash);
3045                         assert_eq!(payment_failed_permanently, false);
3046                 },
3047                 _ => panic!("Unexpected event"),
3048         }
3049         match events[1] {
3050                 Event::PendingHTLCsForwardable { .. } => {},
3051                 _ => panic!("Unexpected event"),
3052         }
3053         match events[2] {
3054                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
3055                         assert_eq!(payment_hash, ev_payment_hash);
3056                         assert_eq!(payment_failed_permanently, false);
3057                 },
3058                 _ => panic!("Unexpected event"),
3059         }
3060
3061         nodes[0].node.process_pending_htlc_forwards();
3062         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
3063         check_added_monitors!(nodes[0], 1);
3064
3065         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
3066         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
3067         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3068         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
3069         commitment_signed_dance!(nodes[0], nodes[1], &bs_fail_update.commitment_signed, false, true);
3070
3071         let mut events = nodes[0].node.get_and_clear_pending_events();
3072         assert_eq!(events.len(), 2);
3073         match events[0] {
3074                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
3075                         assert_eq!(payment_hash, ev_payment_hash);
3076                         assert_eq!(payment_failed_permanently, false);
3077                 },
3078                 _ => panic!("Unexpected event"),
3079         }
3080         match events[1] {
3081                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
3082                         assert_eq!(payment_hash, *ev_payment_hash);
3083                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
3084                         assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
3085                 },
3086                 _ => panic!("Unexpected event"),
3087         }
3088 }
3089
3090 #[test]
3091 fn test_simple_partial_retry() {
3092         // In the first version of the in-`ChannelManager` payment retries, retries were sent for the
3093         // full amount of the payment, rather than only the missing amount. Here we simply test for
3094         // this by sending a payment with two parts, failing one, and retrying the second. Note that
3095         // `TestRouter` will check that the `RouteParameters` (which contain the amount) matches the
3096         // request.
3097         let chanmon_cfgs = create_chanmon_cfgs(3);
3098         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
3099         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
3100         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
3101
3102         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
3103         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
3104
3105         let amt_msat = 200_000_000;
3106         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
3107         #[cfg(feature = "std")]
3108         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
3109         #[cfg(not(feature = "std"))]
3110         let payment_expiry_secs = 60 * 60;
3111         let mut invoice_features = Bolt11InvoiceFeatures::empty();
3112         invoice_features.set_variable_length_onion_required();
3113         invoice_features.set_payment_secret_required();
3114         invoice_features.set_basic_mpp_optional();
3115         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
3116                 .with_expiry_time(payment_expiry_secs as u64)
3117                 .with_bolt11_features(invoice_features).unwrap();
3118         let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
3119         route_params.max_total_routing_fee_msat = None;
3120
3121         let mut route = Route {
3122                 paths: vec![
3123                         Path { hops: vec![RouteHop {
3124                                 pubkey: nodes[1].node.get_our_node_id(),
3125                                 node_features: nodes[1].node.node_features(),
3126                                 short_channel_id: chan_1_scid,
3127                                 channel_features: nodes[1].node.channel_features(),
3128                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
3129                                 cltv_expiry_delta: 100,
3130                                 maybe_announced_channel: true,
3131                         }, RouteHop {
3132                                 pubkey: nodes[2].node.get_our_node_id(),
3133                                 node_features: nodes[2].node.node_features(),
3134                                 short_channel_id: chan_2_scid,
3135                                 channel_features: nodes[2].node.channel_features(),
3136                                 fee_msat: 100_000_000,
3137                                 cltv_expiry_delta: 100,
3138                                 maybe_announced_channel: true,
3139                         }], blinded_tail: None },
3140                         Path { hops: vec![RouteHop {
3141                                 pubkey: nodes[1].node.get_our_node_id(),
3142                                 node_features: nodes[1].node.node_features(),
3143                                 short_channel_id: chan_1_scid,
3144                                 channel_features: nodes[1].node.channel_features(),
3145                                 fee_msat: 100_000,
3146                                 cltv_expiry_delta: 100,
3147                                 maybe_announced_channel: true,
3148                         }, RouteHop {
3149                                 pubkey: nodes[2].node.get_our_node_id(),
3150                                 node_features: nodes[2].node.node_features(),
3151                                 short_channel_id: chan_2_scid,
3152                                 channel_features: nodes[2].node.channel_features(),
3153                                 fee_msat: 100_000_000,
3154                                 cltv_expiry_delta: 100,
3155                                 maybe_announced_channel: true,
3156                         }], blinded_tail: None }
3157                 ],
3158                 route_params: Some(route_params.clone()),
3159         };
3160
3161         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3162
3163         let mut second_payment_params = route_params.payment_params.clone();
3164         second_payment_params.previously_failed_channels = vec![chan_2_scid];
3165         // On retry, we'll only be asked for one path (or 100k sats)
3166         route.paths.remove(0);
3167         let mut retry_params = RouteParameters::from_payment_params_and_value(second_payment_params, amt_msat / 2);
3168         retry_params.max_total_routing_fee_msat = None;
3169         route.route_params = Some(retry_params.clone());
3170         nodes[0].router.expect_find_route(retry_params, Ok(route.clone()));
3171
3172         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3173                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
3174         let htlc_updates = SendEvent::from_node(&nodes[0]);
3175         check_added_monitors!(nodes[0], 1);
3176         assert_eq!(htlc_updates.msgs.len(), 1);
3177
3178         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
3179         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
3180         check_added_monitors!(nodes[1], 1);
3181         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3182
3183         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
3184         check_added_monitors!(nodes[0], 1);
3185         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
3186
3187         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
3188         check_added_monitors!(nodes[0], 1);
3189         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
3190
3191         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
3192         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
3193         check_added_monitors!(nodes[1], 1);
3194         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
3195
3196         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
3197         check_added_monitors!(nodes[1], 1);
3198         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3199
3200         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
3201         check_added_monitors!(nodes[0], 1);
3202
3203         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
3204         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
3205         check_added_monitors!(nodes[0], 1);
3206         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
3207
3208         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
3209         check_added_monitors!(nodes[1], 1);
3210
3211         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
3212         check_added_monitors!(nodes[1], 1);
3213
3214         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
3215
3216         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
3217         check_added_monitors!(nodes[0], 1);
3218
3219         let mut events = nodes[0].node.get_and_clear_pending_events();
3220         assert_eq!(events.len(), 2);
3221         match events[0] {
3222                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
3223                         assert_eq!(payment_hash, ev_payment_hash);
3224                         assert_eq!(payment_failed_permanently, false);
3225                 },
3226                 _ => panic!("Unexpected event"),
3227         }
3228         match events[1] {
3229                 Event::PendingHTLCsForwardable { .. } => {},
3230                 _ => panic!("Unexpected event"),
3231         }
3232
3233         nodes[0].node.process_pending_htlc_forwards();
3234         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
3235         check_added_monitors!(nodes[0], 1);
3236
3237         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
3238         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
3239
3240         expect_pending_htlcs_forwardable!(nodes[1]);
3241         check_added_monitors!(nodes[1], 1);
3242
3243         let bs_forward_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
3244         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[0]);
3245         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[1]);
3246         commitment_signed_dance!(nodes[2], nodes[1], &bs_forward_update.commitment_signed, false);
3247
3248         expect_pending_htlcs_forwardable!(nodes[2]);
3249         expect_payment_claimable!(nodes[2], payment_hash, payment_secret, amt_msat);
3250 }
3251
3252 #[test]
3253 #[cfg(feature = "std")]
3254 fn test_threaded_payment_retries() {
3255         // In the first version of the in-`ChannelManager` payment retries, retries weren't limited to
3256         // a single thread and would happily let multiple threads run retries at the same time. Because
3257         // retries are done by first calculating the amount we need to retry, then dropping the
3258         // relevant lock, then actually sending, we would happily let multiple threads retry the same
3259         // amount at the same time, overpaying our original HTLC!
3260         let chanmon_cfgs = create_chanmon_cfgs(4);
3261         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3262         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3263         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3264
3265         // There is one mitigating guardrail when retrying payments - we can never over-pay by more
3266         // than 10% of the original value. Thus, we want all our retries to be below that. In order to
3267         // keep things simple, we route one HTLC for 0.1% of the payment over channel 1 and the rest
3268         // out over channel 3+4. This will let us ignore 99% of the payment value and deal with only
3269         // our channel.
3270         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
3271         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 10_000_000, 0);
3272         let chan_3_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 10_000_000, 0).0.contents.short_channel_id;
3273         let chan_4_scid = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 10_000_000, 0).0.contents.short_channel_id;
3274
3275         let amt_msat = 100_000_000;
3276         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
3277         #[cfg(feature = "std")]
3278         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
3279         #[cfg(not(feature = "std"))]
3280         let payment_expiry_secs = 60 * 60;
3281         let mut invoice_features = Bolt11InvoiceFeatures::empty();
3282         invoice_features.set_variable_length_onion_required();
3283         invoice_features.set_payment_secret_required();
3284         invoice_features.set_basic_mpp_optional();
3285         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
3286                 .with_expiry_time(payment_expiry_secs as u64)
3287                 .with_bolt11_features(invoice_features).unwrap();
3288         let mut route_params = RouteParameters {
3289                 payment_params, final_value_msat: amt_msat, max_total_routing_fee_msat: Some(500_000),
3290         };
3291
3292         let mut route = Route {
3293                 paths: vec![
3294                         Path { hops: vec![RouteHop {
3295                                 pubkey: nodes[1].node.get_our_node_id(),
3296                                 node_features: nodes[1].node.node_features(),
3297                                 short_channel_id: chan_1_scid,
3298                                 channel_features: nodes[1].node.channel_features(),
3299                                 fee_msat: 0,
3300                                 cltv_expiry_delta: 100,
3301                                 maybe_announced_channel: true,
3302                         }, RouteHop {
3303                                 pubkey: nodes[3].node.get_our_node_id(),
3304                                 node_features: nodes[2].node.node_features(),
3305                                 short_channel_id: 42, // Set a random SCID which nodes[1] will fail as unknown
3306                                 channel_features: nodes[2].node.channel_features(),
3307                                 fee_msat: amt_msat / 1000,
3308                                 cltv_expiry_delta: 100,
3309                                 maybe_announced_channel: true,
3310                         }], blinded_tail: None },
3311                         Path { hops: vec![RouteHop {
3312                                 pubkey: nodes[2].node.get_our_node_id(),
3313                                 node_features: nodes[2].node.node_features(),
3314                                 short_channel_id: chan_3_scid,
3315                                 channel_features: nodes[2].node.channel_features(),
3316                                 fee_msat: 100_000,
3317                                 cltv_expiry_delta: 100,
3318                                 maybe_announced_channel: true,
3319                         }, RouteHop {
3320                                 pubkey: nodes[3].node.get_our_node_id(),
3321                                 node_features: nodes[3].node.node_features(),
3322                                 short_channel_id: chan_4_scid,
3323                                 channel_features: nodes[3].node.channel_features(),
3324                                 fee_msat: amt_msat - amt_msat / 1000,
3325                                 cltv_expiry_delta: 100,
3326                                 maybe_announced_channel: true,
3327                         }], blinded_tail: None }
3328                 ],
3329                 route_params: Some(route_params.clone()),
3330         };
3331         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3332
3333         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3334                 PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(0xdeadbeef)).unwrap();
3335         check_added_monitors!(nodes[0], 2);
3336         let mut send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
3337         assert_eq!(send_msg_events.len(), 2);
3338         send_msg_events.retain(|msg|
3339                 if let MessageSendEvent::UpdateHTLCs { node_id, .. } = msg {
3340                         // Drop the commitment update for nodes[2], we can just let that one sit pending
3341                         // forever.
3342                         *node_id == nodes[1].node.get_our_node_id()
3343                 } else { panic!(); }
3344         );
3345
3346         // from here on out, the retry `RouteParameters` amount will be amt/1000
3347         route_params.final_value_msat /= 1000;
3348         route.route_params = Some(route_params.clone());
3349         route.paths.pop();
3350
3351         let end_time = Instant::now() + Duration::from_secs(1);
3352         macro_rules! thread_body { () => { {
3353                 // We really want std::thread::scope, but its not stable until 1.63. Until then, we get unsafe.
3354                 let node_ref = NodePtr::from_node(&nodes[0]);
3355                 move || {
3356                         let _ = &node_ref;
3357                         let node_a = unsafe { &*node_ref.0 };
3358                         while Instant::now() < end_time {
3359                                 node_a.node.get_and_clear_pending_events(); // wipe the PendingHTLCsForwardable
3360                                 // Ignore if we have any pending events, just always pretend we just got a
3361                                 // PendingHTLCsForwardable
3362                                 node_a.node.process_pending_htlc_forwards();
3363                         }
3364                 }
3365         } } }
3366         let mut threads = Vec::new();
3367         for _ in 0..16 { threads.push(std::thread::spawn(thread_body!())); }
3368
3369         // Back in the main thread, poll pending messages and make sure that we never have more than
3370         // one HTLC pending at a time. Note that the commitment_signed_dance will fail horribly if
3371         // there are HTLC messages shoved in while its running. This allows us to test that we never
3372         // generate an additional update_add_htlc until we've fully failed the first.
3373         let mut previously_failed_channels = Vec::new();
3374         loop {
3375                 assert_eq!(send_msg_events.len(), 1);
3376                 let send_event = SendEvent::from_event(send_msg_events.pop().unwrap());
3377                 assert_eq!(send_event.msgs.len(), 1);
3378
3379                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
3380                 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
3381
3382                 // Note that we only push one route into `expect_find_route` at a time, because that's all
3383                 // the retries (should) need. If the bug is reintroduced "real" routes may be selected, but
3384                 // we should still ultimately fail for the same reason - because we're trying to send too
3385                 // many HTLCs at once.
3386                 let mut new_route_params = route_params.clone();
3387                 previously_failed_channels.push(route.paths[0].hops[1].short_channel_id);
3388                 new_route_params.payment_params.previously_failed_channels = previously_failed_channels.clone();
3389                 new_route_params.max_total_routing_fee_msat.as_mut().map(|m| *m -= 100_000);
3390                 route.paths[0].hops[1].short_channel_id += 1;
3391                 route.route_params = Some(new_route_params.clone());
3392                 nodes[0].router.expect_find_route(new_route_params, Ok(route.clone()));
3393
3394                 let bs_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3395                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_updates.update_fail_htlcs[0]);
3396                 // The "normal" commitment_signed_dance delivers the final RAA and then calls
3397                 // `check_added_monitors` to ensure only the one RAA-generated monitor update was created.
3398                 // This races with our other threads which may generate an add-HTLCs commitment update via
3399                 // `process_pending_htlc_forwards`. Instead, we defer the monitor update check until after
3400                 // *we've* called `process_pending_htlc_forwards` when its guaranteed to have two updates.
3401                 let last_raa = commitment_signed_dance!(nodes[0], nodes[1], bs_fail_updates.commitment_signed, false, true, false, true);
3402                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &last_raa);
3403
3404                 let cur_time = Instant::now();
3405                 if cur_time > end_time {
3406                         for thread in threads.drain(..) { thread.join().unwrap(); }
3407                 }
3408
3409                 // Make sure we have some events to handle when we go around...
3410                 nodes[0].node.get_and_clear_pending_events(); // wipe the PendingHTLCsForwardable
3411                 nodes[0].node.process_pending_htlc_forwards();
3412                 send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
3413                 check_added_monitors!(nodes[0], 2);
3414
3415                 if cur_time > end_time {
3416                         break;
3417                 }
3418         }
3419 }
3420
3421 fn do_no_missing_sent_on_reload(persist_manager_with_payment: bool, at_midpoint: bool) {
3422         // Test that if we reload in the middle of an HTLC claim commitment signed dance we'll still
3423         // receive the PaymentSent event even if the ChannelManager had no idea about the payment when
3424         // it was last persisted.
3425         let chanmon_cfgs = create_chanmon_cfgs(2);
3426         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
3427         let (persister_a, persister_b, persister_c);
3428         let (chain_monitor_a, chain_monitor_b, chain_monitor_c);
3429         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
3430         let (nodes_0_deserialized, nodes_0_deserialized_b, nodes_0_deserialized_c);
3431         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
3432
3433         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
3434
3435         let mut nodes_0_serialized = Vec::new();
3436         if !persist_manager_with_payment {
3437                 nodes_0_serialized = nodes[0].node.encode();
3438         }
3439
3440         let (our_payment_preimage, our_payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 1_000_000);
3441
3442         if persist_manager_with_payment {
3443                 nodes_0_serialized = nodes[0].node.encode();
3444         }
3445
3446         nodes[1].node.claim_funds(our_payment_preimage);
3447         check_added_monitors!(nodes[1], 1);
3448         expect_payment_claimed!(nodes[1], our_payment_hash, 1_000_000);
3449
3450         if at_midpoint {
3451                 let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3452                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3453                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
3454                 check_added_monitors!(nodes[0], 1);
3455         } else {
3456                 let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3457                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
3458                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fulfill_updates.commitment_signed, false);
3459                 // Ignore the PaymentSent event which is now pending on nodes[0] - if we were to handle it we'd
3460                 // be expected to ignore the eventual conflicting PaymentFailed, but by not looking at it we
3461                 // expect to get the PaymentSent again later.
3462                 check_added_monitors(&nodes[0], 0);
3463         }
3464
3465         // The ChannelMonitor should always be the latest version, as we're required to persist it
3466         // during the commitment signed handling.
3467         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3468         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister_a, chain_monitor_a, nodes_0_deserialized);
3469
3470         let events = nodes[0].node.get_and_clear_pending_events();
3471         assert_eq!(events.len(), 2);
3472         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[0] {} else { panic!(); }
3473         if let Event::PaymentSent { payment_preimage, .. } = events[1] { assert_eq!(payment_preimage, our_payment_preimage); } else { panic!(); }
3474         // Note that we don't get a PaymentPathSuccessful here as we leave the HTLC pending to avoid
3475         // the double-claim that would otherwise appear at the end of this test.
3476         nodes[0].node.timer_tick_occurred();
3477         let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3478         assert_eq!(as_broadcasted_txn.len(), 1);
3479
3480         // Ensure that, even after some time, if we restart we still include *something* in the current
3481         // `ChannelManager` which prevents a `PaymentFailed` when we restart even if pending resolved
3482         // payments have since been timed out thanks to `IDEMPOTENCY_TIMEOUT_TICKS`.
3483         // A naive implementation of the fix here would wipe the pending payments set, causing a
3484         // failure event when we restart.
3485         for _ in 0..(IDEMPOTENCY_TIMEOUT_TICKS * 2) { nodes[0].node.timer_tick_occurred(); }
3486
3487         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3488         reload_node!(nodes[0], test_default_channel_config(), &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister_b, chain_monitor_b, nodes_0_deserialized_b);
3489         let events = nodes[0].node.get_and_clear_pending_events();
3490         assert!(events.is_empty());
3491
3492         // Ensure that we don't generate any further events even after the channel-closing commitment
3493         // transaction is confirmed on-chain.
3494         confirm_transaction(&nodes[0], &as_broadcasted_txn[0]);
3495         for _ in 0..(IDEMPOTENCY_TIMEOUT_TICKS * 2) { nodes[0].node.timer_tick_occurred(); }
3496
3497         let events = nodes[0].node.get_and_clear_pending_events();
3498         assert!(events.is_empty());
3499
3500         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3501         reload_node!(nodes[0], test_default_channel_config(), &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister_c, chain_monitor_c, nodes_0_deserialized_c);
3502         let events = nodes[0].node.get_and_clear_pending_events();
3503         assert!(events.is_empty());
3504         check_added_monitors(&nodes[0], 1);
3505 }
3506
3507 #[test]
3508 fn no_missing_sent_on_midpoint_reload() {
3509         do_no_missing_sent_on_reload(false, true);
3510         do_no_missing_sent_on_reload(true, true);
3511 }
3512
3513 #[test]
3514 fn no_missing_sent_on_reload() {
3515         do_no_missing_sent_on_reload(false, false);
3516         do_no_missing_sent_on_reload(true, false);
3517 }
3518
3519 fn do_claim_from_closed_chan(fail_payment: bool) {
3520         // Previously, LDK would refuse to claim a payment if a channel on which the payment was
3521         // received had been closed between when the HTLC was received and when we went to claim it.
3522         // This makes sense in the payment case - why pay an on-chain fee to claim the HTLC when
3523         // presumably the sender may retry later. Long ago it also reduced total code in the claim
3524         // pipeline.
3525         //
3526         // However, this doesn't make sense if you're trying to do an atomic swap or some other
3527         // protocol that requires atomicity with some other action - if your money got claimed
3528         // elsewhere you need to be able to claim the HTLC in lightning no matter what. Further, this
3529         // is an over-optimization - there should be a very, very low likelihood that a channel closes
3530         // between when we receive the last HTLC for a payment and the user goes to claim the payment.
3531         // Since we now have code to handle this anyway we should allow it.
3532
3533         // Build 4 nodes and send an MPP payment across two paths. By building a route manually set the
3534         // CLTVs on the paths to different value resulting in a different claim deadline.
3535         let chanmon_cfgs = create_chanmon_cfgs(4);
3536         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3537         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3538         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3539
3540         create_announced_chan_between_nodes(&nodes, 0, 1);
3541         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
3542         let chan_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
3543         create_announced_chan_between_nodes(&nodes, 2, 3);
3544
3545         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3]);
3546         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3547                 .with_bolt11_features(nodes[1].node.bolt11_invoice_features()).unwrap();
3548         let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, 10_000_000);
3549         let mut route = nodes[0].router.find_route(&nodes[0].node.get_our_node_id(), &route_params,
3550                 None, nodes[0].node.compute_inflight_htlcs()).unwrap();
3551         // Make sure the route is ordered as the B->D path before C->D
3552         route.paths.sort_by(|a, _| if a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
3553                 std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater });
3554
3555         // Note that we add an extra 1 in the send pipeline to compensate for any blocks found while
3556         // the HTLC is being relayed.
3557         route.paths[0].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 8;
3558         route.paths[1].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 12;
3559         let final_cltv = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 8 + 1;
3560
3561         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3562         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3563                 PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(1)).unwrap();
3564         check_added_monitors(&nodes[0], 2);
3565         let mut send_msgs = nodes[0].node.get_and_clear_pending_msg_events();
3566         send_msgs.sort_by(|a, _| {
3567                 let a_node_id =
3568                         if let MessageSendEvent::UpdateHTLCs { node_id, .. } = a { node_id } else { panic!() };
3569                 let node_b_id = nodes[1].node.get_our_node_id();
3570                 if *a_node_id == node_b_id { std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater }
3571         });
3572
3573         assert_eq!(send_msgs.len(), 2);
3574         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 10_000_000,
3575                 payment_hash, Some(payment_secret), send_msgs.remove(0), false, None);
3576         let receive_event = pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 10_000_000,
3577                 payment_hash, Some(payment_secret), send_msgs.remove(0), true, None);
3578
3579         match receive_event.unwrap() {
3580                 Event::PaymentClaimable { claim_deadline, .. } => {
3581                         assert_eq!(claim_deadline.unwrap(), final_cltv - HTLC_FAIL_BACK_BUFFER);
3582                 },
3583                 _ => panic!(),
3584         }
3585
3586         // Ensure that the claim_deadline is correct, with the payment failing at exactly the given
3587         // height.
3588         connect_blocks(&nodes[3], final_cltv - HTLC_FAIL_BACK_BUFFER - nodes[3].best_block_info().1
3589                 - if fail_payment { 0 } else { 2 });
3590         if fail_payment {
3591                 // We fail the HTLC on the A->B->D path first as it expires 4 blocks earlier. We go ahead
3592                 // and expire both immediately, though, by connecting another 4 blocks.
3593                 let reason = HTLCDestination::FailedPayment { payment_hash };
3594                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason.clone()]);
3595                 connect_blocks(&nodes[3], 4);
3596                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason]);
3597                 pass_failed_payment_back(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
3598         } else {
3599                 nodes[1].node.force_close_broadcasting_latest_txn(&chan_bd, &nodes[3].node.get_our_node_id()).unwrap();
3600                 check_closed_event!(&nodes[1], 1, ClosureReason::HolderForceClosed, false,
3601                         [nodes[3].node.get_our_node_id()], 1000000);
3602                 check_closed_broadcast(&nodes[1], 1, true);
3603                 let bs_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3604                 assert_eq!(bs_tx.len(), 1);
3605
3606                 mine_transaction(&nodes[3], &bs_tx[0]);
3607                 check_added_monitors(&nodes[3], 1);
3608                 check_closed_broadcast(&nodes[3], 1, true);
3609                 check_closed_event!(&nodes[3], 1, ClosureReason::CommitmentTxConfirmed, false,
3610                         [nodes[1].node.get_our_node_id()], 1000000);
3611
3612                 nodes[3].node.claim_funds(payment_preimage);
3613                 check_added_monitors(&nodes[3], 2);
3614                 expect_payment_claimed!(nodes[3], payment_hash, 10_000_000);
3615
3616                 let ds_tx = nodes[3].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3617                 assert_eq!(ds_tx.len(), 1);
3618                 check_spends!(&ds_tx[0], &bs_tx[0]);
3619
3620                 mine_transactions(&nodes[1], &[&bs_tx[0], &ds_tx[0]]);
3621                 check_added_monitors(&nodes[1], 1);
3622                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[3], Some(1000), false, true);
3623
3624                 let bs_claims = nodes[1].node.get_and_clear_pending_msg_events();
3625                 check_added_monitors(&nodes[1], 1);
3626                 assert_eq!(bs_claims.len(), 1);
3627                 if let MessageSendEvent::UpdateHTLCs { updates, .. } = &bs_claims[0] {
3628                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3629                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
3630                 } else { panic!(); }
3631
3632                 expect_payment_sent!(nodes[0], payment_preimage);
3633
3634                 let ds_claim_msgs = nodes[3].node.get_and_clear_pending_msg_events();
3635                 assert_eq!(ds_claim_msgs.len(), 1);
3636                 let cs_claim_msgs = if let MessageSendEvent::UpdateHTLCs { updates, .. } = &ds_claim_msgs[0] {
3637                         nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3638                         let cs_claim_msgs = nodes[2].node.get_and_clear_pending_msg_events();
3639                         check_added_monitors(&nodes[2], 1);
3640                         commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
3641                         expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
3642                         cs_claim_msgs
3643                 } else { panic!(); };
3644
3645                 assert_eq!(cs_claim_msgs.len(), 1);
3646                 if let MessageSendEvent::UpdateHTLCs { updates, .. } = &cs_claim_msgs[0] {
3647                         nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3648                         commitment_signed_dance!(nodes[0], nodes[2], updates.commitment_signed, false, true);
3649                 } else { panic!(); }
3650
3651                 expect_payment_path_successful!(nodes[0]);
3652         }
3653 }
3654
3655 #[test]
3656 fn claim_from_closed_chan() {
3657         do_claim_from_closed_chan(true);
3658         do_claim_from_closed_chan(false);
3659 }
3660
3661 #[test]
3662 fn test_custom_tlvs_basic() {
3663         do_test_custom_tlvs(false, false, false);
3664         do_test_custom_tlvs(true, false, false);
3665 }
3666
3667 #[test]
3668 fn test_custom_tlvs_explicit_claim() {
3669         // Test that when receiving even custom TLVs the user must explicitly accept in case they
3670         // are unknown.
3671         do_test_custom_tlvs(false, true, false);
3672         do_test_custom_tlvs(false, true, true);
3673 }
3674
3675 fn do_test_custom_tlvs(spontaneous: bool, even_tlvs: bool, known_tlvs: bool) {
3676         let chanmon_cfgs = create_chanmon_cfgs(2);
3677         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
3678         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None; 2]);
3679         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
3680
3681         create_announced_chan_between_nodes(&nodes, 0, 1);
3682
3683         let amt_msat = 100_000;
3684         let (mut route, our_payment_hash, our_payment_preimage, our_payment_secret) = get_route_and_payment_hash!(&nodes[0], &nodes[1], amt_msat);
3685         let payment_id = PaymentId(our_payment_hash.0);
3686         let custom_tlvs = vec![
3687                 (if even_tlvs { 5482373482 } else { 5482373483 }, vec![1, 2, 3, 4]),
3688                 (5482373487, vec![0x42u8; 16]),
3689         ];
3690         let onion_fields = RecipientOnionFields {
3691                 payment_secret: if spontaneous { None } else { Some(our_payment_secret) },
3692                 payment_metadata: None,
3693                 custom_tlvs: custom_tlvs.clone()
3694         };
3695         if spontaneous {
3696                 nodes[0].node.send_spontaneous_payment(&route, Some(our_payment_preimage), onion_fields, payment_id).unwrap();
3697         } else {
3698                 nodes[0].node.send_payment_with_route(&route, our_payment_hash, onion_fields, payment_id).unwrap();
3699         }
3700         check_added_monitors(&nodes[0], 1);
3701
3702         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
3703         let ev = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
3704         let mut payment_event = SendEvent::from_event(ev);
3705
3706         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
3707         check_added_monitors!(&nodes[1], 0);
3708         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
3709         expect_pending_htlcs_forwardable!(nodes[1]);
3710
3711         let events = nodes[1].node.get_and_clear_pending_events();
3712         assert_eq!(events.len(), 1);
3713         match events[0] {
3714                 Event::PaymentClaimable { ref onion_fields, .. } => {
3715                         assert_eq!(onion_fields.clone().unwrap().custom_tlvs().clone(), custom_tlvs);
3716                 },
3717                 _ => panic!("Unexpected event"),
3718         }
3719
3720         match (known_tlvs, even_tlvs) {
3721                 (true, _) => {
3722                         nodes[1].node.claim_funds_with_known_custom_tlvs(our_payment_preimage);
3723                         let expected_total_fee_msat = pass_claimed_payment_along_route(&nodes[0], &[&[&nodes[1]]], &[0; 1], false, our_payment_preimage);
3724                         expect_payment_sent!(&nodes[0], our_payment_preimage, Some(expected_total_fee_msat));
3725                 },
3726                 (false, false) => {
3727                         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
3728                 },
3729                 (false, true) => {
3730                         nodes[1].node.claim_funds(our_payment_preimage);
3731                         let expected_destinations = vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }];
3732                         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], expected_destinations);
3733                         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, our_payment_hash, PaymentFailureReason::RecipientRejected);
3734                 }
3735         }
3736 }
3737
3738 #[test]
3739 fn test_retry_custom_tlvs() {
3740         // Test that custom TLVs are successfully sent on retries
3741         let chanmon_cfgs = create_chanmon_cfgs(3);
3742         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
3743         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
3744         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
3745
3746         create_announced_chan_between_nodes(&nodes, 0, 1);
3747         let (chan_2_update, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 2, 1);
3748
3749         // Rebalance
3750         send_payment(&nodes[2], &vec!(&nodes[1])[..], 1_500_000);
3751
3752         let amt_msat = 1_000_000;
3753         let (mut route, payment_hash, payment_preimage, payment_secret) =
3754                 get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
3755
3756         // Initiate the payment
3757         let payment_id = PaymentId(payment_hash.0);
3758         let mut route_params = route.route_params.clone().unwrap();
3759
3760         let custom_tlvs = vec![((1 << 16) + 1, vec![0x42u8; 16])];
3761         let onion_fields = RecipientOnionFields::secret_only(payment_secret);
3762         let onion_fields = onion_fields.with_custom_tlvs(custom_tlvs.clone()).unwrap();
3763
3764         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3765         nodes[0].node.send_payment(payment_hash, onion_fields,
3766                 payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
3767         check_added_monitors!(nodes[0], 1); // one monitor per path
3768
3769         // Add the HTLC along the first hop.
3770         let htlc_updates = get_htlc_update_msgs(&nodes[0], &nodes[1].node.get_our_node_id());
3771         let msgs::CommitmentUpdate { update_add_htlcs, commitment_signed, .. } = htlc_updates;
3772         assert_eq!(update_add_htlcs.len(), 1);
3773         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_htlcs[0]);
3774         commitment_signed_dance!(nodes[1], nodes[0], commitment_signed, false);
3775
3776         // Attempt to forward the payment and complete the path's failure.
3777         expect_pending_htlcs_forwardable!(&nodes[1]);
3778         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1],
3779                 vec![HTLCDestination::NextHopChannel {
3780                         node_id: Some(nodes[2].node.get_our_node_id()),
3781                         channel_id: chan_2_id
3782                 }]);
3783         check_added_monitors!(nodes[1], 1);
3784
3785         let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3786         let msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. } = htlc_updates;
3787         assert_eq!(update_fail_htlcs.len(), 1);
3788         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
3789         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
3790
3791         let mut events = nodes[0].node.get_and_clear_pending_events();
3792         match events[1] {
3793                 Event::PendingHTLCsForwardable { .. } => {},
3794                 _ => panic!("Unexpected event")
3795         }
3796         events.remove(1);
3797         expect_payment_failed_conditions_event(events, payment_hash, false,
3798                 PaymentFailedConditions::new().mpp_parts_remain());
3799
3800         // Rebalance the channel so the retry of the payment can succeed.
3801         send_payment(&nodes[2], &vec!(&nodes[1])[..], 1_500_000);
3802
3803         // Retry the payment and make sure it succeeds
3804         route_params.payment_params.previously_failed_channels.push(chan_2_update.contents.short_channel_id);
3805         route.route_params = Some(route_params.clone());
3806         nodes[0].router.expect_find_route(route_params, Ok(route));
3807         nodes[0].node.process_pending_htlc_forwards();
3808         check_added_monitors!(nodes[0], 1);
3809         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
3810         assert_eq!(events.len(), 1);
3811         let payment_claimable = pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000,
3812                 payment_hash, Some(payment_secret), events.pop().unwrap(), true, None).unwrap();
3813         match payment_claimable {
3814                 Event::PaymentClaimable { onion_fields, .. } => {
3815                         assert_eq!(&onion_fields.unwrap().custom_tlvs()[..], &custom_tlvs[..]);
3816                 },
3817                 _ => panic!("Unexpected event"),
3818         };
3819         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
3820 }
3821
3822 #[test]
3823 fn test_custom_tlvs_consistency() {
3824         let even_type_1 = 1 << 16;
3825         let odd_type_1  = (1 << 16)+ 1;
3826         let even_type_2 = (1 << 16) + 2;
3827         let odd_type_2  = (1 << 16) + 3;
3828         let value_1 = || vec![1, 2, 3, 4];
3829         let differing_value_1 = || vec![1, 2, 3, 5];
3830         let value_2 = || vec![42u8; 16];
3831
3832         // Drop missing odd tlvs
3833         do_test_custom_tlvs_consistency(
3834                 vec![(odd_type_1, value_1()), (odd_type_2, value_2())],
3835                 vec![(odd_type_1, value_1())],
3836                 Some(vec![(odd_type_1, value_1())]),
3837         );
3838         // Drop non-matching odd tlvs
3839         do_test_custom_tlvs_consistency(
3840                 vec![(odd_type_1, value_1()), (odd_type_2, value_2())],
3841                 vec![(odd_type_1, differing_value_1()), (odd_type_2, value_2())],
3842                 Some(vec![(odd_type_2, value_2())]),
3843         );
3844         // Fail missing even tlvs
3845         do_test_custom_tlvs_consistency(
3846                 vec![(odd_type_1, value_1()), (even_type_2, value_2())],
3847                 vec![(odd_type_1, value_1())],
3848                 None,
3849         );
3850         // Fail non-matching even tlvs
3851         do_test_custom_tlvs_consistency(
3852                 vec![(even_type_1, value_1()), (odd_type_2, value_2())],
3853                 vec![(even_type_1, differing_value_1()), (odd_type_2, value_2())],
3854                 None,
3855         );
3856 }
3857
3858 fn do_test_custom_tlvs_consistency(first_tlvs: Vec<(u64, Vec<u8>)>, second_tlvs: Vec<(u64, Vec<u8>)>,
3859         expected_receive_tlvs: Option<Vec<(u64, Vec<u8>)>>) {
3860
3861         let chanmon_cfgs = create_chanmon_cfgs(4);
3862         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3863         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3864         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3865
3866         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
3867         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0);
3868         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0);
3869         let chan_2_3 = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0);
3870
3871         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3872                 .with_bolt11_features(nodes[3].node.bolt11_invoice_features()).unwrap();
3873         let mut route = get_route!(nodes[0], payment_params, 15_000_000).unwrap();
3874         assert_eq!(route.paths.len(), 2);
3875         route.paths.sort_by(|path_a, _| {
3876                 // Sort the path so that the path through nodes[1] comes first
3877                 if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
3878                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
3879         });
3880
3881         let (our_payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(&nodes[3]);
3882         let payment_id = PaymentId([42; 32]);
3883         let amt_msat = 15_000_000;
3884
3885         // Send first part
3886         let onion_fields = RecipientOnionFields {
3887                 payment_secret: Some(our_payment_secret),
3888                 payment_metadata: None,
3889                 custom_tlvs: first_tlvs
3890         };
3891         let session_privs = nodes[0].node.test_add_new_pending_payment(our_payment_hash,
3892                         onion_fields.clone(), payment_id, &route).unwrap();
3893         let cur_height = nodes[0].best_block_info().1;
3894         nodes[0].node.test_send_payment_along_path(&route.paths[0], &our_payment_hash,
3895                 onion_fields.clone(), amt_msat, cur_height, payment_id,
3896                 &None, session_privs[0]).unwrap();
3897         check_added_monitors!(nodes[0], 1);
3898
3899         {
3900                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
3901                 assert_eq!(events.len(), 1);
3902                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], amt_msat, our_payment_hash,
3903                         Some(our_payment_secret), events.pop().unwrap(), false, None);
3904         }
3905         assert!(nodes[3].node.get_and_clear_pending_events().is_empty());
3906
3907         // Send second part
3908         let onion_fields = RecipientOnionFields {
3909                 payment_secret: Some(our_payment_secret),
3910                 payment_metadata: None,
3911                 custom_tlvs: second_tlvs
3912         };
3913         nodes[0].node.test_send_payment_along_path(&route.paths[1], &our_payment_hash,
3914                 onion_fields.clone(), amt_msat, cur_height, payment_id, &None, session_privs[1]).unwrap();
3915         check_added_monitors!(nodes[0], 1);
3916
3917         {
3918                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
3919                 assert_eq!(events.len(), 1);
3920                 let payment_event = SendEvent::from_event(events.pop().unwrap());
3921
3922                 nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
3923                 commitment_signed_dance!(nodes[2], nodes[0], payment_event.commitment_msg, false);
3924
3925                 expect_pending_htlcs_forwardable!(nodes[2]);
3926                 check_added_monitors!(nodes[2], 1);
3927
3928                 let mut events = nodes[2].node.get_and_clear_pending_msg_events();
3929                 assert_eq!(events.len(), 1);
3930                 let payment_event = SendEvent::from_event(events.pop().unwrap());
3931
3932                 nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
3933                 check_added_monitors!(nodes[3], 0);
3934                 commitment_signed_dance!(nodes[3], nodes[2], payment_event.commitment_msg, true, true);
3935         }
3936         expect_pending_htlcs_forwardable_ignore!(nodes[3]);
3937         nodes[3].node.process_pending_htlc_forwards();
3938
3939         if let Some(expected_tlvs) = expected_receive_tlvs {
3940                 // Claim and match expected
3941                 let events = nodes[3].node.get_and_clear_pending_events();
3942                 assert_eq!(events.len(), 1);
3943                 match events[0] {
3944                         Event::PaymentClaimable { ref onion_fields, .. } => {
3945                                 assert_eq!(onion_fields.clone().unwrap().custom_tlvs, expected_tlvs);
3946                         },
3947                         _ => panic!("Unexpected event"),
3948                 }
3949
3950                 do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]],
3951                         false, our_payment_preimage);
3952                 expect_payment_sent(&nodes[0], our_payment_preimage, Some(Some(2000)), true, true);
3953         } else {
3954                 // Expect fail back
3955                 let expected_destinations = vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }];
3956                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], expected_destinations);
3957                 check_added_monitors!(nodes[3], 1);
3958
3959                 let fail_updates_1 = get_htlc_update_msgs!(nodes[3], nodes[2].node.get_our_node_id());
3960                 nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &fail_updates_1.update_fail_htlcs[0]);
3961                 commitment_signed_dance!(nodes[2], nodes[3], fail_updates_1.commitment_signed, false);
3962
3963                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![
3964                         HTLCDestination::NextHopChannel {
3965                                 node_id: Some(nodes[3].node.get_our_node_id()),
3966                                 channel_id: chan_2_3.2
3967                         }]);
3968                 check_added_monitors!(nodes[2], 1);
3969
3970                 let fail_updates_2 = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
3971                 nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &fail_updates_2.update_fail_htlcs[0]);
3972                 commitment_signed_dance!(nodes[0], nodes[2], fail_updates_2.commitment_signed, false);
3973
3974                 expect_payment_failed_conditions(&nodes[0], our_payment_hash, true,
3975                         PaymentFailedConditions::new().mpp_parts_remain());
3976         }
3977 }
3978
3979 fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) {
3980         // Check that a payment metadata received on one HTLC that doesn't match the one received on
3981         // another results in the HTLC being rejected.
3982         //
3983         // We first set up a diamond shaped network, allowing us to split a payment into two HTLCs, the
3984         // first of which we'll deliver and the second of which we'll fail and then re-send with
3985         // modified payment metadata, which will in turn result in it being failed by the recipient.
3986         let chanmon_cfgs = create_chanmon_cfgs(4);
3987         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3988         let persister;
3989         let new_chain_monitor;
3990
3991         let mut config = test_default_channel_config();
3992         config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 50;
3993         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, Some(config), Some(config), Some(config)]);
3994         let nodes_0_deserialized;
3995
3996         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3997
3998         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
3999         let chan_id_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
4000         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
4001         let chan_id_cd = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
4002
4003         // Pay more than half of each channel's max, requiring MPP
4004         let amt_msat = 750_000_000;
4005         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3], Some(amt_msat));
4006         let payment_id = PaymentId(payment_hash.0);
4007         let payment_metadata = vec![44, 49, 52, 142];
4008
4009         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
4010                 .with_bolt11_features(nodes[1].node.bolt11_invoice_features()).unwrap();
4011         let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
4012
4013         // Send the MPP payment, delivering the updated commitment state to nodes[1].
4014         nodes[0].node.send_payment(payment_hash, RecipientOnionFields {
4015                         payment_secret: Some(payment_secret), payment_metadata: Some(payment_metadata), custom_tlvs: vec![],
4016                 }, payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
4017         check_added_monitors!(nodes[0], 2);
4018
4019         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
4020         assert_eq!(send_events.len(), 2);
4021         let first_send = SendEvent::from_event(send_events.pop().unwrap());
4022         let second_send = SendEvent::from_event(send_events.pop().unwrap());
4023
4024         let (b_recv_ev, c_recv_ev) = if first_send.node_id == nodes[1].node.get_our_node_id() {
4025                 (&first_send, &second_send)
4026         } else {
4027                 (&second_send, &first_send)
4028         };
4029         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &b_recv_ev.msgs[0]);
4030         commitment_signed_dance!(nodes[1], nodes[0], b_recv_ev.commitment_msg, false, true);
4031
4032         expect_pending_htlcs_forwardable!(nodes[1]);
4033         check_added_monitors(&nodes[1], 1);
4034         let b_forward_ev = SendEvent::from_node(&nodes[1]);
4035         nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &b_forward_ev.msgs[0]);
4036         commitment_signed_dance!(nodes[3], nodes[1], b_forward_ev.commitment_msg, false, true);
4037
4038         expect_pending_htlcs_forwardable!(nodes[3]);
4039
4040         // Before delivering the second MPP HTLC to nodes[2], disconnect nodes[2] and nodes[3], which
4041         // will result in nodes[2] failing the HTLC back.
4042         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
4043         nodes[3].node.peer_disconnected(&nodes[2].node.get_our_node_id());
4044
4045         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &c_recv_ev.msgs[0]);
4046         commitment_signed_dance!(nodes[2], nodes[0], c_recv_ev.commitment_msg, false, true);
4047
4048         let cs_fail = get_htlc_update_msgs(&nodes[2], &nodes[0].node.get_our_node_id());
4049         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail.update_fail_htlcs[0]);
4050         commitment_signed_dance!(nodes[0], nodes[2], cs_fail.commitment_signed, false, true);
4051
4052         let payment_fail_retryable_evs = nodes[0].node.get_and_clear_pending_events();
4053         assert_eq!(payment_fail_retryable_evs.len(), 2);
4054         if let Event::PaymentPathFailed { .. } = payment_fail_retryable_evs[0] {} else { panic!(); }
4055         if let Event::PendingHTLCsForwardable { .. } = payment_fail_retryable_evs[1] {} else { panic!(); }
4056
4057         // Before we allow the HTLC to be retried, optionally change the payment_metadata we have
4058         // stored for our payment.
4059         if do_modify {
4060                 nodes[0].node.test_set_payment_metadata(payment_id, Some(Vec::new()));
4061         }
4062
4063         // Optionally reload nodes[3] to check that the payment_metadata is properly serialized with
4064         // the payment state.
4065         if do_reload {
4066                 let mon_bd = get_monitor!(nodes[3], chan_id_bd).encode();
4067                 let mon_cd = get_monitor!(nodes[3], chan_id_cd).encode();
4068                 reload_node!(nodes[3], config, &nodes[3].node.encode(), &[&mon_bd, &mon_cd],
4069                         persister, new_chain_monitor, nodes_0_deserialized);
4070                 nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
4071                 reconnect_nodes(ReconnectArgs::new(&nodes[1], &nodes[3]));
4072         }
4073         let mut reconnect_args = ReconnectArgs::new(&nodes[2], &nodes[3]);
4074         reconnect_args.send_channel_ready = (true, true);
4075         reconnect_nodes(reconnect_args);
4076
4077         // Create a new channel between C and D as A will refuse to retry on the existing one because
4078         // it just failed.
4079         let chan_id_cd_2 = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
4080
4081         // Now retry the failed HTLC.
4082         nodes[0].node.process_pending_htlc_forwards();
4083         check_added_monitors(&nodes[0], 1);
4084         let as_resend = SendEvent::from_node(&nodes[0]);
4085         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_resend.msgs[0]);
4086         commitment_signed_dance!(nodes[2], nodes[0], as_resend.commitment_msg, false, true);
4087
4088         expect_pending_htlcs_forwardable!(nodes[2]);
4089         check_added_monitors(&nodes[2], 1);
4090         let cs_forward = SendEvent::from_node(&nodes[2]);
4091         nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &cs_forward.msgs[0]);
4092         commitment_signed_dance!(nodes[3], nodes[2], cs_forward.commitment_msg, false, true);
4093
4094         // Finally, check that nodes[3] does the correct thing - either accepting the payment or, if
4095         // the payment metadata was modified, failing only the one modified HTLC and retaining the
4096         // other.
4097         if do_modify {
4098                 expect_pending_htlcs_forwardable_ignore!(nodes[3]);
4099                 nodes[3].node.process_pending_htlc_forwards();
4100                 expect_pending_htlcs_forwardable_conditions(nodes[3].node.get_and_clear_pending_events(),
4101                         &[HTLCDestination::FailedPayment {payment_hash}]);
4102                 nodes[3].node.process_pending_htlc_forwards();
4103
4104                 check_added_monitors(&nodes[3], 1);
4105                 let ds_fail = get_htlc_update_msgs(&nodes[3], &nodes[2].node.get_our_node_id());
4106
4107                 nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &ds_fail.update_fail_htlcs[0]);
4108                 commitment_signed_dance!(nodes[2], nodes[3], ds_fail.commitment_signed, false, true);
4109                 expect_pending_htlcs_forwardable_conditions(nodes[2].node.get_and_clear_pending_events(),
4110                         &[HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_id_cd_2 }]);
4111         } else {
4112                 expect_pending_htlcs_forwardable!(nodes[3]);
4113                 expect_payment_claimable!(nodes[3], payment_hash, payment_secret, amt_msat);
4114                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
4115         }
4116 }
4117
4118 #[test]
4119 fn test_payment_metadata_consistency() {
4120         do_test_payment_metadata_consistency(true, true);
4121         do_test_payment_metadata_consistency(true, false);
4122         do_test_payment_metadata_consistency(false, true);
4123         do_test_payment_metadata_consistency(false, false);
4124 }
4125
4126 #[test]
4127 fn  test_htlc_forward_considers_anchor_outputs_value() {
4128         // Tests that:
4129         //
4130         // 1) Forwarding nodes don't forward HTLCs that would cause their balance to dip below the
4131         //    reserve when considering the value of anchor outputs.
4132         //
4133         // 2) Recipients of `update_add_htlc` properly reject HTLCs that would cause the initiator's
4134         //    balance to dip below the reserve when considering the value of anchor outputs.
4135         let mut config = test_default_channel_config();
4136         config.channel_handshake_config.negotiate_anchors_zero_fee_htlc_tx = true;
4137         config.manually_accept_inbound_channels = true;
4138         config.channel_config.forwarding_fee_base_msat = 0;
4139         config.channel_config.forwarding_fee_proportional_millionths = 0;
4140
4141         // Set up a test network of three nodes that replicates a production failure leading to the
4142         // discovery of this bug.
4143         let chanmon_cfgs = create_chanmon_cfgs(3);
4144         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
4145         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[Some(config), Some(config), Some(config)]);
4146         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
4147
4148         const CHAN_AMT: u64 = 1_000_000;
4149         const PUSH_MSAT: u64 = 900_000_000;
4150         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, CHAN_AMT, 500_000_000);
4151         let (_, _, chan_id_2, _) = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, CHAN_AMT, PUSH_MSAT);
4152
4153         let channel_reserve_msat = get_holder_selected_channel_reserve_satoshis(CHAN_AMT, &config) * 1000;
4154         let commitment_fee_msat = commit_tx_fee_msat(
4155                 *nodes[1].fee_estimator.sat_per_kw.lock().unwrap(), 2, &ChannelTypeFeatures::anchors_zero_htlc_fee_and_dependencies()
4156         );
4157         let anchor_outpus_value_msat = ANCHOR_OUTPUT_VALUE_SATOSHI * 2 * 1000;
4158         let sendable_balance_msat = CHAN_AMT * 1000 - PUSH_MSAT - channel_reserve_msat - commitment_fee_msat - anchor_outpus_value_msat;
4159         let channel_details = nodes[1].node.list_channels().into_iter().find(|channel| channel.channel_id == chan_id_2).unwrap();
4160         assert!(sendable_balance_msat >= channel_details.next_outbound_htlc_minimum_msat);
4161         assert!(sendable_balance_msat <= channel_details.next_outbound_htlc_limit_msat);
4162
4163         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], sendable_balance_msat);
4164         send_payment(&nodes[2], &[&nodes[1], &nodes[0]], sendable_balance_msat);
4165
4166         // Send out an HTLC that would cause the forwarding node to dip below its reserve when
4167         // considering the value of anchor outputs.
4168         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(
4169                 nodes[0], nodes[2], sendable_balance_msat + anchor_outpus_value_msat
4170         );
4171         nodes[0].node.send_payment_with_route(
4172                 &route, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
4173         ).unwrap();
4174         check_added_monitors!(nodes[0], 1);
4175
4176         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
4177         assert_eq!(events.len(), 1);
4178         let mut update_add_htlc = if let MessageSendEvent::UpdateHTLCs { updates, .. } = events.pop().unwrap() {
4179                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &updates.update_add_htlcs[0]);
4180                 check_added_monitors(&nodes[1], 0);
4181                 commitment_signed_dance!(nodes[1], nodes[0], &updates.commitment_signed, false);
4182                 updates.update_add_htlcs[0].clone()
4183         } else {
4184                 panic!("Unexpected event");
4185         };
4186
4187         // The forwarding node should reject forwarding it as expected.
4188         expect_pending_htlcs_forwardable!(nodes[1]);
4189         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1], vec![HTLCDestination::NextHopChannel {
4190                 node_id: Some(nodes[2].node.get_our_node_id()),
4191                 channel_id: chan_id_2
4192         }]);
4193         check_added_monitors(&nodes[1], 1);
4194
4195         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
4196         assert_eq!(events.len(), 1);
4197         if let MessageSendEvent::UpdateHTLCs { updates, .. } = events.pop().unwrap() {
4198                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
4199                 check_added_monitors(&nodes[0], 0);
4200                 commitment_signed_dance!(nodes[0], nodes[1], &updates.commitment_signed, false);
4201         } else {
4202                 panic!("Unexpected event");
4203         }
4204
4205         expect_payment_failed!(nodes[0], payment_hash, false);
4206
4207         // Assume that the forwarding node did forward it, and make sure the recipient rejects it as an
4208         // invalid update and closes the channel.
4209         update_add_htlc.channel_id = chan_id_2;
4210         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &update_add_htlc);
4211         check_closed_event(&nodes[2], 1, ClosureReason::ProcessingError {
4212                 err: "Remote HTLC add would put them under remote reserve value".to_owned()
4213         }, false, &[nodes[1].node.get_our_node_id()], 1_000_000);
4214         check_closed_broadcast(&nodes[2], 1, true);
4215         check_added_monitors(&nodes[2], 1);
4216 }
4217
4218 #[test]
4219 fn peel_payment_onion_custom_tlvs() {
4220         let chanmon_cfgs = create_chanmon_cfgs(2);
4221         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
4222         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
4223         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
4224         create_announced_chan_between_nodes(&nodes, 0, 1);
4225         let secp_ctx = Secp256k1::new();
4226
4227         let amt_msat = 1000;
4228         let payment_params = PaymentParameters::for_keysend(nodes[1].node.get_our_node_id(),
4229                 TEST_FINAL_CLTV, false);
4230         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
4231         let route = functional_test_utils::get_route(&nodes[0], &route_params).unwrap();
4232         let mut recipient_onion = RecipientOnionFields::spontaneous_empty()
4233                 .with_custom_tlvs(vec![(414141, vec![42; 1200])]).unwrap();
4234         let prng_seed = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
4235         let session_priv = SecretKey::from_slice(&prng_seed[..]).expect("RNG is busted");
4236         let keysend_preimage = PaymentPreimage([42; 32]);
4237         let payment_hash = PaymentHash(Sha256::hash(&keysend_preimage.0).to_byte_array());
4238
4239         let (onion_routing_packet, first_hop_msat, cltv_expiry) = onion_utils::create_payment_onion(
4240                 &secp_ctx, &route.paths[0], &session_priv, amt_msat, recipient_onion.clone(),
4241                 nodes[0].best_block_info().1, &payment_hash, &Some(keysend_preimage), prng_seed
4242         ).unwrap();
4243
4244         let update_add = msgs::UpdateAddHTLC {
4245                 channel_id: ChannelId([0; 32]),
4246                 htlc_id: 42,
4247                 amount_msat: first_hop_msat,
4248                 payment_hash,
4249                 cltv_expiry,
4250                 skimmed_fee_msat: None,
4251                 onion_routing_packet,
4252                 blinding_point: None,
4253         };
4254         let peeled_onion = crate::ln::onion_payment::peel_payment_onion(
4255                 &update_add, &&chanmon_cfgs[1].keys_manager, &&chanmon_cfgs[1].logger, &secp_ctx,
4256                 nodes[1].best_block_info().1, true, false
4257         ).unwrap();
4258         assert_eq!(peeled_onion.incoming_amt_msat, Some(amt_msat));
4259         match peeled_onion.routing {
4260                 PendingHTLCRouting::ReceiveKeysend {
4261                         payment_data, payment_metadata, custom_tlvs, ..
4262                 } => {
4263                         #[cfg(not(c_bindings))]
4264                         assert_eq!(&custom_tlvs, recipient_onion.custom_tlvs());
4265                         #[cfg(c_bindings)]
4266                         assert_eq!(custom_tlvs, recipient_onion.custom_tlvs());
4267                         assert!(payment_metadata.is_none());
4268                         assert!(payment_data.is_none());
4269                 },
4270                 _ => panic!()
4271         }
4272 }