_ Git - rust-lightning/blob - lightning/src/ln/payment_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Tests that test the payment retry logic in ChannelManager, including various edge-cases around
  11 //! serialization ordering between ChannelManager/ChannelMonitors and ensuring we can still retry
  12 //! payments thereafter.
  13
  14 use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Listen, Watch};
  15 use crate::chain::channelmonitor::{ANTI_REORG_DELAY, HTLC_FAIL_BACK_BUFFER, LATENCY_GRACE_PERIOD_BLOCKS};
  16 use crate::sign::EntropySource;
  17 use crate::chain::transaction::OutPoint;
  18 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentFailureReason, PaymentPurpose};
  19 use crate::ln::channel::{EXPIRE_PREV_CONFIG_TICKS, commit_tx_fee_msat, get_holder_selected_channel_reserve_satoshis, ANCHOR_OUTPUT_VALUE_SATOSHI};
  20 use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, RecentPaymentDetails, RecipientOnionFields, HTLCForwardInfo, PendingHTLCRouting, PendingAddHTLCInfo};
  21 use crate::ln::features::{Bolt11InvoiceFeatures, ChannelTypeFeatures};
  22 use crate::ln::{msgs, ChannelId, PaymentHash, PaymentSecret, PaymentPreimage};
  23 use crate::ln::msgs::ChannelMessageHandler;
  24 use crate::ln::onion_utils;
  25 use crate::ln::outbound_payment::{IDEMPOTENCY_TIMEOUT_TICKS, Retry};
  26 use crate::routing::gossip::{EffectiveCapacity, RoutingFees};
  27 use crate::routing::router::{get_route, Path, PaymentParameters, Route, Router, RouteHint, RouteHintHop, RouteHop, RouteParameters, find_route};
  28 use crate::routing::scoring::ChannelUsage;
  29 use crate::util::config::UserConfig;
  30 use crate::util::test_utils;
  31 use crate::util::errors::APIError;
  32 use crate::util::ser::Writeable;
  33 use crate::util::string::UntrustedString;
  34
  35 use bitcoin::hashes::Hash;
  36 use bitcoin::hashes::sha256::Hash as Sha256;
  37 use bitcoin::network::constants::Network;
  38 use bitcoin::secp256k1::{Secp256k1, SecretKey};
  39
  40 use crate::prelude::*;
  41
  42 use crate::ln::functional_test_utils;
  43 use crate::ln::functional_test_utils::*;
  44 use crate::routing::gossip::NodeId;
  45 #[cfg(feature = "std")]
  46 use {
  47         crate::util::time::tests::SinceEpoch,
  48         std::time::{SystemTime, Instant, Duration}
  49 };
  50
  51 #[test]
  52 fn mpp_failure() {
  53         let chanmon_cfgs = create_chanmon_cfgs(4);
  54         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
  55         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
  56         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
  57
  58         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
  59         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
  60         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
  61         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3).0.contents.short_channel_id;
  62
  63         let (mut route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
  64         let path = route.paths[0].clone();
  65         route.paths.push(path);
  66         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
  67         route.paths[0].hops[0].short_channel_id = chan_1_id;
  68         route.paths[0].hops[1].short_channel_id = chan_3_id;
  69         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
  70         route.paths[1].hops[0].short_channel_id = chan_2_id;
  71         route.paths[1].hops[1].short_channel_id = chan_4_id;
  72         send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
  73         fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
  74 }
  75
  76 #[test]
  77 fn mpp_retry() {
  78         let chanmon_cfgs = create_chanmon_cfgs(4);
  79         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
  80         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
  81         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
  82
  83         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
  84         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
  85         let (chan_3_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
  86         let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes(&nodes, 3, 2);
  87         // Rebalance
  88         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
  89
  90         let amt_msat = 1_000_000;
  91         let max_total_routing_fee_msat = 50_000;
  92         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
  93                 .with_bolt11_features(nodes[3].node.bolt11_invoice_features()).unwrap();
  94         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(
  95                 nodes[0], nodes[3], payment_params, amt_msat, Some(max_total_routing_fee_msat));
  96         let path = route.paths[0].clone();
  97         route.paths.push(path);
  98         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
  99         route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
 100         route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
 101         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
 102         route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
 103         route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
 104
 105         // Initiate the MPP payment.
 106         let payment_id = PaymentId(payment_hash.0);
 107         let mut route_params = route.route_params.clone().unwrap();
 108
 109         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
 110         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 111                 payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
 112         check_added_monitors!(nodes[0], 2); // one monitor per path
 113         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 114         assert_eq!(events.len(), 2);
 115
 116         // Pass half of the payment along the success path.
 117         let success_path_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 118         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), success_path_msgs, false, None);
 119
 120         // Add the HTLC along the first hop.
 121         let fail_path_msgs_1 = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 122         let send_event = SendEvent::from_event(fail_path_msgs_1);
 123         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 124         commitment_signed_dance!(nodes[2], nodes[0], &send_event.commitment_msg, false);
 125
 126         // Attempt to forward the payment and complete the 2nd path's failure.
 127         expect_pending_htlcs_forwardable!(&nodes[2]);
 128         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id }]);
 129         let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 130         assert!(htlc_updates.update_add_htlcs.is_empty());
 131         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 132         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 133         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 134         check_added_monitors!(nodes[2], 1);
 135         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_updates.update_fail_htlcs[0]);
 136         commitment_signed_dance!(nodes[0], nodes[2], htlc_updates.commitment_signed, false);
 137         let mut events = nodes[0].node.get_and_clear_pending_events();
 138         match events[1] {
 139                 Event::PendingHTLCsForwardable { .. } => {},
 140                 _ => panic!("Unexpected event")
 141         }
 142         events.remove(1);
 143         expect_payment_failed_conditions_event(events, payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain());
 144
 145         // Rebalance the channel so the second half of the payment can succeed.
 146         send_payment(&nodes[3], &vec!(&nodes[2])[..], 1_500_000);
 147
 148         // Retry the second half of the payment and make sure it succeeds.
 149         route.paths.remove(0);
 150         route_params.final_value_msat = 1_000_000;
 151         route_params.payment_params.previously_failed_channels.push(chan_4_update.contents.short_channel_id);
 152         // Check the remaining max total routing fee for the second attempt is 50_000 - 1_000 msat fee
 153         // used by the first path
 154         route_params.max_total_routing_fee_msat = Some(max_total_routing_fee_msat - 1_000);
 155         route.route_params = Some(route_params.clone());
 156         nodes[0].router.expect_find_route(route_params, Ok(route));
 157         nodes[0].node.process_pending_htlc_forwards();
 158         check_added_monitors!(nodes[0], 1);
 159         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 160         assert_eq!(events.len(), 1);
 161         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 2_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 162         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 163 }
 164
 165 #[test]
 166 fn mpp_retry_overpay() {
 167         // We create an MPP scenario with two paths in which we need to overpay to reach
 168         // htlc_minimum_msat. We then fail the overpaid path and check that on retry our
 169         // max_total_routing_fee_msat only accounts for the path's fees, but not for the fees overpaid
 170         // in the first attempt.
 171         let chanmon_cfgs = create_chanmon_cfgs(4);
 172         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 173         let mut user_config = test_default_channel_config();
 174         user_config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 100;
 175         let mut limited_config_1 = user_config.clone();
 176         limited_config_1.channel_handshake_config.our_htlc_minimum_msat = 35_000_000;
 177         let mut limited_config_2 = user_config.clone();
 178         limited_config_2.channel_handshake_config.our_htlc_minimum_msat = 34_500_000;
 179         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs,
 180                 &[Some(user_config), Some(limited_config_1), Some(limited_config_2), Some(user_config)]);
 181         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 182
 183         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 40_000, 0);
 184         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 40_000, 0);
 185         let (_chan_3_update, _, _, _) = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 40_000, 0);
 186         let (chan_4_update, _, chan_4_id, _) = create_announced_chan_between_nodes_with_value(&nodes, 3, 2, 40_000, 0);
 187
 188         let amt_msat = 70_000_000;
 189         let max_total_routing_fee_msat = Some(1_000_000);
 190
 191         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
 192                 .with_bolt11_features(nodes[3].node.bolt11_invoice_features()).unwrap();
 193         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(
 194                 nodes[0], nodes[3], payment_params, amt_msat, max_total_routing_fee_msat);
 195
 196         // Check we overpay on the second path which we're about to fail.
 197         assert_eq!(chan_1_update.contents.fee_proportional_millionths, 0);
 198         let overpaid_amount_1 = route.paths[0].fee_msat() as u32 - chan_1_update.contents.fee_base_msat;
 199         assert_eq!(overpaid_amount_1, 0);
 200
 201         assert_eq!(chan_2_update.contents.fee_proportional_millionths, 0);
 202         let overpaid_amount_2 = route.paths[1].fee_msat() as u32 - chan_2_update.contents.fee_base_msat;
 203
 204         let total_overpaid_amount = overpaid_amount_1 + overpaid_amount_2;
 205
 206         // Initiate the payment.
 207         let payment_id = PaymentId(payment_hash.0);
 208         let mut route_params = route.route_params.clone().unwrap();
 209
 210         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
 211         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 212                 payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
 213         check_added_monitors!(nodes[0], 2); // one monitor per path
 214         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 215         assert_eq!(events.len(), 2);
 216
 217         // Pass half of the payment along the success path.
 218         let success_path_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 219         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], amt_msat, payment_hash,
 220                 Some(payment_secret), success_path_msgs, false, None);
 221
 222         // Add the HTLC along the first hop.
 223         let fail_path_msgs_1 = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 224         let send_event = SendEvent::from_event(fail_path_msgs_1);
 225         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 226         commitment_signed_dance!(nodes[2], nodes[0], &send_event.commitment_msg, false);
 227
 228         // Attempt to forward the payment and complete the 2nd path's failure.
 229         expect_pending_htlcs_forwardable!(&nodes[2]);
 230         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[2],
 231                 vec![HTLCDestination::NextHopChannel {
 232                         node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_id
 233                 }]
 234         );
 235         let htlc_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 236         assert!(htlc_updates.update_add_htlcs.is_empty());
 237         assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
 238         assert!(htlc_updates.update_fulfill_htlcs.is_empty());
 239         assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
 240         check_added_monitors!(nodes[2], 1);
 241         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(),
 242                 &htlc_updates.update_fail_htlcs[0]);
 243         commitment_signed_dance!(nodes[0], nodes[2], htlc_updates.commitment_signed, false);
 244         let mut events = nodes[0].node.get_and_clear_pending_events();
 245         match events[1] {
 246                 Event::PendingHTLCsForwardable { .. } => {},
 247                 _ => panic!("Unexpected event")
 248         }
 249         events.remove(1);
 250         expect_payment_failed_conditions_event(events, payment_hash, false,
 251                 PaymentFailedConditions::new().mpp_parts_remain());
 252
 253         // Rebalance the channel so the second half of the payment can succeed.
 254         send_payment(&nodes[3], &vec!(&nodes[2])[..], 38_000_000);
 255
 256         // Retry the second half of the payment and make sure it succeeds.
 257         let first_path_value = route.paths[0].final_value_msat();
 258         assert_eq!(first_path_value, 36_000_000);
 259
 260         route.paths.remove(0);
 261         route_params.final_value_msat -= first_path_value;
 262         route_params.payment_params.previously_failed_channels.push(chan_4_update.contents.short_channel_id);
 263         // Check the remaining max total routing fee for the second attempt accounts only for 1_000 msat
 264         // base fee, but not for overpaid value of the first try.
 265         route_params.max_total_routing_fee_msat.as_mut().map(|m| *m -= 1000);
 266
 267         route.route_params = Some(route_params.clone());
 268         nodes[0].router.expect_find_route(route_params, Ok(route));
 269         nodes[0].node.process_pending_htlc_forwards();
 270
 271         check_added_monitors!(nodes[0], 1);
 272         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 273         assert_eq!(events.len(), 1);
 274         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], amt_msat, payment_hash,
 275                 Some(payment_secret), events.pop().unwrap(), true, None);
 276
 277         // Can't use claim_payment_along_route as it doesn't support overpayment, so we break out the
 278         // individual steps here.
 279         let extra_fees = vec![0, total_overpaid_amount];
 280         let expected_total_fee_msat = do_claim_payment_along_route_with_extra_penultimate_hop_fees(
 281                 &nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], &extra_fees[..], false,
 282                 payment_preimage);
 283         expect_payment_sent!(&nodes[0], payment_preimage, Some(expected_total_fee_msat));
 284 }
 285
 286 fn do_mpp_receive_timeout(send_partial_mpp: bool) {
 287         let chanmon_cfgs = create_chanmon_cfgs(4);
 288         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 289         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 290         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 291
 292         let (chan_1_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
 293         let (chan_2_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 0, 2);
 294         let (chan_3_update, _, chan_3_id, _) = create_announced_chan_between_nodes(&nodes, 1, 3);
 295         let (chan_4_update, _, _, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 296
 297         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 100_000);
 298         let path = route.paths[0].clone();
 299         route.paths.push(path);
 300         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
 301         route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
 302         route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
 303         route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
 304         route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
 305         route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
 306
 307         // Initiate the MPP payment.
 308         nodes[0].node.send_payment_with_route(&route, payment_hash,
 309                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 310         check_added_monitors!(nodes[0], 2); // one monitor per path
 311         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 312         assert_eq!(events.len(), 2);
 313
 314         // Pass half of the payment along the first path.
 315         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 316         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_1_msgs, false, None);
 317
 318         if send_partial_mpp {
 319                 // Time out the partial MPP
 320                 for _ in 0..MPP_TIMEOUT_TICKS {
 321                         nodes[3].node.timer_tick_occurred();
 322                 }
 323
 324                 // Failed HTLC from node 3 -> 1
 325                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 326                 let htlc_fail_updates_3_1 = get_htlc_update_msgs!(nodes[3], nodes[1].node.get_our_node_id());
 327                 assert_eq!(htlc_fail_updates_3_1.update_fail_htlcs.len(), 1);
 328                 nodes[1].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &htlc_fail_updates_3_1.update_fail_htlcs[0]);
 329                 check_added_monitors!(nodes[3], 1);
 330                 commitment_signed_dance!(nodes[1], nodes[3], htlc_fail_updates_3_1.commitment_signed, false);
 331
 332                 // Failed HTLC from node 1 -> 0
 333                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_3_id }]);
 334                 let htlc_fail_updates_1_0 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 335                 assert_eq!(htlc_fail_updates_1_0.update_fail_htlcs.len(), 1);
 336                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates_1_0.update_fail_htlcs[0]);
 337                 check_added_monitors!(nodes[1], 1);
 338                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates_1_0.commitment_signed, false);
 339
 340                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new().mpp_parts_remain().expected_htlc_error_data(23, &[][..]));
 341         } else {
 342                 // Pass half of the payment along the second path.
 343                 let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 344                 pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash, Some(payment_secret), node_2_msgs, true, None);
 345
 346                 // Even after MPP_TIMEOUT_TICKS we should not timeout the MPP if we have all the parts
 347                 for _ in 0..MPP_TIMEOUT_TICKS {
 348                         nodes[3].node.timer_tick_occurred();
 349                 }
 350
 351                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 352         }
 353 }
 354
 355 #[test]
 356 fn mpp_receive_timeout() {
 357         do_mpp_receive_timeout(true);
 358         do_mpp_receive_timeout(false);
 359 }
 360
 361 #[test]
 362 fn test_keysend_payments() {
 363         do_test_keysend_payments(false, false);
 364         do_test_keysend_payments(false, true);
 365         do_test_keysend_payments(true, false);
 366         do_test_keysend_payments(true, true);
 367 }
 368
 369 fn do_test_keysend_payments(public_node: bool, with_retry: bool) {
 370         let chanmon_cfgs = create_chanmon_cfgs(2);
 371         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 372         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 373         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 374
 375         if public_node {
 376                 create_announced_chan_between_nodes(&nodes, 0, 1);
 377         } else {
 378                 create_chan_between_nodes(&nodes[0], &nodes[1]);
 379         }
 380         let payer_pubkey = nodes[0].node.get_our_node_id();
 381         let payee_pubkey = nodes[1].node.get_our_node_id();
 382         let route_params = RouteParameters::from_payment_params_and_value(
 383                 PaymentParameters::for_keysend(payee_pubkey, 40, false), 10000);
 384
 385         let network_graph = nodes[0].network_graph;
 386         let channels = nodes[0].node.list_usable_channels();
 387         let first_hops = channels.iter().collect::<Vec<_>>();
 388         let first_hops = if public_node { None } else { Some(first_hops.as_slice()) };
 389
 390         let scorer = test_utils::TestScorer::new();
 391         let random_seed_bytes = chanmon_cfgs[1].keys_manager.get_secure_random_bytes();
 392         let route = find_route(
 393                 &payer_pubkey, &route_params, &network_graph, first_hops,
 394                 nodes[0].logger, &scorer, &Default::default(), &random_seed_bytes
 395         ).unwrap();
 396
 397         {
 398                 let test_preimage = PaymentPreimage([42; 32]);
 399                 if with_retry {
 400                         nodes[0].node.send_spontaneous_payment_with_retry(Some(test_preimage),
 401                                 RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0),
 402                                 route_params, Retry::Attempts(1)).unwrap()
 403                 } else {
 404                         nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage),
 405                                 RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0)).unwrap()
 406                 };
 407         }
 408         check_added_monitors!(nodes[0], 1);
 409         let send_event = SendEvent::from_node(&nodes[0]);
 410         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 411         do_commitment_signed_dance(&nodes[1], &nodes[0], &send_event.commitment_msg, false, false);
 412         expect_pending_htlcs_forwardable!(nodes[1]);
 413         // Previously, a refactor caused us to stop including the payment preimage in the onion which
 414         // is sent as a part of keysend payments. Thus, to be extra careful here, we scope the preimage
 415         // above to demonstrate that we have no way to get the preimage at this point except by
 416         // extracting it from the onion nodes[1] received.
 417         let event = nodes[1].node.get_and_clear_pending_events();
 418         assert_eq!(event.len(), 1);
 419         if let Event::PaymentClaimable { purpose: PaymentPurpose::SpontaneousPayment(preimage), .. } = event[0] {
 420                 claim_payment(&nodes[0], &[&nodes[1]], preimage);
 421         } else { panic!(); }
 422 }
 423
 424 #[test]
 425 fn test_mpp_keysend() {
 426         let mut mpp_keysend_config = test_default_channel_config();
 427         mpp_keysend_config.accept_mpp_keysend = true;
 428         let chanmon_cfgs = create_chanmon_cfgs(4);
 429         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 430         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, Some(mpp_keysend_config)]);
 431         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 432
 433         create_announced_chan_between_nodes(&nodes, 0, 1);
 434         create_announced_chan_between_nodes(&nodes, 0, 2);
 435         create_announced_chan_between_nodes(&nodes, 1, 3);
 436         create_announced_chan_between_nodes(&nodes, 2, 3);
 437         let network_graph = nodes[0].network_graph;
 438
 439         let payer_pubkey = nodes[0].node.get_our_node_id();
 440         let payee_pubkey = nodes[3].node.get_our_node_id();
 441         let recv_value = 15_000_000;
 442         let route_params = RouteParameters::from_payment_params_and_value(
 443                 PaymentParameters::for_keysend(payee_pubkey, 40, true), recv_value);
 444         let scorer = test_utils::TestScorer::new();
 445         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
 446         let route = find_route(&payer_pubkey, &route_params, &network_graph, None, nodes[0].logger,
 447                 &scorer, &Default::default(), &random_seed_bytes).unwrap();
 448
 449         let payment_preimage = PaymentPreimage([42; 32]);
 450         let payment_secret = PaymentSecret(payment_preimage.0);
 451         let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage),
 452                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_preimage.0)).unwrap();
 453         check_added_monitors!(nodes[0], 2);
 454
 455         let expected_route: &[&[&Node]] = &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]];
 456         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 457         assert_eq!(events.len(), 2);
 458
 459         let ev = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
 460         pass_along_path(&nodes[0], expected_route[0], recv_value, payment_hash.clone(),
 461                 Some(payment_secret), ev.clone(), false, Some(payment_preimage));
 462
 463         let ev = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut events);
 464         pass_along_path(&nodes[0], expected_route[1], recv_value, payment_hash.clone(),
 465                 Some(payment_secret), ev.clone(), true, Some(payment_preimage));
 466         claim_payment_along_route(&nodes[0], expected_route, false, payment_preimage);
 467 }
 468
 469 #[test]
 470 fn test_reject_mpp_keysend_htlc() {
 471         // This test enforces that we reject MPP keysend HTLCs if our config states we don't support
 472         // MPP keysend. When receiving a payment, if we don't support MPP keysend we'll reject the
 473         // payment if it's keysend and has a payment secret, never reaching our payment validation
 474         // logic. To check that we enforce rejecting MPP keysends in our payment logic, here we send
 475         // keysend payments without payment secrets, then modify them by adding payment secrets in the
 476         // final node in between receiving the HTLCs and actually processing them.
 477         let mut reject_mpp_keysend_cfg = test_default_channel_config();
 478         reject_mpp_keysend_cfg.accept_mpp_keysend = false;
 479
 480         let chanmon_cfgs = create_chanmon_cfgs(4);
 481         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 482         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, Some(reject_mpp_keysend_cfg)]);
 483         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 484         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
 485         let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2).0.contents.short_channel_id;
 486         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3).0.contents.short_channel_id;
 487         let (update_a, _, chan_4_channel_id, _) = create_announced_chan_between_nodes(&nodes, 2, 3);
 488         let chan_4_id = update_a.contents.short_channel_id;
 489         let amount = 40_000;
 490         let (mut route, payment_hash, payment_preimage, _) = get_route_and_payment_hash!(nodes[0], nodes[3], amount);
 491
 492         // Pay along nodes[1]
 493         route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
 494         route.paths[0].hops[0].short_channel_id = chan_1_id;
 495         route.paths[0].hops[1].short_channel_id = chan_3_id;
 496
 497         let payment_id_0 = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 498         nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), RecipientOnionFields::spontaneous_empty(), payment_id_0).unwrap();
 499         check_added_monitors!(nodes[0], 1);
 500
 501         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 502         let update_add_0 = update_0.update_add_htlcs[0].clone();
 503         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_0);
 504         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
 505         expect_pending_htlcs_forwardable!(nodes[1]);
 506
 507         check_added_monitors!(&nodes[1], 1);
 508         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[3].node.get_our_node_id());
 509         let update_add_1 = update_1.update_add_htlcs[0].clone();
 510         nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &update_add_1);
 511         commitment_signed_dance!(nodes[3], nodes[1], update_1.commitment_signed, false, true);
 512
 513         assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
 514         for (_, pending_forwards) in nodes[3].node.forward_htlcs.lock().unwrap().iter_mut() {
 515                 for f in pending_forwards.iter_mut() {
 516                         match f {
 517                                 &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) => {
 518                                         match forward_info.routing {
 519                                                 PendingHTLCRouting::ReceiveKeysend { ref mut payment_data, .. } => {
 520                                                         *payment_data = Some(msgs::FinalOnionHopData {
 521                                                                 payment_secret: PaymentSecret([42; 32]),
 522                                                                 total_msat: amount * 2,
 523                                                         });
 524                                                 },
 525                                                 _ => panic!("Expected PendingHTLCRouting::ReceiveKeysend"),
 526                                         }
 527                                 },
 528                                 _ => {},
 529                         }
 530                 }
 531         }
 532         expect_pending_htlcs_forwardable!(nodes[3]);
 533
 534         // Pay along nodes[2]
 535         route.paths[0].hops[0].pubkey = nodes[2].node.get_our_node_id();
 536         route.paths[0].hops[0].short_channel_id = chan_2_id;
 537         route.paths[0].hops[1].short_channel_id = chan_4_id;
 538
 539         let payment_id_1 = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 540         nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), RecipientOnionFields::spontaneous_empty(), payment_id_1).unwrap();
 541         check_added_monitors!(nodes[0], 1);
 542
 543         let update_2 = get_htlc_update_msgs!(nodes[0], nodes[2].node.get_our_node_id());
 544         let update_add_2 = update_2.update_add_htlcs[0].clone();
 545         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_2);
 546         commitment_signed_dance!(nodes[2], nodes[0], &update_2.commitment_signed, false, true);
 547         expect_pending_htlcs_forwardable!(nodes[2]);
 548
 549         check_added_monitors!(&nodes[2], 1);
 550         let update_3 = get_htlc_update_msgs!(nodes[2], nodes[3].node.get_our_node_id());
 551         let update_add_3 = update_3.update_add_htlcs[0].clone();
 552         nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &update_add_3);
 553         commitment_signed_dance!(nodes[3], nodes[2], update_3.commitment_signed, false, true);
 554
 555         assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
 556         for (_, pending_forwards) in nodes[3].node.forward_htlcs.lock().unwrap().iter_mut() {
 557                 for f in pending_forwards.iter_mut() {
 558                         match f {
 559                                 &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) => {
 560                                         match forward_info.routing {
 561                                                 PendingHTLCRouting::ReceiveKeysend { ref mut payment_data, .. } => {
 562                                                         *payment_data = Some(msgs::FinalOnionHopData {
 563                                                                 payment_secret: PaymentSecret([42; 32]),
 564                                                                 total_msat: amount * 2,
 565                                                         });
 566                                                 },
 567                                                 _ => panic!("Expected PendingHTLCRouting::ReceiveKeysend"),
 568                                         }
 569                                 },
 570                                 _ => {},
 571                         }
 572                 }
 573         }
 574         expect_pending_htlcs_forwardable!(nodes[3]);
 575         check_added_monitors!(nodes[3], 1);
 576
 577         // Fail back along nodes[2]
 578         let update_fail_0 = get_htlc_update_msgs!(&nodes[3], &nodes[2].node.get_our_node_id());
 579         nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &update_fail_0.update_fail_htlcs[0]);
 580         commitment_signed_dance!(nodes[2], nodes[3], update_fail_0.commitment_signed, false);
 581         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_4_channel_id }]);
 582         check_added_monitors!(nodes[2], 1);
 583
 584         let update_fail_1 = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 585         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_1.update_fail_htlcs[0]);
 586         commitment_signed_dance!(nodes[0], nodes[2], update_fail_1.commitment_signed, false);
 587
 588         expect_payment_failed_conditions(&nodes[0], payment_hash, true, PaymentFailedConditions::new());
 589         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], vec![HTLCDestination::FailedPayment { payment_hash }]);
 590 }
 591
 592
 593 #[test]
 594 fn no_pending_leak_on_initial_send_failure() {
 595         // In an earlier version of our payment tracking, we'd have a retry entry even when the initial
 596         // HTLC for payment failed to send due to local channel errors (e.g. peer disconnected). In this
 597         // case, the user wouldn't have a PaymentId to retry the payment with, but we'd think we have a
 598         // pending payment forever and never time it out.
 599         // Here we test exactly that - retrying a payment when a peer was disconnected on the first
 600         // try, and then check that no pending payment is being tracked.
 601         let chanmon_cfgs = create_chanmon_cfgs(2);
 602         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 603         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 604         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 605
 606         create_announced_chan_between_nodes(&nodes, 0, 1);
 607
 608         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 609
 610         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 611         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 612
 613         unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash,
 614                         RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
 615                 ), true, APIError::ChannelUnavailable { ref err },
 616                 assert_eq!(err, "Peer for first hop currently disconnected"));
 617
 618         assert!(!nodes[0].node.has_pending_payments());
 619 }
 620
 621 fn do_retry_with_no_persist(confirm_before_reload: bool) {
 622         // If we send a pending payment and `send_payment` returns success, we should always either
 623         // return a payment failure event or a payment success event, and on failure the payment should
 624         // be retryable.
 625         //
 626         // In order to do so when the ChannelManager isn't immediately persisted (which is normal - its
 627         // always persisted asynchronously), the ChannelManager has to reload some payment data from
 628         // ChannelMonitor(s) in some cases. This tests that reloading.
 629         //
 630         // `confirm_before_reload` confirms the channel-closing commitment transaction on-chain prior
 631         // to reloading the ChannelManager, increasing test coverage in ChannelMonitor HTLC tracking
 632         // which has separate codepaths for "commitment transaction already confirmed" and not.
 633         let chanmon_cfgs = create_chanmon_cfgs(3);
 634         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 635         let persister;
 636         let new_chain_monitor;
 637         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 638         let nodes_0_deserialized;
 639         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 640
 641         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 642         let (_, _, chan_id_2, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
 643
 644         // Serialize the ChannelManager prior to sending payments
 645         let nodes_0_serialized = nodes[0].node.encode();
 646
 647         // Send two payments - one which will get to nodes[2] and will be claimed, one which we'll time
 648         // out and retry.
 649         let amt_msat = 1_000_000;
 650         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
 651         let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000);
 652         let route_params = route.route_params.unwrap().clone();
 653         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
 654                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
 655         check_added_monitors!(nodes[0], 1);
 656
 657         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 658         assert_eq!(events.len(), 1);
 659         let payment_event = SendEvent::from_event(events.pop().unwrap());
 660         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 661
 662         // We relay the payment to nodes[1] while its disconnected from nodes[2], causing the payment
 663         // to be returned immediately to nodes[0], without having nodes[2] fail the inbound payment
 664         // which would prevent retry.
 665         nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id());
 666         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 667
 668         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 669         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true);
 670         // nodes[1] now immediately fails the HTLC as the next-hop channel is disconnected
 671         let _ = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 672
 673         reconnect_nodes(ReconnectArgs::new(&nodes[1], &nodes[2]));
 674
 675         let as_commitment_tx = get_local_commitment_txn!(nodes[0], chan_id)[0].clone();
 676         if confirm_before_reload {
 677                 mine_transaction(&nodes[0], &as_commitment_tx);
 678                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 679         }
 680
 681         // The ChannelMonitor should always be the latest version, as we're required to persist it
 682         // during the `commitment_signed_dance!()`.
 683         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 684         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 685
 686         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 687         // force-close the channel.
 688         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[1].node.get_our_node_id()], 100000);
 689         assert!(nodes[0].node.list_channels().is_empty());
 690         assert!(nodes[0].node.has_pending_payments());
 691         nodes[0].node.timer_tick_occurred();
 692         if !confirm_before_reload {
 693                 let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 694                 assert_eq!(as_broadcasted_txn.len(), 1);
 695                 assert_eq!(as_broadcasted_txn[0].txid(), as_commitment_tx.txid());
 696         } else {
 697                 assert!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
 698         }
 699         check_added_monitors!(nodes[0], 1);
 700
 701         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 702         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 703                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 704         }, true).unwrap();
 705         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 706
 707         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 708         // error, as the channel has hit the chain.
 709         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 710                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 711         }, false).unwrap();
 712         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 713         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 714         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 715         assert_eq!(as_err.len(), 2);
 716         match as_err[1] {
 717                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 718                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 719                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 720                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}",
 721                                 &nodes[1].node.get_our_node_id())) }, [nodes[0].node.get_our_node_id()], 100000);
 722                         check_added_monitors!(nodes[1], 1);
 723                         assert_eq!(nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 724                 },
 725                 _ => panic!("Unexpected event"),
 726         }
 727         check_closed_broadcast!(nodes[1], false);
 728
 729         // Now claim the first payment, which should allow nodes[1] to claim the payment on-chain when
 730         // we close in a moment.
 731         nodes[2].node.claim_funds(payment_preimage_1);
 732         check_added_monitors!(nodes[2], 1);
 733         expect_payment_claimed!(nodes[2], payment_hash_1, 1_000_000);
 734
 735         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 736         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
 737         check_added_monitors!(nodes[1], 1);
 738         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 739         expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], None, true, false);
 740
 741         if confirm_before_reload {
 742                 let best_block = nodes[0].blocks.lock().unwrap().last().unwrap().clone();
 743                 nodes[0].node.best_block_updated(&best_block.0.header, best_block.1);
 744         }
 745
 746         // Create a new channel on which to retry the payment before we fail the payment via the
 747         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 748         // connecting several blocks while creating the channel (implying time has passed).
 749         create_announced_chan_between_nodes(&nodes, 0, 1);
 750         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 751
 752         mine_transaction(&nodes[1], &as_commitment_tx);
 753         let bs_htlc_claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 754         assert_eq!(bs_htlc_claim_txn.len(), 1);
 755         check_spends!(bs_htlc_claim_txn[0], as_commitment_tx);
 756
 757         if !confirm_before_reload {
 758                 mine_transaction(&nodes[0], &as_commitment_tx);
 759         }
 760         mine_transaction(&nodes[0], &bs_htlc_claim_txn[0]);
 761         expect_payment_sent(&nodes[0], payment_preimage_1, None, true, false);
 762         connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
 763         let (first_htlc_timeout_tx, second_htlc_timeout_tx) = {
 764                 let mut txn = nodes[0].tx_broadcaster.unique_txn_broadcast();
 765                 assert_eq!(txn.len(), 2);
 766                 (txn.remove(0), txn.remove(0))
 767         };
 768         check_spends!(first_htlc_timeout_tx, as_commitment_tx);
 769         check_spends!(second_htlc_timeout_tx, as_commitment_tx);
 770         if first_htlc_timeout_tx.input[0].previous_output == bs_htlc_claim_txn[0].input[0].previous_output {
 771                 confirm_transaction(&nodes[0], &second_htlc_timeout_tx);
 772         } else {
 773                 confirm_transaction(&nodes[0], &first_htlc_timeout_tx);
 774         }
 775         nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 776         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new());
 777
 778         // Finally, retry the payment (which was reloaded from the ChannelMonitor when nodes[0] was
 779         // reloaded) via a route over the new channel, which work without issue and eventually be
 780         // received and claimed at the recipient just like any other payment.
 781         let (mut new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 782
 783         // Update the fee on the middle hop to ensure PaymentSent events have the correct (retried) fee
 784         // and not the original fee. We also update node[1]'s relevant config as
 785         // do_claim_payment_along_route expects us to never overpay.
 786         {
 787                 let per_peer_state = nodes[1].node.per_peer_state.read().unwrap();
 788                 let mut peer_state = per_peer_state.get(&nodes[2].node.get_our_node_id())
 789                         .unwrap().lock().unwrap();
 790                 let mut channel = peer_state.channel_by_id.get_mut(&chan_id_2).unwrap();
 791                 let mut new_config = channel.context().config();
 792                 new_config.forwarding_fee_base_msat += 100_000;
 793                 channel.context_mut().update_config(&new_config);
 794                 new_route.paths[0].hops[0].fee_msat += 100_000;
 795         }
 796
 797         // Force expiration of the channel's previous config.
 798         for _ in 0..EXPIRE_PREV_CONFIG_TICKS {
 799                 nodes[1].node.timer_tick_occurred();
 800         }
 801
 802         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash, // Shouldn't be allowed to retry a fulfilled payment
 803                 RecipientOnionFields::secret_only(payment_secret), payment_id_1).is_err());
 804         nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 805                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 806         check_added_monitors!(nodes[0], 1);
 807         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 808         assert_eq!(events.len(), 1);
 809         pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
 810         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
 811         expect_payment_sent!(nodes[0], payment_preimage, Some(new_route.paths[0].hops[0].fee_msat));
 812 }
 813
 814 #[test]
 815 fn retry_with_no_persist() {
 816         do_retry_with_no_persist(true);
 817         do_retry_with_no_persist(false);
 818 }
 819
 820 fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 821         // Test that an off-chain completed payment is not retryable on restart. This was previously
 822         // broken for dust payments, but we test for both dust and non-dust payments.
 823         //
 824         // `use_dust` switches to using a dust HTLC, which results in the HTLC not having an on-chain
 825         // output at all.
 826         let chanmon_cfgs = create_chanmon_cfgs(3);
 827         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 828
 829         let mut manually_accept_config = test_default_channel_config();
 830         manually_accept_config.manually_accept_inbound_channels = true;
 831
 832         let first_persister;
 833         let first_new_chain_monitor;
 834         let second_persister;
 835         let second_new_chain_monitor;
 836         let third_persister;
 837         let third_new_chain_monitor;
 838
 839         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(manually_accept_config), None]);
 840         let first_nodes_0_deserialized;
 841         let second_nodes_0_deserialized;
 842         let third_nodes_0_deserialized;
 843
 844         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 845
 846         // Because we set nodes[1] to manually accept channels, just open a 0-conf channel.
 847         let (funding_tx, chan_id) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 848         confirm_transaction(&nodes[0], &funding_tx);
 849         confirm_transaction(&nodes[1], &funding_tx);
 850         // Ignore the announcement_signatures messages
 851         nodes[0].node.get_and_clear_pending_msg_events();
 852         nodes[1].node.get_and_clear_pending_msg_events();
 853         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 854
 855         // Serialize the ChannelManager prior to sending payments
 856         let mut nodes_0_serialized = nodes[0].node.encode();
 857
 858         let route = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 }).0;
 859         let (payment_preimage, payment_hash, payment_secret, payment_id) = send_along_route(&nodes[0], route, &[&nodes[1], &nodes[2]], if use_dust { 1_000 } else { 1_000_000 });
 860
 861         // The ChannelMonitor should always be the latest version, as we're required to persist it
 862         // during the `commitment_signed_dance!()`.
 863         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 864
 865         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized], first_persister, first_new_chain_monitor, first_nodes_0_deserialized);
 866         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 867
 868         // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
 869         // force-close the channel.
 870         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[1].node.get_our_node_id()], 100000);
 871         nodes[0].node.timer_tick_occurred();
 872         assert!(nodes[0].node.list_channels().is_empty());
 873         assert!(nodes[0].node.has_pending_payments());
 874         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
 875         check_added_monitors!(nodes[0], 1);
 876
 877         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 878                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 879         }, true).unwrap();
 880         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 881
 882         // Now nodes[1] should send a channel reestablish, which nodes[0] will respond to with an
 883         // error, as the channel has hit the chain.
 884         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 885                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 886         }, false).unwrap();
 887         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
 888         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 889         let as_err = nodes[0].node.get_and_clear_pending_msg_events();
 890         assert_eq!(as_err.len(), 2);
 891         let bs_commitment_tx;
 892         match as_err[1] {
 893                 MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
 894                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
 895                         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
 896                         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) }
 897                                 , [nodes[0].node.get_our_node_id()], 100000);
 898                         check_added_monitors!(nodes[1], 1);
 899                         bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 900                 },
 901                 _ => panic!("Unexpected event"),
 902         }
 903         check_closed_broadcast!(nodes[1], false);
 904
 905         // Now fail back the payment from nodes[2] to nodes[1]. This doesn't really matter as the
 906         // previous hop channel is already on-chain, but it makes nodes[2] willing to see additional
 907         // incoming HTLCs with the same payment hash later.
 908         nodes[2].node.fail_htlc_backwards(&payment_hash);
 909         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
 910         check_added_monitors!(nodes[2], 1);
 911
 912         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 913         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fail_htlcs[0]);
 914         commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
 915         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1],
 916                 [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 917
 918         // Connect the HTLC-Timeout transaction, timing out the HTLC on both nodes (but not confirming
 919         // the HTLC-Timeout transaction beyond 1 conf). For dust HTLCs, the HTLC is considered resolved
 920         // after the commitment transaction, so always connect the commitment transaction.
 921         mine_transaction(&nodes[0], &bs_commitment_tx[0]);
 922         mine_transaction(&nodes[1], &bs_commitment_tx[0]);
 923         if !use_dust {
 924                 connect_blocks(&nodes[0], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
 925                 connect_blocks(&nodes[1], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
 926                 let as_htlc_timeout = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 927                 check_spends!(as_htlc_timeout[0], bs_commitment_tx[0]);
 928                 assert_eq!(as_htlc_timeout.len(), 1);
 929
 930                 mine_transaction(&nodes[0], &as_htlc_timeout[0]);
 931                 // nodes[0] may rebroadcast (or RBF-bump) its HTLC-Timeout, so wipe the announced set.
 932                 nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 933                 mine_transaction(&nodes[1], &as_htlc_timeout[0]);
 934         }
 935
 936         // Create a new channel on which to retry the payment before we fail the payment via the
 937         // HTLC-Timeout transaction. This avoids ChannelManager timing out the payment due to us
 938         // connecting several blocks while creating the channel (implying time has passed).
 939         // We do this with a zero-conf channel to avoid connecting blocks as a side-effect.
 940         let (_, chan_id_3) = open_zero_conf_channel(&nodes[0], &nodes[1], None);
 941         assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
 942
 943         // If we attempt to retry prior to the HTLC-Timeout (or commitment transaction, for dust HTLCs)
 944         // confirming, we will fail as it's considered still-pending...
 945         let (new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 });
 946         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 947                 Err(PaymentSendFailure::DuplicatePayment) => {},
 948                 _ => panic!("Unexpected error")
 949         }
 950         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 951
 952         // After ANTI_REORG_DELAY confirmations, the HTLC should be failed and we can try the payment
 953         // again. We serialize the node first as we'll then test retrying the HTLC after a restart
 954         // (which should also still work).
 955         connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
 956         connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 957         expect_payment_failed_conditions(&nodes[0], payment_hash, false, PaymentFailedConditions::new());
 958
 959         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 960         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 961         nodes_0_serialized = nodes[0].node.encode();
 962
 963         // After the payment failed, we're free to send it again.
 964         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 965                 RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
 966         assert!(!nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 967
 968         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], second_persister, second_new_chain_monitor, second_nodes_0_deserialized);
 969         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 970
 971         nodes[0].node.test_process_background_events();
 972         check_added_monitors(&nodes[0], 1);
 973
 974         let mut reconnect_args = ReconnectArgs::new(&nodes[0], &nodes[1]);
 975         reconnect_args.send_channel_ready = (true, true);
 976         reconnect_nodes(reconnect_args);
 977
 978         // Now resend the payment, delivering the HTLC and actually claiming it this time. This ensures
 979         // the payment is not (spuriously) listed as still pending.
 980         assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
 981                 RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
 982         check_added_monitors!(nodes[0], 1);
 983         pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], if use_dust { 1_000 } else { 1_000_000 }, payment_hash, payment_secret);
 984         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
 985
 986         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
 987                 Err(PaymentSendFailure::DuplicatePayment) => {},
 988                 _ => panic!("Unexpected error")
 989         }
 990         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 991
 992         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 993         let chan_1_monitor_serialized = get_monitor!(nodes[0], chan_id_3).encode();
 994         nodes_0_serialized = nodes[0].node.encode();
 995
 996         // Check that after reload we can send the payment again (though we shouldn't, since it was
 997         // claimed previously).
 998         reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], third_persister, third_new_chain_monitor, third_nodes_0_deserialized);
 999         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
1000
1001         nodes[0].node.test_process_background_events();
1002         check_added_monitors(&nodes[0], 1);
1003
1004         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1005
1006         match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
1007                 Err(PaymentSendFailure::DuplicatePayment) => {},
1008                 _ => panic!("Unexpected error")
1009         }
1010         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1011 }
1012
1013 #[test]
1014 fn test_completed_payment_not_retryable_on_reload() {
1015         do_test_completed_payment_not_retryable_on_reload(true);
1016         do_test_completed_payment_not_retryable_on_reload(false);
1017 }
1018
1019
1020 fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, confirm_commitment_tx: bool, payment_timeout: bool) {
1021         // When a Channel is closed, any outbound HTLCs which were relayed through it are simply
1022         // dropped when the Channel is. From there, the ChannelManager relies on the ChannelMonitor
1023         // having a copy of the relevant fail-/claim-back data and processes the HTLC fail/claim when
1024         // the ChannelMonitor tells it to.
1025         //
1026         // If, due to an on-chain event, an HTLC is failed/claimed, we should avoid providing the
1027         // ChannelManager the HTLC event until after the monitor is re-persisted. This should prevent a
1028         // duplicate HTLC fail/claim (e.g. via a PaymentPathFailed event).
1029         let chanmon_cfgs = create_chanmon_cfgs(2);
1030         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1031         let persister;
1032         let new_chain_monitor;
1033         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1034         let nodes_0_deserialized;
1035         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1036
1037         let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1);
1038
1039         // Route a payment, but force-close the channel before the HTLC fulfill message arrives at
1040         // nodes[0].
1041         let (payment_preimage, payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 10_000_000);
1042         nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap();
1043         check_closed_broadcast!(nodes[0], true);
1044         check_added_monitors!(nodes[0], 1);
1045         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 100000);
1046
1047         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1048         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
1049
1050         // Connect blocks until the CLTV timeout is up so that we get an HTLC-Timeout transaction
1051         connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
1052         let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
1053         assert_eq!(node_txn.len(), 3);
1054         assert_eq!(node_txn[0].txid(), node_txn[1].txid());
1055         check_spends!(node_txn[1], funding_tx);
1056         check_spends!(node_txn[2], node_txn[1]);
1057         let timeout_txn = vec![node_txn[2].clone()];
1058
1059         nodes[1].node.claim_funds(payment_preimage);
1060         check_added_monitors!(nodes[1], 1);
1061         expect_payment_claimed!(nodes[1], payment_hash, 10_000_000);
1062
1063         connect_block(&nodes[1], &create_dummy_block(nodes[1].best_block_hash(), 42, vec![node_txn[1].clone()]));
1064         check_closed_broadcast!(nodes[1], true);
1065         check_added_monitors!(nodes[1], 1);
1066         check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed, [nodes[0].node.get_our_node_id()], 100000);
1067         let claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
1068         assert_eq!(claim_txn.len(), 1);
1069         check_spends!(claim_txn[0], node_txn[1]);
1070
1071         connect_block(&nodes[0], &create_dummy_block(nodes[0].best_block_hash(), 42, vec![node_txn[1].clone()]));
1072
1073         if confirm_commitment_tx {
1074                 connect_blocks(&nodes[0], BREAKDOWN_TIMEOUT as u32 - 1);
1075         }
1076
1077         let claim_block = create_dummy_block(nodes[0].best_block_hash(), 42, if payment_timeout { timeout_txn } else { vec![claim_txn[0].clone()] });
1078
1079         if payment_timeout {
1080                 assert!(confirm_commitment_tx); // Otherwise we're spending below our CSV!
1081                 connect_block(&nodes[0], &claim_block);
1082                 connect_blocks(&nodes[0], ANTI_REORG_DELAY - 2);
1083         }
1084
1085         // Now connect the HTLC claim transaction with the ChainMonitor-generated ChannelMonitor update
1086         // returning InProgress. This should cause the claim event to never make its way to the
1087         // ChannelManager.
1088         chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap().clear();
1089         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
1090
1091         if payment_timeout {
1092                 connect_blocks(&nodes[0], 1);
1093         } else {
1094                 connect_block(&nodes[0], &claim_block);
1095         }
1096
1097         let funding_txo = OutPoint { txid: funding_tx.txid(), index: 0 };
1098         let mon_updates: Vec<_> = chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap()
1099                 .get_mut(&funding_txo).unwrap().drain().collect();
1100         // If we are using chain::Confirm instead of chain::Listen, we will get the same update twice.
1101         // If we're testing connection idempotency we may get substantially more.
1102         assert!(mon_updates.len() >= 1);
1103         assert!(nodes[0].chain_monitor.release_pending_monitor_events().is_empty());
1104         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1105
1106         // If we persist the ChannelManager here, we should get the PaymentSent event after
1107         // deserialization.
1108         let mut chan_manager_serialized = Vec::new();
1109         if !persist_manager_post_event {
1110                 chan_manager_serialized = nodes[0].node.encode();
1111         }
1112
1113         // Now persist the ChannelMonitor and inform the ChainMonitor that we're done, generating the
1114         // payment sent event.
1115         chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
1116         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
1117         for update in mon_updates {
1118                 nodes[0].chain_monitor.chain_monitor.channel_monitor_updated(funding_txo, update).unwrap();
1119         }
1120         if payment_timeout {
1121                 expect_payment_failed!(nodes[0], payment_hash, false);
1122         } else {
1123                 expect_payment_sent(&nodes[0], payment_preimage, None, true, false);
1124         }
1125
1126         // If we persist the ChannelManager after we get the PaymentSent event, we shouldn't get it
1127         // twice.
1128         if persist_manager_post_event {
1129                 chan_manager_serialized = nodes[0].node.encode();
1130         }
1131
1132         // Now reload nodes[0]...
1133         reload_node!(nodes[0], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
1134
1135         if persist_manager_post_event {
1136                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1137         } else if payment_timeout {
1138                 expect_payment_failed!(nodes[0], payment_hash, false);
1139         } else {
1140                 expect_payment_sent(&nodes[0], payment_preimage, None, true, false);
1141         }
1142
1143         // Note that if we re-connect the block which exposed nodes[0] to the payment preimage (but
1144         // which the current ChannelMonitor has not seen), the ChannelManager's de-duplication of
1145         // payment events should kick in, leaving us with no pending events here.
1146         let height = nodes[0].blocks.lock().unwrap().len() as u32 - 1;
1147         nodes[0].chain_monitor.chain_monitor.block_connected(&claim_block, height);
1148         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1149         check_added_monitors(&nodes[0], 1);
1150 }
1151
1152 #[test]
1153 fn test_dup_htlc_onchain_fails_on_reload() {
1154         do_test_dup_htlc_onchain_fails_on_reload(true, true, true);
1155         do_test_dup_htlc_onchain_fails_on_reload(true, true, false);
1156         do_test_dup_htlc_onchain_fails_on_reload(true, false, false);
1157         do_test_dup_htlc_onchain_fails_on_reload(false, true, true);
1158         do_test_dup_htlc_onchain_fails_on_reload(false, true, false);
1159         do_test_dup_htlc_onchain_fails_on_reload(false, false, false);
1160 }
1161
1162 #[test]
1163 fn test_fulfill_restart_failure() {
1164         // When we receive an update_fulfill_htlc message, we immediately consider the HTLC fully
1165         // fulfilled. At this point, the peer can reconnect and decide to either fulfill the HTLC
1166         // again, or fail it, giving us free money.
1167         //
1168         // Of course probably they won't fail it and give us free money, but because we have code to
1169         // handle it, we should test the logic for it anyway. We do that here.
1170         let chanmon_cfgs = create_chanmon_cfgs(2);
1171         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1172         let persister;
1173         let new_chain_monitor;
1174         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1175         let nodes_1_deserialized;
1176         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1177
1178         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1179         let (payment_preimage, payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 100_000);
1180
1181         // The simplest way to get a failure after a fulfill is to reload nodes[1] from a state
1182         // pre-fulfill, which we do by serializing it here.
1183         let chan_manager_serialized = nodes[1].node.encode();
1184         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id).encode();
1185
1186         nodes[1].node.claim_funds(payment_preimage);
1187         check_added_monitors!(nodes[1], 1);
1188         expect_payment_claimed!(nodes[1], payment_hash, 100_000);
1189
1190         let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1191         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
1192         expect_payment_sent(&nodes[0], payment_preimage, None, false, false);
1193
1194         // Now reload nodes[1]...
1195         reload_node!(nodes[1], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1196
1197         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1198         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1199
1200         nodes[1].node.fail_htlc_backwards(&payment_hash);
1201         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]);
1202         check_added_monitors!(nodes[1], 1);
1203         let htlc_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1204         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates.update_fail_htlcs[0]);
1205         commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates.commitment_signed, false);
1206         // nodes[0] shouldn't generate any events here, while it just got a payment failure completion
1207         // it had already considered the payment fulfilled, and now they just got free money.
1208         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1209 }
1210
1211 #[test]
1212 fn get_ldk_payment_preimage() {
1213         // Ensure that `ChannelManager::get_payment_preimage` can successfully be used to claim a payment.
1214         let chanmon_cfgs = create_chanmon_cfgs(2);
1215         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1216         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1217         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1218         create_announced_chan_between_nodes(&nodes, 0, 1);
1219
1220         let amt_msat = 60_000;
1221         let expiry_secs = 60 * 60;
1222         let (payment_hash, payment_secret) = nodes[1].node.create_inbound_payment(Some(amt_msat), expiry_secs, None).unwrap();
1223
1224         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
1225                 .with_bolt11_features(nodes[1].node.bolt11_invoice_features()).unwrap();
1226         let scorer = test_utils::TestScorer::new();
1227         let keys_manager = test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet);
1228         let random_seed_bytes = keys_manager.get_secure_random_bytes();
1229         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
1230         let route = get_route( &nodes[0].node.get_our_node_id(), &route_params,
1231                 &nodes[0].network_graph.read_only(),
1232                 Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()), nodes[0].logger,
1233                 &scorer, &Default::default(), &random_seed_bytes).unwrap();
1234         nodes[0].node.send_payment_with_route(&route, payment_hash,
1235                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
1236         check_added_monitors!(nodes[0], 1);
1237
1238         // Make sure to use `get_payment_preimage`
1239         let payment_preimage = nodes[1].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1240         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1241         assert_eq!(events.len(), 1);
1242         pass_along_path(&nodes[0], &[&nodes[1]], amt_msat, payment_hash, Some(payment_secret), events.pop().unwrap(), true, Some(payment_preimage));
1243         claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, payment_preimage);
1244 }
1245
1246 #[test]
1247 fn sent_probe_is_probe_of_sending_node() {
1248         let chanmon_cfgs = create_chanmon_cfgs(3);
1249         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1250         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1251         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1252
1253         create_announced_chan_between_nodes(&nodes, 0, 1);
1254         create_announced_chan_between_nodes(&nodes, 1, 2);
1255
1256         // First check we refuse to build a single-hop probe
1257         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[1], 100_000);
1258         assert!(nodes[0].node.send_probe(route.paths[0].clone()).is_err());
1259
1260         // Then build an actual two-hop probing path
1261         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], 100_000);
1262
1263         match nodes[0].node.send_probe(route.paths[0].clone()) {
1264                 Ok((payment_hash, payment_id)) => {
1265                         assert!(nodes[0].node.payment_is_probe(&payment_hash, &payment_id));
1266                         assert!(!nodes[1].node.payment_is_probe(&payment_hash, &payment_id));
1267                         assert!(!nodes[2].node.payment_is_probe(&payment_hash, &payment_id));
1268                 },
1269                 _ => panic!(),
1270         }
1271
1272         get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1273         check_added_monitors!(nodes[0], 1);
1274 }
1275
1276 #[test]
1277 fn successful_probe_yields_event() {
1278         let chanmon_cfgs = create_chanmon_cfgs(3);
1279         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1280         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1281         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1282
1283         create_announced_chan_between_nodes(&nodes, 0, 1);
1284         create_announced_chan_between_nodes(&nodes, 1, 2);
1285
1286         let recv_value = 100_000;
1287         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], recv_value);
1288
1289         let res = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1290
1291         let expected_route: &[&[&Node]] = &[&[&nodes[1], &nodes[2]]];
1292
1293         send_probe_along_route(&nodes[0], expected_route);
1294
1295         expect_probe_successful_events(&nodes[0], vec![res]);
1296
1297         assert!(!nodes[0].node.has_pending_payments());
1298 }
1299
1300 #[test]
1301 fn failed_probe_yields_event() {
1302         let chanmon_cfgs = create_chanmon_cfgs(3);
1303         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1304         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None, None]);
1305         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1306
1307         create_announced_chan_between_nodes(&nodes, 0, 1);
1308         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 100000, 90000000);
1309
1310         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1311
1312         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], payment_params, 9_998_000);
1313
1314         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1315
1316         // node[0] -- update_add_htlcs -> node[1]
1317         check_added_monitors!(nodes[0], 1);
1318         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1319         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1320         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1321         check_added_monitors!(nodes[1], 0);
1322         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1323         expect_pending_htlcs_forwardable!(nodes[1]);
1324
1325         // node[0] <- update_fail_htlcs -- node[1]
1326         check_added_monitors!(nodes[1], 1);
1327         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1328         // Skip the PendingHTLCsForwardable event
1329         let _events = nodes[1].node.get_and_clear_pending_events();
1330         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
1331         check_added_monitors!(nodes[0], 0);
1332         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1333
1334         let mut events = nodes[0].node.get_and_clear_pending_events();
1335         assert_eq!(events.len(), 1);
1336         match events.drain(..).next().unwrap() {
1337                 crate::events::Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1338                         assert_eq!(payment_id, ev_pid);
1339                         assert_eq!(payment_hash, ev_ph);
1340                 },
1341                 _ => panic!(),
1342         };
1343         assert!(!nodes[0].node.has_pending_payments());
1344 }
1345
1346 #[test]
1347 fn onchain_failed_probe_yields_event() {
1348         // Tests that an attempt to probe over a channel that is eventaully closed results in a failure
1349         // event.
1350         let chanmon_cfgs = create_chanmon_cfgs(3);
1351         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1352         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1353         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1354
1355         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1356         create_announced_chan_between_nodes(&nodes, 1, 2);
1357
1358         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), 42);
1359
1360         // Send a dust HTLC, which will be treated as if it timed out once the channel hits the chain.
1361         let (route, _, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[2], payment_params, 1_000);
1362         let (payment_hash, payment_id) = nodes[0].node.send_probe(route.paths[0].clone()).unwrap();
1363
1364         // node[0] -- update_add_htlcs -> node[1]
1365         check_added_monitors!(nodes[0], 1);
1366         let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1367         let probe_event = SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), updates);
1368         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &probe_event.msgs[0]);
1369         check_added_monitors!(nodes[1], 0);
1370         commitment_signed_dance!(nodes[1], nodes[0], probe_event.commitment_msg, false);
1371         expect_pending_htlcs_forwardable!(nodes[1]);
1372
1373         check_added_monitors!(nodes[1], 1);
1374         let _ = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1375
1376         // Don't bother forwarding the HTLC onwards and just confirm the force-close transaction on
1377         // Node A, which after 6 confirmations should result in a probe failure event.
1378         let bs_txn = get_local_commitment_txn!(nodes[1], chan_id);
1379         confirm_transaction(&nodes[0], &bs_txn[0]);
1380         check_closed_broadcast!(&nodes[0], true);
1381         check_added_monitors!(nodes[0], 1);
1382
1383         let mut events = nodes[0].node.get_and_clear_pending_events();
1384         assert_eq!(events.len(), 2);
1385         let mut found_probe_failed = false;
1386         for event in events.drain(..) {
1387                 match event {
1388                         Event::ProbeFailed { payment_id: ev_pid, payment_hash: ev_ph, .. } => {
1389                                 assert_eq!(payment_id, ev_pid);
1390                                 assert_eq!(payment_hash, ev_ph);
1391                                 found_probe_failed = true;
1392                         },
1393                         Event::ChannelClosed { .. } => {},
1394                         _ => panic!(),
1395                 }
1396         }
1397         assert!(found_probe_failed);
1398         assert!(!nodes[0].node.has_pending_payments());
1399 }
1400
1401 #[test]
1402 fn preflight_probes_yield_event_skip_private_hop() {
1403         let chanmon_cfgs = create_chanmon_cfgs(5);
1404         let node_cfgs = create_node_cfgs(5, &chanmon_cfgs);
1405
1406         // We alleviate the HTLC max-in-flight limit, as otherwise we'd always be limited through that.
1407         let mut no_htlc_limit_config = test_default_channel_config();
1408         no_htlc_limit_config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 100;
1409
1410         let user_configs = std::iter::repeat(no_htlc_limit_config).take(5).map(|c| Some(c)).collect::<Vec<Option<UserConfig>>>();
1411         let node_chanmgrs = create_node_chanmgrs(5, &node_cfgs, &user_configs);
1412         let nodes = create_network(5, &node_cfgs, &node_chanmgrs);
1413
1414         // Setup channel topology:
1415         //            N0 -(1M:0)- N1 -(1M:0)- N2 -(70k:0)- N3 -(50k:0)- N4
1416
1417         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
1418         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 1_000_000, 0);
1419         create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 70_000, 0);
1420         create_unannounced_chan_between_nodes_with_value(&nodes, 3, 4, 50_000, 0);
1421
1422         let mut invoice_features = Bolt11InvoiceFeatures::empty();
1423         invoice_features.set_basic_mpp_optional();
1424
1425         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
1426                 .with_bolt11_features(invoice_features).unwrap();
1427
1428         let recv_value = 50_000_000;
1429         let route_params = RouteParameters::from_payment_params_and_value(payment_params, recv_value);
1430         let res = nodes[0].node.send_preflight_probes(route_params, None).unwrap();
1431
1432         let expected_route: &[&[&Node]] = &[&[&nodes[1], &nodes[2], &nodes[3]]];
1433
1434         assert_eq!(res.len(), expected_route.len());
1435
1436         send_probe_along_route(&nodes[0], expected_route);
1437
1438         expect_probe_successful_events(&nodes[0], res.clone());
1439
1440         assert!(!nodes[0].node.has_pending_payments());
1441 }
1442
1443 #[test]
1444 fn preflight_probes_yield_event() {
1445         let chanmon_cfgs = create_chanmon_cfgs(4);
1446         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
1447
1448         // We alleviate the HTLC max-in-flight limit, as otherwise we'd always be limited through that.
1449         let mut no_htlc_limit_config = test_default_channel_config();
1450         no_htlc_limit_config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 100;
1451
1452         let user_configs = std::iter::repeat(no_htlc_limit_config).take(4).map(|c| Some(c)).collect::<Vec<Option<UserConfig>>>();
1453         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &user_configs);
1454         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
1455
1456         // Setup channel topology:
1457         //                    (1M:0)- N1 -(30k:0)
1458         //                   /                  \
1459         //                 N0                    N4
1460         //                   \                  /
1461         //                    (1M:0)- N2 -(70k:0)
1462         //
1463         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
1464         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
1465         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 30_000, 0);
1466         create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 70_000, 0);
1467
1468         let mut invoice_features = Bolt11InvoiceFeatures::empty();
1469         invoice_features.set_basic_mpp_optional();
1470
1471         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
1472                 .with_bolt11_features(invoice_features).unwrap();
1473
1474         let recv_value = 50_000_000;
1475         let route_params = RouteParameters::from_payment_params_and_value(payment_params, recv_value);
1476         let res = nodes[0].node.send_preflight_probes(route_params, None).unwrap();
1477
1478         let expected_route: &[&[&Node]] = &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]];
1479
1480         assert_eq!(res.len(), expected_route.len());
1481
1482         send_probe_along_route(&nodes[0], expected_route);
1483
1484         expect_probe_successful_events(&nodes[0], res.clone());
1485
1486         assert!(!nodes[0].node.has_pending_payments());
1487 }
1488
1489 #[test]
1490 fn preflight_probes_yield_event_and_skip() {
1491         let chanmon_cfgs = create_chanmon_cfgs(5);
1492         let node_cfgs = create_node_cfgs(5, &chanmon_cfgs);
1493
1494         // We alleviate the HTLC max-in-flight limit, as otherwise we'd always be limited through that.
1495         let mut no_htlc_limit_config = test_default_channel_config();
1496         no_htlc_limit_config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 100;
1497
1498         let user_configs = std::iter::repeat(no_htlc_limit_config).take(5).map(|c| Some(c)).collect::<Vec<Option<UserConfig>>>();
1499         let node_chanmgrs = create_node_chanmgrs(5, &node_cfgs, &user_configs);
1500         let nodes = create_network(5, &node_cfgs, &node_chanmgrs);
1501
1502         // Setup channel topology:
1503         //                    (30k:0)- N2 -(1M:0)
1504         //                   /                  \
1505         //  N0 -(100k:0)-> N1                    N4
1506         //                   \                  /
1507         //                    (70k:0)- N3 -(1M:0)
1508         //
1509         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
1510         create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 30_000, 0);
1511         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 70_000, 0);
1512         create_announced_chan_between_nodes_with_value(&nodes, 2, 4, 1_000_000, 0);
1513         create_announced_chan_between_nodes_with_value(&nodes, 3, 4, 1_000_000, 0);
1514
1515         let mut invoice_features = Bolt11InvoiceFeatures::empty();
1516         invoice_features.set_basic_mpp_optional();
1517
1518         let payment_params = PaymentParameters::from_node_id(nodes[4].node.get_our_node_id(), TEST_FINAL_CLTV)
1519                 .with_bolt11_features(invoice_features).unwrap();
1520
1521         let recv_value = 80_000_000;
1522         let route_params = RouteParameters::from_payment_params_and_value(payment_params, recv_value);
1523         let res = nodes[0].node.send_preflight_probes(route_params, None).unwrap();
1524
1525         let expected_route : &[&[&Node]] = &[&[&nodes[1], &nodes[2], &nodes[4]]];
1526
1527         // We check that only one probe was sent, the other one was skipped due to limited liquidity.
1528         assert_eq!(res.len(), 1);
1529
1530         send_probe_along_route(&nodes[0], expected_route);
1531
1532         expect_probe_successful_events(&nodes[0], res.clone());
1533
1534         assert!(!nodes[0].node.has_pending_payments());
1535 }
1536
1537 #[test]
1538 fn claimed_send_payment_idempotent() {
1539         // Tests that `send_payment` (and friends) are (reasonably) idempotent.
1540         let chanmon_cfgs = create_chanmon_cfgs(2);
1541         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1542         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1543         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1544
1545         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1546
1547         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1548         let (first_payment_preimage, _, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1549
1550         macro_rules! check_send_rejected {
1551                 () => {
1552                         // If we try to resend a new payment with a different payment_hash but with the same
1553                         // payment_id, it should be rejected.
1554                         let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1555                                 RecipientOnionFields::secret_only(second_payment_secret), payment_id);
1556                         match send_result {
1557                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1558                                 _ => panic!("Unexpected send result: {:?}", send_result),
1559                         }
1560
1561                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1562                         // also be rejected.
1563                         let send_result = nodes[0].node.send_spontaneous_payment(
1564                                 &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
1565                         match send_result {
1566                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1567                                 _ => panic!("Unexpected send result: {:?}", send_result),
1568                         }
1569                 }
1570         }
1571
1572         check_send_rejected!();
1573
1574         // Claim the payment backwards, but note that the PaymentSent event is still pending and has
1575         // not been seen by the user. At this point, from the user perspective nothing has changed, so
1576         // we must remain just as idempotent as we were before.
1577         do_claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, first_payment_preimage);
1578
1579         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1580                 nodes[0].node.timer_tick_occurred();
1581         }
1582
1583         check_send_rejected!();
1584
1585         // Once the user sees and handles the `PaymentSent` event, we expect them to no longer call
1586         // `send_payment`, and our idempotency guarantees are off - they should have atomically marked
1587         // the payment complete. However, they could have called `send_payment` while the event was
1588         // being processed, leading to a race in our idempotency guarantees. Thus, even immediately
1589         // after the event is handled a duplicate payment should sitll be rejected.
1590         expect_payment_sent!(&nodes[0], first_payment_preimage, Some(0));
1591         check_send_rejected!();
1592
1593         // If relatively little time has passed, a duplicate payment should still fail.
1594         nodes[0].node.timer_tick_occurred();
1595         check_send_rejected!();
1596
1597         // However, after some time has passed (at least more than the one timer tick above), a
1598         // duplicate payment should go through, as ChannelManager should no longer have any remaining
1599         // references to the old payment data.
1600         for _ in 0..IDEMPOTENCY_TIMEOUT_TICKS {
1601                 nodes[0].node.timer_tick_occurred();
1602         }
1603
1604         nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1605                 RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
1606         check_added_monitors!(nodes[0], 1);
1607         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1608         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1609 }
1610
1611 #[test]
1612 fn abandoned_send_payment_idempotent() {
1613         // Tests that `send_payment` (and friends) allow duplicate PaymentIds immediately after
1614         // abandon_payment.
1615         let chanmon_cfgs = create_chanmon_cfgs(2);
1616         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1617         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1618         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1619
1620         create_announced_chan_between_nodes(&nodes, 0, 1).2;
1621
1622         let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
1623         let (_, first_payment_hash, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
1624
1625         macro_rules! check_send_rejected {
1626                 () => {
1627                         // If we try to resend a new payment with a different payment_hash but with the same
1628                         // payment_id, it should be rejected.
1629                         let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1630                                 RecipientOnionFields::secret_only(second_payment_secret), payment_id);
1631                         match send_result {
1632                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1633                                 _ => panic!("Unexpected send result: {:?}", send_result),
1634                         }
1635
1636                         // Further, if we try to send a spontaneous payment with the same payment_id it should
1637                         // also be rejected.
1638                         let send_result = nodes[0].node.send_spontaneous_payment(
1639                                 &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
1640                         match send_result {
1641                                 Err(PaymentSendFailure::DuplicatePayment) => {},
1642                                 _ => panic!("Unexpected send result: {:?}", send_result),
1643                         }
1644                 }
1645         }
1646
1647         check_send_rejected!();
1648
1649         nodes[1].node.fail_htlc_backwards(&first_payment_hash);
1650         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash: first_payment_hash }]);
1651
1652         // Until we abandon the payment upon path failure, no matter how many timer ticks pass, we still cannot reuse the
1653         // PaymentId.
1654         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1655                 nodes[0].node.timer_tick_occurred();
1656         }
1657         check_send_rejected!();
1658
1659         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash, PaymentFailureReason::RecipientRejected);
1660
1661         // However, we can reuse the PaymentId immediately after we `abandon_payment` upon passing the
1662         // failed payment back.
1663         nodes[0].node.send_payment_with_route(&route, second_payment_hash,
1664                 RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
1665         check_added_monitors!(nodes[0], 1);
1666         pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
1667         claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
1668 }
1669
1670 #[derive(PartialEq)]
1671 enum InterceptTest {
1672         Forward,
1673         Fail,
1674         Timeout,
1675 }
1676
1677 #[test]
1678 fn test_trivial_inflight_htlc_tracking(){
1679         // In this test, we test three scenarios:
1680         // (1) Sending + claiming a payment successfully should return `None` when querying InFlightHtlcs
1681         // (2) Sending a payment without claiming it should return the payment's value (500000) when querying InFlightHtlcs
1682         // (3) After we claim the payment sent in (2), InFlightHtlcs should return `None` for the query.
1683         let chanmon_cfgs = create_chanmon_cfgs(3);
1684         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1685         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1686         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1687
1688         let (_, _, chan_1_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1);
1689         let (_, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
1690
1691         // Send and claim the payment. Inflight HTLCs should be empty.
1692         let (_, payment_hash, _, payment_id) = send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 500000);
1693         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1694         {
1695                 let mut node_0_per_peer_lock;
1696                 let mut node_0_peer_state_lock;
1697                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1698
1699                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1700                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1701                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1702                         channel_1.context().get_short_channel_id().unwrap()
1703                 );
1704                 assert_eq!(chan_1_used_liquidity, None);
1705         }
1706         {
1707                 let mut node_1_per_peer_lock;
1708                 let mut node_1_peer_state_lock;
1709                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1710
1711                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1712                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1713                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1714                         channel_2.context().get_short_channel_id().unwrap()
1715                 );
1716
1717                 assert_eq!(chan_2_used_liquidity, None);
1718         }
1719         let pending_payments = nodes[0].node.list_recent_payments();
1720         assert_eq!(pending_payments.len(), 1);
1721         assert_eq!(pending_payments[0], RecentPaymentDetails::Fulfilled { payment_hash: Some(payment_hash), payment_id });
1722
1723         // Remove fulfilled payment
1724         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1725                 nodes[0].node.timer_tick_occurred();
1726         }
1727
1728         // Send the payment, but do not claim it. Our inflight HTLCs should contain the pending payment.
1729         let (payment_preimage, payment_hash,  _, payment_id) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 500000);
1730         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1731         {
1732                 let mut node_0_per_peer_lock;
1733                 let mut node_0_peer_state_lock;
1734                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1735
1736                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1737                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1738                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1739                         channel_1.context().get_short_channel_id().unwrap()
1740                 );
1741                 // First hop accounts for expected 1000 msat fee
1742                 assert_eq!(chan_1_used_liquidity, Some(501000));
1743         }
1744         {
1745                 let mut node_1_per_peer_lock;
1746                 let mut node_1_peer_state_lock;
1747                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1748
1749                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1750                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1751                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1752                         channel_2.context().get_short_channel_id().unwrap()
1753                 );
1754
1755                 assert_eq!(chan_2_used_liquidity, Some(500000));
1756         }
1757         let pending_payments = nodes[0].node.list_recent_payments();
1758         assert_eq!(pending_payments.len(), 1);
1759         assert_eq!(pending_payments[0], RecentPaymentDetails::Pending { payment_id, payment_hash, total_msat: 500000 });
1760
1761         // Now, let's claim the payment. This should result in the used liquidity to return `None`.
1762         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
1763
1764         // Remove fulfilled payment
1765         for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
1766                 nodes[0].node.timer_tick_occurred();
1767         }
1768
1769         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1770         {
1771                 let mut node_0_per_peer_lock;
1772                 let mut node_0_peer_state_lock;
1773                 let channel_1 =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, chan_1_id);
1774
1775                 let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
1776                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1777                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1778                         channel_1.context().get_short_channel_id().unwrap()
1779                 );
1780                 assert_eq!(chan_1_used_liquidity, None);
1781         }
1782         {
1783                 let mut node_1_per_peer_lock;
1784                 let mut node_1_peer_state_lock;
1785                 let channel_2 =  get_channel_ref!(&nodes[1], nodes[2], node_1_per_peer_lock, node_1_peer_state_lock, chan_2_id);
1786
1787                 let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
1788                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
1789                         &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
1790                         channel_2.context().get_short_channel_id().unwrap()
1791                 );
1792                 assert_eq!(chan_2_used_liquidity, None);
1793         }
1794
1795         let pending_payments = nodes[0].node.list_recent_payments();
1796         assert_eq!(pending_payments.len(), 0);
1797 }
1798
1799 #[test]
1800 fn test_holding_cell_inflight_htlcs() {
1801         let chanmon_cfgs = create_chanmon_cfgs(2);
1802         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1803         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1804         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1805         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1806
1807         let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
1808         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1809
1810         // Queue up two payments - one will be delivered right away, one immediately goes into the
1811         // holding cell as nodes[0] is AwaitingRAA.
1812         {
1813                 nodes[0].node.send_payment_with_route(&route, payment_hash_1,
1814                         RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
1815                 check_added_monitors!(nodes[0], 1);
1816                 nodes[0].node.send_payment_with_route(&route, payment_hash_2,
1817                         RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
1818                 check_added_monitors!(nodes[0], 0);
1819         }
1820
1821         let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
1822
1823         {
1824                 let mut node_0_per_peer_lock;
1825                 let mut node_0_peer_state_lock;
1826                 let channel =  get_channel_ref!(&nodes[0], nodes[1], node_0_per_peer_lock, node_0_peer_state_lock, channel_id);
1827
1828                 let used_liquidity = inflight_htlcs.used_liquidity_msat(
1829                         &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
1830                         &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
1831                         channel.context().get_short_channel_id().unwrap()
1832                 );
1833
1834                 assert_eq!(used_liquidity, Some(2000000));
1835         }
1836
1837         // Clear pending events so test doesn't throw a "Had excess message on node..." error
1838         nodes[0].node.get_and_clear_pending_msg_events();
1839 }
1840
1841 #[test]
1842 fn intercepted_payment() {
1843         // Test that detecting an intercept scid on payment forward will signal LDK to generate an
1844         // intercept event, which the LSP can then use to either (a) open a JIT channel to forward the
1845         // payment or (b) fail the payment.
1846         do_test_intercepted_payment(InterceptTest::Forward);
1847         do_test_intercepted_payment(InterceptTest::Fail);
1848         // Make sure that intercepted payments will be automatically failed back if too many blocks pass.
1849         do_test_intercepted_payment(InterceptTest::Timeout);
1850 }
1851
1852 fn do_test_intercepted_payment(test: InterceptTest) {
1853         let chanmon_cfgs = create_chanmon_cfgs(3);
1854         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1855
1856         let mut zero_conf_chan_config = test_default_channel_config();
1857         zero_conf_chan_config.manually_accept_inbound_channels = true;
1858         let mut intercept_forwards_config = test_default_channel_config();
1859         intercept_forwards_config.accept_intercept_htlcs = true;
1860         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(zero_conf_chan_config)]);
1861
1862         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1863         let scorer = test_utils::TestScorer::new();
1864         let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
1865
1866         let _ = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1867
1868         let amt_msat = 100_000;
1869         let intercept_scid = nodes[1].node.get_intercept_scid();
1870         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
1871                 .with_route_hints(vec![
1872                         RouteHint(vec![RouteHintHop {
1873                                 src_node_id: nodes[1].node.get_our_node_id(),
1874                                 short_channel_id: intercept_scid,
1875                                 fees: RoutingFees {
1876                                         base_msat: 1000,
1877                                         proportional_millionths: 0,
1878                                 },
1879                                 cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
1880                                 htlc_minimum_msat: None,
1881                                 htlc_maximum_msat: None,
1882                         }])
1883                 ]).unwrap()
1884                 .with_bolt11_features(nodes[2].node.bolt11_invoice_features()).unwrap();
1885         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
1886         let route = get_route(
1887                 &nodes[0].node.get_our_node_id(), &route_params, &nodes[0].network_graph.read_only(), None,
1888                 nodes[0].logger, &scorer, &Default::default(), &random_seed_bytes
1889         ).unwrap();
1890
1891         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
1892         nodes[0].node.send_payment_with_route(&route, payment_hash,
1893                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
1894         let payment_event = {
1895                 {
1896                         let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
1897                         assert_eq!(added_monitors.len(), 1);
1898                         added_monitors.clear();
1899                 }
1900                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1901                 assert_eq!(events.len(), 1);
1902                 SendEvent::from_event(events.remove(0))
1903         };
1904         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1905         commitment_signed_dance!(nodes[1], nodes[0], &payment_event.commitment_msg, false, true);
1906
1907         // Check that we generate the PaymentIntercepted event when an intercept forward is detected.
1908         let events = nodes[1].node.get_and_clear_pending_events();
1909         assert_eq!(events.len(), 1);
1910         let (intercept_id, expected_outbound_amount_msat) = match events[0] {
1911                 crate::events::Event::HTLCIntercepted {
1912                         intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, inbound_amount_msat, requested_next_hop_scid: short_channel_id
1913                 } => {
1914                         assert_eq!(pmt_hash, payment_hash);
1915                         assert_eq!(inbound_amount_msat, route.get_total_amount() + route.get_total_fees());
1916                         assert_eq!(short_channel_id, intercept_scid);
1917                         (intercept_id, expected_outbound_amount_msat)
1918                 },
1919                 _ => panic!()
1920         };
1921
1922         // Check for unknown channel id error.
1923         let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &ChannelId::from_bytes([42; 32]), nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1924         assert_eq!(unknown_chan_id_err , APIError::ChannelUnavailable  {
1925                 err: format!("Channel with id {} not found for the passed counterparty node_id {}",
1926                         log_bytes!([42; 32]), nodes[2].node.get_our_node_id()) });
1927
1928         if test == InterceptTest::Fail {
1929                 // Ensure we can fail the intercepted payment back.
1930                 nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap();
1931                 expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::UnknownNextHop { requested_forward_scid: intercept_scid }]);
1932                 nodes[1].node.process_pending_htlc_forwards();
1933                 let update_fail = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1934                 check_added_monitors!(&nodes[1], 1);
1935                 assert!(update_fail.update_fail_htlcs.len() == 1);
1936                 let fail_msg = update_fail.update_fail_htlcs[0].clone();
1937                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
1938                 commitment_signed_dance!(nodes[0], nodes[1], update_fail.commitment_signed, false);
1939
1940                 // Ensure the payment fails with the expected error.
1941                 let fail_conditions = PaymentFailedConditions::new()
1942                         .blamed_scid(intercept_scid)
1943                         .blamed_chan_closed(true)
1944                         .expected_htlc_error_data(0x4000 | 10, &[]);
1945                 expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
1946         } else if test == InterceptTest::Forward {
1947                 // Check that we'll fail as expected when sending to a channel that isn't in `ChannelReady` yet.
1948                 let temp_chan_id = nodes[1].node.create_channel(nodes[2].node.get_our_node_id(), 100_000, 0, 42, None, None).unwrap();
1949                 let unusable_chan_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &temp_chan_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
1950                 assert_eq!(unusable_chan_err , APIError::ChannelUnavailable {
1951                         err: format!("Channel with id {} for the passed counterparty node_id {} is still opening.",
1952                                 temp_chan_id, nodes[2].node.get_our_node_id()) });
1953                 assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
1954
1955                 // Open the just-in-time channel so the payment can then be forwarded.
1956                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
1957
1958                 // Finally, forward the intercepted payment through and claim it.
1959                 nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap();
1960                 expect_pending_htlcs_forwardable!(nodes[1]);
1961
1962                 let payment_event = {
1963                         {
1964                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
1965                                 assert_eq!(added_monitors.len(), 1);
1966                                 added_monitors.clear();
1967                         }
1968                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
1969                         assert_eq!(events.len(), 1);
1970                         SendEvent::from_event(events.remove(0))
1971                 };
1972                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1973                 commitment_signed_dance!(nodes[2], nodes[1], &payment_event.commitment_msg, false, true);
1974                 expect_pending_htlcs_forwardable!(nodes[2]);
1975
1976                 let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
1977                 expect_payment_claimable!(&nodes[2], payment_hash, payment_secret, amt_msat, Some(payment_preimage), nodes[2].node.get_our_node_id());
1978                 do_claim_payment_along_route(&nodes[0], &vec!(&vec!(&nodes[1], &nodes[2])[..]), false, payment_preimage);
1979                 let events = nodes[0].node.get_and_clear_pending_events();
1980                 assert_eq!(events.len(), 2);
1981                 match events[0] {
1982                         Event::PaymentSent { payment_preimage: ref ev_preimage, payment_hash: ref ev_hash, ref fee_paid_msat, .. } => {
1983                                 assert_eq!(payment_preimage, *ev_preimage);
1984                                 assert_eq!(payment_hash, *ev_hash);
1985                                 assert_eq!(fee_paid_msat, &Some(1000));
1986                         },
1987                         _ => panic!("Unexpected event")
1988                 }
1989                 match events[1] {
1990                         Event::PaymentPathSuccessful { payment_hash: hash, .. } => {
1991                                 assert_eq!(hash, Some(payment_hash));
1992                         },
1993                         _ => panic!("Unexpected event")
1994                 }
1995                 check_added_monitors(&nodes[0], 1);
1996         } else if test == InterceptTest::Timeout {
1997                 let mut block = create_dummy_block(nodes[0].best_block_hash(), 42, Vec::new());
1998                 connect_block(&nodes[0], &block);
1999                 connect_block(&nodes[1], &block);
2000                 for _ in 0..TEST_FINAL_CLTV {
2001                         block.header.prev_blockhash = block.block_hash();
2002                         connect_block(&nodes[0], &block);
2003                         connect_block(&nodes[1], &block);
2004                 }
2005                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::InvalidForward { requested_forward_scid: intercept_scid }]);
2006                 check_added_monitors!(nodes[1], 1);
2007                 let htlc_timeout_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2008                 assert!(htlc_timeout_updates.update_add_htlcs.is_empty());
2009                 assert_eq!(htlc_timeout_updates.update_fail_htlcs.len(), 1);
2010                 assert!(htlc_timeout_updates.update_fail_malformed_htlcs.is_empty());
2011                 assert!(htlc_timeout_updates.update_fee.is_none());
2012
2013                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_timeout_updates.update_fail_htlcs[0]);
2014                 commitment_signed_dance!(nodes[0], nodes[1], htlc_timeout_updates.commitment_signed, false);
2015                 expect_payment_failed!(nodes[0], payment_hash, false, 0x2000 | 2, []);
2016
2017                 // Check for unknown intercept id error.
2018                 let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
2019                 let unknown_intercept_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
2020                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
2021                 let unknown_intercept_id_err = nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap_err();
2022                 assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
2023         }
2024 }
2025
2026 #[test]
2027 fn accept_underpaying_htlcs_config() {
2028         do_accept_underpaying_htlcs_config(1);
2029         do_accept_underpaying_htlcs_config(2);
2030         do_accept_underpaying_htlcs_config(3);
2031 }
2032
2033 fn do_accept_underpaying_htlcs_config(num_mpp_parts: usize) {
2034         let chanmon_cfgs = create_chanmon_cfgs(3);
2035         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2036         let mut intercept_forwards_config = test_default_channel_config();
2037         intercept_forwards_config.accept_intercept_htlcs = true;
2038         let mut underpay_config = test_default_channel_config();
2039         underpay_config.channel_config.accept_underpaying_htlcs = true;
2040         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(underpay_config)]);
2041         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2042
2043         let mut chan_ids = Vec::new();
2044         for _ in 0..num_mpp_parts {
2045                 let _ = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000, 0);
2046                 let channel_id = create_unannounced_chan_between_nodes_with_value(&nodes, 1, 2, 2_000_000, 0).0.channel_id;
2047                 chan_ids.push(channel_id);
2048         }
2049
2050         // Send the initial payment.
2051         let amt_msat = 900_000;
2052         let skimmed_fee_msat = 20;
2053         let mut route_hints = Vec::new();
2054         for _ in 0..num_mpp_parts {
2055                 route_hints.push(RouteHint(vec![RouteHintHop {
2056                         src_node_id: nodes[1].node.get_our_node_id(),
2057                         short_channel_id: nodes[1].node.get_intercept_scid(),
2058                         fees: RoutingFees {
2059                                 base_msat: 1000,
2060                                 proportional_millionths: 0,
2061                         },
2062                         cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
2063                         htlc_minimum_msat: None,
2064                         htlc_maximum_msat: Some(amt_msat / num_mpp_parts as u64 + 5),
2065                 }]));
2066         }
2067         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
2068                 .with_route_hints(route_hints).unwrap()
2069                 .with_bolt11_features(nodes[2].node.bolt11_invoice_features()).unwrap();
2070         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2071         let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
2072         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2073                 PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
2074         check_added_monitors!(nodes[0], num_mpp_parts); // one monitor per path
2075         let mut events: Vec<SendEvent> = nodes[0].node.get_and_clear_pending_msg_events().into_iter().map(|e| SendEvent::from_event(e)).collect();
2076         assert_eq!(events.len(), num_mpp_parts);
2077
2078         // Forward the intercepted payments.
2079         for (idx, ev) in events.into_iter().enumerate() {
2080                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &ev.msgs[0]);
2081                 do_commitment_signed_dance(&nodes[1], &nodes[0], &ev.commitment_msg, false, true);
2082
2083                 let events = nodes[1].node.get_and_clear_pending_events();
2084                 assert_eq!(events.len(), 1);
2085                 let (intercept_id, expected_outbound_amt_msat) = match events[0] {
2086                         crate::events::Event::HTLCIntercepted {
2087                                 intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, ..
2088                         } => {
2089                                 assert_eq!(pmt_hash, payment_hash);
2090                                 (intercept_id, expected_outbound_amount_msat)
2091                         },
2092                         _ => panic!()
2093                 };
2094                 nodes[1].node.forward_intercepted_htlc(intercept_id, &chan_ids[idx],
2095                         nodes[2].node.get_our_node_id(), expected_outbound_amt_msat - skimmed_fee_msat).unwrap();
2096                 expect_pending_htlcs_forwardable!(nodes[1]);
2097                 let payment_event = {
2098                         {
2099                                 let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
2100                                 assert_eq!(added_monitors.len(), 1);
2101                                 added_monitors.clear();
2102                         }
2103                         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
2104                         assert_eq!(events.len(), 1);
2105                         SendEvent::from_event(events.remove(0))
2106                 };
2107                 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
2108                 do_commitment_signed_dance(&nodes[2], &nodes[1], &payment_event.commitment_msg, false, true);
2109                 if idx == num_mpp_parts - 1 {
2110                         expect_pending_htlcs_forwardable!(nodes[2]);
2111                 }
2112         }
2113
2114         // Claim the payment and check that the skimmed fee is as expected.
2115         let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
2116         let events = nodes[2].node.get_and_clear_pending_events();
2117         assert_eq!(events.len(), 1);
2118         match events[0] {
2119                 crate::events::Event::PaymentClaimable {
2120                         ref payment_hash, ref purpose, amount_msat, counterparty_skimmed_fee_msat, receiver_node_id, ..
2121                 } => {
2122                         assert_eq!(payment_hash, payment_hash);
2123                         assert_eq!(amt_msat - skimmed_fee_msat * num_mpp_parts as u64, amount_msat);
2124                         assert_eq!(skimmed_fee_msat * num_mpp_parts as u64, counterparty_skimmed_fee_msat);
2125                         assert_eq!(nodes[2].node.get_our_node_id(), receiver_node_id.unwrap());
2126                         match purpose {
2127                                 crate::events::PaymentPurpose::InvoicePayment { payment_preimage: ev_payment_preimage,
2128                                         payment_secret: ev_payment_secret, .. } =>
2129                                 {
2130                                         assert_eq!(payment_preimage, ev_payment_preimage.unwrap());
2131                                         assert_eq!(payment_secret, *ev_payment_secret);
2132                                 },
2133                                 _ => panic!(),
2134                         }
2135                 },
2136                 _ => panic!("Unexpected event"),
2137         }
2138         let mut expected_paths_vecs = Vec::new();
2139         let mut expected_paths = Vec::new();
2140         for _ in 0..num_mpp_parts { expected_paths_vecs.push(vec!(&nodes[1], &nodes[2])); }
2141         for i in 0..num_mpp_parts { expected_paths.push(&expected_paths_vecs[i][..]); }
2142         let total_fee_msat = do_claim_payment_along_route_with_extra_penultimate_hop_fees(
2143                 &nodes[0], &expected_paths[..], &vec![skimmed_fee_msat as u32; num_mpp_parts][..], false,
2144                 payment_preimage);
2145         // The sender doesn't know that the penultimate hop took an extra fee.
2146         expect_payment_sent(&nodes[0], payment_preimage,
2147                 Some(Some(total_fee_msat - skimmed_fee_msat * num_mpp_parts as u64)), true, true);
2148 }
2149
2150 #[derive(PartialEq)]
2151 enum AutoRetry {
2152         Success,
2153         Spontaneous,
2154         FailAttempts,
2155         FailTimeout,
2156         FailOnRestart,
2157         FailOnRetry,
2158 }
2159
2160 #[test]
2161 fn automatic_retries() {
2162         do_automatic_retries(AutoRetry::Success);
2163         do_automatic_retries(AutoRetry::Spontaneous);
2164         do_automatic_retries(AutoRetry::FailAttempts);
2165         do_automatic_retries(AutoRetry::FailTimeout);
2166         do_automatic_retries(AutoRetry::FailOnRestart);
2167         do_automatic_retries(AutoRetry::FailOnRetry);
2168 }
2169 fn do_automatic_retries(test: AutoRetry) {
2170         // Test basic automatic payment retries in ChannelManager. See individual `test` variant comments
2171         // below.
2172         let chanmon_cfgs = create_chanmon_cfgs(3);
2173         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2174         let persister;
2175         let new_chain_monitor;
2176
2177         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2178         let node_0_deserialized;
2179
2180         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2181         let channel_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
2182         let channel_id_2 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
2183
2184         // Marshall data to send the payment
2185         #[cfg(feature = "std")]
2186         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2187         #[cfg(not(feature = "std"))]
2188         let payment_expiry_secs = 60 * 60;
2189         let amt_msat = 1000;
2190         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2191         invoice_features.set_variable_length_onion_required();
2192         invoice_features.set_payment_secret_required();
2193         invoice_features.set_basic_mpp_optional();
2194         let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
2195                 .with_expiry_time(payment_expiry_secs as u64)
2196                 .with_bolt11_features(invoice_features).unwrap();
2197         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2198         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
2199
2200         macro_rules! pass_failed_attempt_with_retry_along_path {
2201                 ($failing_channel_id: expr, $expect_pending_htlcs_forwardable: expr) => {
2202                         // Send a payment attempt that fails due to lack of liquidity on the second hop
2203                         check_added_monitors!(nodes[0], 1);
2204                         let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2205                         let mut update_add = update_0.update_add_htlcs[0].clone();
2206                         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
2207                         commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
2208                         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
2209                         nodes[1].node.process_pending_htlc_forwards();
2210                         expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1],
2211                                 vec![HTLCDestination::NextHopChannel {
2212                                         node_id: Some(nodes[2].node.get_our_node_id()),
2213                                         channel_id: $failing_channel_id,
2214                                 }]);
2215                         nodes[1].node.process_pending_htlc_forwards();
2216                         let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2217                         check_added_monitors!(&nodes[1], 1);
2218                         assert!(update_1.update_fail_htlcs.len() == 1);
2219                         let fail_msg = update_1.update_fail_htlcs[0].clone();
2220                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
2221                         commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
2222
2223                         // Ensure the attempt fails and a new PendingHTLCsForwardable event is generated for the retry
2224                         let mut events = nodes[0].node.get_and_clear_pending_events();
2225                         assert_eq!(events.len(), 2);
2226                         match events[0] {
2227                                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
2228                                         assert_eq!(payment_hash, ev_payment_hash);
2229                                         assert_eq!(payment_failed_permanently, false);
2230                                 },
2231                                 _ => panic!("Unexpected event"),
2232                         }
2233                         if $expect_pending_htlcs_forwardable {
2234                                 match events[1] {
2235                                         Event::PendingHTLCsForwardable { .. } => {},
2236                                         _ => panic!("Unexpected event"),
2237                                 }
2238                         } else {
2239                                 match events[1] {
2240                                         Event::PaymentFailed { payment_hash: ev_payment_hash, .. } => {
2241                                                 assert_eq!(payment_hash, ev_payment_hash);
2242                                         },
2243                                         _ => panic!("Unexpected event"),
2244                                 }
2245                         }
2246                 }
2247         }
2248
2249         if test == AutoRetry::Success {
2250                 // Test that we can succeed on the first retry.
2251                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2252                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2253                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2254
2255                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
2256                 // attempt, since the initial second hop channel will be excluded from pathfinding
2257                 create_announced_chan_between_nodes(&nodes, 1, 2);
2258
2259                 // We retry payments in `process_pending_htlc_forwards`
2260                 nodes[0].node.process_pending_htlc_forwards();
2261                 check_added_monitors!(nodes[0], 1);
2262                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2263                 assert_eq!(msg_events.len(), 1);
2264                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, Some(payment_secret), msg_events.pop().unwrap(), true, None);
2265                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
2266         } else if test == AutoRetry::Spontaneous {
2267                 nodes[0].node.send_spontaneous_payment_with_retry(Some(payment_preimage),
2268                         RecipientOnionFields::spontaneous_empty(), PaymentId(payment_hash.0), route_params,
2269                         Retry::Attempts(1)).unwrap();
2270                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2271
2272                 // Open a new channel with liquidity on the second hop so we can find a route for the retry
2273                 // attempt, since the initial second hop channel will be excluded from pathfinding
2274                 create_announced_chan_between_nodes(&nodes, 1, 2);
2275
2276                 // We retry payments in `process_pending_htlc_forwards`
2277                 nodes[0].node.process_pending_htlc_forwards();
2278                 check_added_monitors!(nodes[0], 1);
2279                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2280                 assert_eq!(msg_events.len(), 1);
2281                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, None, msg_events.pop().unwrap(), true, Some(payment_preimage));
2282                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
2283         } else if test == AutoRetry::FailAttempts {
2284                 // Ensure ChannelManager will not retry a payment if it has run out of payment attempts.
2285                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2286                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2287                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2288
2289                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
2290                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
2291                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
2292
2293                 // We retry payments in `process_pending_htlc_forwards`
2294                 nodes[0].node.process_pending_htlc_forwards();
2295                 pass_failed_attempt_with_retry_along_path!(channel_id_3, false);
2296
2297                 // Ensure we won't retry a second time.
2298                 nodes[0].node.process_pending_htlc_forwards();
2299                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2300                 assert_eq!(msg_events.len(), 0);
2301         } else if test == AutoRetry::FailTimeout {
2302                 #[cfg(not(feature = "no-std"))] {
2303                         // Ensure ChannelManager will not retry a payment if it times out due to Retry::Timeout.
2304                         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2305                                 PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
2306                         pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2307
2308                         // Advance the time so the second attempt fails due to timeout.
2309                         SinceEpoch::advance(Duration::from_secs(61));
2310
2311                         // Make sure we don't retry again.
2312                         nodes[0].node.process_pending_htlc_forwards();
2313                         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2314                         assert_eq!(msg_events.len(), 0);
2315
2316                         let mut events = nodes[0].node.get_and_clear_pending_events();
2317                         assert_eq!(events.len(), 1);
2318                         match events[0] {
2319                                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2320                                         assert_eq!(payment_hash, *ev_payment_hash);
2321                                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2322                                         assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
2323                                 },
2324                                 _ => panic!("Unexpected event"),
2325                         }
2326                 }
2327         } else if test == AutoRetry::FailOnRestart {
2328                 // Ensure ChannelManager will not retry a payment after restart, even if there were retry
2329                 // attempts remaining prior to restart.
2330                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2331                         PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
2332                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2333
2334                 // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
2335                 // the retry attempt, since the initial second hop channel will be excluded from pathfinding
2336                 let channel_id_3 = create_announced_chan_between_nodes(&nodes, 2, 1).2;
2337
2338                 // Ensure the first retry attempt fails, with 1 retry attempt remaining
2339                 nodes[0].node.process_pending_htlc_forwards();
2340                 pass_failed_attempt_with_retry_along_path!(channel_id_3, true);
2341
2342                 // Restart the node and ensure that ChannelManager does not use its remaining retry attempt
2343                 let node_encoded = nodes[0].node.encode();
2344                 let chan_1_monitor_serialized = get_monitor!(nodes[0], channel_id_1).encode();
2345                 reload_node!(nodes[0], node_encoded, &[&chan_1_monitor_serialized], persister, new_chain_monitor, node_0_deserialized);
2346
2347                 let mut events = nodes[0].node.get_and_clear_pending_events();
2348                 expect_pending_htlcs_forwardable_from_events!(nodes[0], events, true);
2349                 // Make sure we don't retry again.
2350                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2351                 assert_eq!(msg_events.len(), 0);
2352
2353                 let mut events = nodes[0].node.get_and_clear_pending_events();
2354                 assert_eq!(events.len(), 1);
2355                 match events[0] {
2356                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2357                                 assert_eq!(payment_hash, *ev_payment_hash);
2358                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2359                                 assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
2360                         },
2361                         _ => panic!("Unexpected event"),
2362                 }
2363         } else if test == AutoRetry::FailOnRetry {
2364                 nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2365                         PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2366                 pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
2367
2368                 // We retry payments in `process_pending_htlc_forwards`. Since our channel closed, we should
2369                 // fail to find a route.
2370                 nodes[0].node.process_pending_htlc_forwards();
2371                 let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2372                 assert_eq!(msg_events.len(), 0);
2373
2374                 let mut events = nodes[0].node.get_and_clear_pending_events();
2375                 assert_eq!(events.len(), 1);
2376                 match events[0] {
2377                         Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
2378                                 assert_eq!(payment_hash, *ev_payment_hash);
2379                                 assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
2380                                 assert_eq!(PaymentFailureReason::RouteNotFound, ev_reason.unwrap());
2381                         },
2382                         _ => panic!("Unexpected event"),
2383                 }
2384         }
2385 }
2386
2387 #[test]
2388 fn auto_retry_partial_failure() {
2389         // Test that we'll retry appropriately on send partial failure and retry partial failure.
2390         let chanmon_cfgs = create_chanmon_cfgs(2);
2391         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2392         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2393         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2394
2395         // Open three channels, the first has plenty of liquidity, the second and third have ~no
2396         // available liquidity, causing any outbound payments routed over it to fail immediately.
2397         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2398         let chan_2_id = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 989_000_000).0.contents.short_channel_id;
2399         let chan_3_id = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 989_000_000).0.contents.short_channel_id;
2400
2401         // Marshall data to send the payment
2402         let amt_msat = 10_000_000;
2403         let (_, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2404         #[cfg(feature = "std")]
2405         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2406         #[cfg(not(feature = "std"))]
2407         let payment_expiry_secs = 60 * 60;
2408         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2409         invoice_features.set_variable_length_onion_required();
2410         invoice_features.set_payment_secret_required();
2411         invoice_features.set_basic_mpp_optional();
2412         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2413                 .with_expiry_time(payment_expiry_secs as u64)
2414                 .with_bolt11_features(invoice_features).unwrap();
2415
2416         // Configure the initial send path
2417         let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2418         route_params.max_total_routing_fee_msat = None;
2419
2420         let send_route = Route {
2421                 paths: vec![
2422                         Path { hops: vec![RouteHop {
2423                                 pubkey: nodes[1].node.get_our_node_id(),
2424                                 node_features: nodes[1].node.node_features(),
2425                                 short_channel_id: chan_1_id,
2426                                 channel_features: nodes[1].node.channel_features(),
2427                                 fee_msat: amt_msat / 2,
2428                                 cltv_expiry_delta: 100,
2429                                 maybe_announced_channel: true,
2430                         }], blinded_tail: None },
2431                         Path { hops: vec![RouteHop {
2432                                 pubkey: nodes[1].node.get_our_node_id(),
2433                                 node_features: nodes[1].node.node_features(),
2434                                 short_channel_id: chan_2_id,
2435                                 channel_features: nodes[1].node.channel_features(),
2436                                 fee_msat: amt_msat / 2,
2437                                 cltv_expiry_delta: 100,
2438                                 maybe_announced_channel: true,
2439                         }], blinded_tail: None },
2440                 ],
2441                 route_params: Some(route_params.clone()),
2442         };
2443         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
2444
2445         // Configure the retry1 paths
2446         let mut payment_params = route_params.payment_params.clone();
2447         payment_params.previously_failed_channels.push(chan_2_id);
2448         let mut retry_1_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat / 2);
2449         retry_1_params.max_total_routing_fee_msat = None;
2450
2451         let retry_1_route = Route {
2452                 paths: vec![
2453                         Path { hops: vec![RouteHop {
2454                                 pubkey: nodes[1].node.get_our_node_id(),
2455                                 node_features: nodes[1].node.node_features(),
2456                                 short_channel_id: chan_1_id,
2457                                 channel_features: nodes[1].node.channel_features(),
2458                                 fee_msat: amt_msat / 4,
2459                                 cltv_expiry_delta: 100,
2460                                 maybe_announced_channel: true,
2461                         }], blinded_tail: None },
2462                         Path { hops: vec![RouteHop {
2463                                 pubkey: nodes[1].node.get_our_node_id(),
2464                                 node_features: nodes[1].node.node_features(),
2465                                 short_channel_id: chan_3_id,
2466                                 channel_features: nodes[1].node.channel_features(),
2467                                 fee_msat: amt_msat / 4,
2468                                 cltv_expiry_delta: 100,
2469                                 maybe_announced_channel: true,
2470                         }], blinded_tail: None },
2471                 ],
2472                 route_params: Some(retry_1_params.clone()),
2473         };
2474         nodes[0].router.expect_find_route(retry_1_params.clone(), Ok(retry_1_route));
2475
2476         // Configure the retry2 path
2477         let mut payment_params = retry_1_params.payment_params.clone();
2478         payment_params.previously_failed_channels.push(chan_3_id);
2479         let mut retry_2_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat / 4);
2480         retry_2_params.max_total_routing_fee_msat = None;
2481
2482         let retry_2_route = Route {
2483                 paths: vec![
2484                         Path { hops: vec![RouteHop {
2485                                 pubkey: nodes[1].node.get_our_node_id(),
2486                                 node_features: nodes[1].node.node_features(),
2487                                 short_channel_id: chan_1_id,
2488                                 channel_features: nodes[1].node.channel_features(),
2489                                 fee_msat: amt_msat / 4,
2490                                 cltv_expiry_delta: 100,
2491                                 maybe_announced_channel: true,
2492                         }], blinded_tail: None },
2493                 ],
2494                 route_params: Some(retry_2_params.clone()),
2495         };
2496         nodes[0].router.expect_find_route(retry_2_params, Ok(retry_2_route));
2497
2498         // Send a payment that will partially fail on send, then partially fail on retry, then succeed.
2499         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2500                 PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap();
2501         let payment_failed_events = nodes[0].node.get_and_clear_pending_events();
2502         assert_eq!(payment_failed_events.len(), 2);
2503         match payment_failed_events[0] {
2504                 Event::PaymentPathFailed { .. } => {},
2505                 _ => panic!("Unexpected event"),
2506         }
2507         match payment_failed_events[1] {
2508                 Event::PaymentPathFailed { .. } => {},
2509                 _ => panic!("Unexpected event"),
2510         }
2511
2512         // Pass the first part of the payment along the path.
2513         check_added_monitors!(nodes[0], 1); // only one HTLC actually made it out
2514         let mut msg_events = nodes[0].node.get_and_clear_pending_msg_events();
2515
2516         // Only one HTLC/channel update actually made it out
2517         assert_eq!(msg_events.len(), 1);
2518         let mut payment_event = SendEvent::from_event(msg_events.remove(0));
2519
2520         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2521         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
2522         check_added_monitors!(nodes[1], 1);
2523         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2524
2525         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2526         check_added_monitors!(nodes[0], 1);
2527         let as_second_htlc_updates = SendEvent::from_node(&nodes[0]);
2528
2529         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2530         check_added_monitors!(nodes[0], 1);
2531         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2532
2533         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2534         check_added_monitors!(nodes[1], 1);
2535
2536         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[0]);
2537         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.msgs[1]);
2538         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_second_htlc_updates.commitment_msg);
2539         check_added_monitors!(nodes[1], 1);
2540         let (bs_second_raa, bs_second_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2541
2542         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2543         check_added_monitors!(nodes[0], 1);
2544
2545         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_cs);
2546         check_added_monitors!(nodes[0], 1);
2547         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2548
2549         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2550         check_added_monitors!(nodes[1], 1);
2551
2552         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
2553         nodes[1].node.process_pending_htlc_forwards();
2554         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, amt_msat);
2555         nodes[1].node.claim_funds(payment_preimage);
2556         expect_payment_claimed!(nodes[1], payment_hash, amt_msat);
2557         let bs_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2558         assert_eq!(bs_claim_update.update_fulfill_htlcs.len(), 1);
2559
2560         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_claim_update.update_fulfill_htlcs[0]);
2561         expect_payment_sent(&nodes[0], payment_preimage, None, false, false);
2562         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_claim_update.commitment_signed);
2563         check_added_monitors!(nodes[0], 1);
2564         let (as_third_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2565
2566         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
2567         check_added_monitors!(nodes[1], 4);
2568         let bs_second_claim_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2569
2570         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2571         check_added_monitors!(nodes[1], 1);
2572         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2573
2574         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
2575         check_added_monitors!(nodes[0], 1);
2576         expect_payment_path_successful!(nodes[0]);
2577
2578         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[0]);
2579         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[1]);
2580         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.commitment_signed);
2581         check_added_monitors!(nodes[0], 1);
2582         let (as_fourth_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2583
2584         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_fourth_raa);
2585         check_added_monitors!(nodes[1], 1);
2586
2587         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
2588         check_added_monitors!(nodes[1], 1);
2589         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2590
2591         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2592         check_added_monitors!(nodes[0], 1);
2593         let events = nodes[0].node.get_and_clear_pending_events();
2594         assert_eq!(events.len(), 2);
2595         if let Event::PaymentPathSuccessful { .. } = events[0] {} else { panic!(); }
2596         if let Event::PaymentPathSuccessful { .. } = events[1] {} else { panic!(); }
2597 }
2598
2599 #[test]
2600 fn auto_retry_zero_attempts_send_error() {
2601         let chanmon_cfgs = create_chanmon_cfgs(2);
2602         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2603         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2604         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2605
2606         // Open a single channel that does not have sufficient liquidity for the payment we want to
2607         // send.
2608         let chan_id  = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 989_000_000).0.contents.short_channel_id;
2609
2610         // Marshall data to send the payment
2611         let amt_msat = 10_000_000;
2612         let (_, payment_hash, payment_secret) = get_payment_preimage_hash(&nodes[1], Some(amt_msat), None);
2613         #[cfg(feature = "std")]
2614         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2615         #[cfg(not(feature = "std"))]
2616         let payment_expiry_secs = 60 * 60;
2617         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2618         invoice_features.set_variable_length_onion_required();
2619         invoice_features.set_payment_secret_required();
2620         invoice_features.set_basic_mpp_optional();
2621         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2622                 .with_expiry_time(payment_expiry_secs as u64)
2623                 .with_bolt11_features(invoice_features).unwrap();
2624         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2625
2626         // Override the route search to return a route, rather than failing at the route-finding step.
2627         let send_route = Route {
2628                 paths: vec![
2629                         Path { hops: vec![RouteHop {
2630                                 pubkey: nodes[1].node.get_our_node_id(),
2631                                 node_features: nodes[1].node.node_features(),
2632                                 short_channel_id: chan_id,
2633                                 channel_features: nodes[1].node.channel_features(),
2634                                 fee_msat: amt_msat,
2635                                 cltv_expiry_delta: 100,
2636                                 maybe_announced_channel: true,
2637                         }], blinded_tail: None },
2638                 ],
2639                 route_params: Some(route_params.clone()),
2640         };
2641         nodes[0].router.expect_find_route(route_params.clone(), Ok(send_route));
2642
2643         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2644                 PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
2645         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
2646         let events = nodes[0].node.get_and_clear_pending_events();
2647         assert_eq!(events.len(), 2);
2648         if let Event::PaymentPathFailed { .. } = events[0] { } else { panic!(); }
2649         if let Event::PaymentFailed { .. } = events[1] { } else { panic!(); }
2650         check_added_monitors!(nodes[0], 0);
2651 }
2652
2653 #[test]
2654 fn fails_paying_after_rejected_by_payee() {
2655         let chanmon_cfgs = create_chanmon_cfgs(2);
2656         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2657         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
2658         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2659
2660         create_announced_chan_between_nodes(&nodes, 0, 1).0.contents.short_channel_id;
2661
2662         // Marshall data to send the payment
2663         let amt_msat = 20_000;
2664         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2665         #[cfg(feature = "std")]
2666         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2667         #[cfg(not(feature = "std"))]
2668         let payment_expiry_secs = 60 * 60;
2669         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2670         invoice_features.set_variable_length_onion_required();
2671         invoice_features.set_payment_secret_required();
2672         invoice_features.set_basic_mpp_optional();
2673         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2674                 .with_expiry_time(payment_expiry_secs as u64)
2675                 .with_bolt11_features(invoice_features).unwrap();
2676         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2677
2678         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2679                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2680         check_added_monitors!(nodes[0], 1);
2681         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
2682         assert_eq!(events.len(), 1);
2683         let mut payment_event = SendEvent::from_event(events.pop().unwrap());
2684         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
2685         check_added_monitors!(nodes[1], 0);
2686         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
2687         expect_pending_htlcs_forwardable!(nodes[1]);
2688         expect_payment_claimable!(&nodes[1], payment_hash, payment_secret, amt_msat);
2689
2690         nodes[1].node.fail_htlc_backwards(&payment_hash);
2691         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash }]);
2692         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
2693 }
2694
2695 #[test]
2696 fn retry_multi_path_single_failed_payment() {
2697         // Tests that we can/will retry after a single path of an MPP payment failed immediately
2698         let chanmon_cfgs = create_chanmon_cfgs(2);
2699         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2700         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2701         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2702
2703         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2704         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2705
2706         let amt_msat = 100_010_000;
2707
2708         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2709         #[cfg(feature = "std")]
2710         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2711         #[cfg(not(feature = "std"))]
2712         let payment_expiry_secs = 60 * 60;
2713         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2714         invoice_features.set_variable_length_onion_required();
2715         invoice_features.set_payment_secret_required();
2716         invoice_features.set_basic_mpp_optional();
2717         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2718                 .with_expiry_time(payment_expiry_secs as u64)
2719                 .with_bolt11_features(invoice_features).unwrap();
2720         let mut route_params = RouteParameters::from_payment_params_and_value(
2721                 payment_params.clone(), amt_msat);
2722         route_params.max_total_routing_fee_msat = None;
2723
2724         let chans = nodes[0].node.list_usable_channels();
2725         let mut route = Route {
2726                 paths: vec![
2727                         Path { hops: vec![RouteHop {
2728                                 pubkey: nodes[1].node.get_our_node_id(),
2729                                 node_features: nodes[1].node.node_features(),
2730                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2731                                 channel_features: nodes[1].node.channel_features(),
2732                                 fee_msat: 10_000,
2733                                 cltv_expiry_delta: 100,
2734                                 maybe_announced_channel: true,
2735                         }], blinded_tail: None },
2736                         Path { hops: vec![RouteHop {
2737                                 pubkey: nodes[1].node.get_our_node_id(),
2738                                 node_features: nodes[1].node.node_features(),
2739                                 short_channel_id: chans[1].short_channel_id.unwrap(),
2740                                 channel_features: nodes[1].node.channel_features(),
2741                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2742                                 cltv_expiry_delta: 100,
2743                                 maybe_announced_channel: true,
2744                         }], blinded_tail: None },
2745                 ],
2746                 route_params: Some(route_params.clone()),
2747         };
2748         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2749         // On retry, split the payment across both channels.
2750         route.paths[0].hops[0].fee_msat = 50_000_001;
2751         route.paths[1].hops[0].fee_msat = 50_000_000;
2752         let mut pay_params = route.route_params.clone().unwrap().payment_params;
2753         pay_params.previously_failed_channels.push(chans[1].short_channel_id.unwrap());
2754
2755         let mut retry_params = RouteParameters::from_payment_params_and_value(pay_params, 100_000_000);
2756         retry_params.max_total_routing_fee_msat = None;
2757         route.route_params = Some(retry_params.clone());
2758         nodes[0].router.expect_find_route(retry_params, Ok(route.clone()));
2759
2760         {
2761                 let scorer = chanmon_cfgs[0].scorer.read().unwrap();
2762                 // The initial send attempt, 2 paths
2763                 scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 10_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2764                 scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 100_000_001, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2765                 // The retry, 2 paths. Ensure that the in-flight HTLC amount is factored in.
2766                 scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_001, inflight_htlc_msat: 10_000, effective_capacity: EffectiveCapacity::Unknown });
2767                 scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
2768         }
2769
2770         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2771                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2772         let events = nodes[0].node.get_and_clear_pending_events();
2773         assert_eq!(events.len(), 1);
2774         match events[0] {
2775                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false,
2776                         failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }},
2777                         short_channel_id: Some(expected_scid), .. } =>
2778                 {
2779                         assert_eq!(payment_hash, ev_payment_hash);
2780                         assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
2781                 },
2782                 _ => panic!("Unexpected event"),
2783         }
2784         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2785         assert_eq!(htlc_msgs.len(), 2);
2786         check_added_monitors!(nodes[0], 2);
2787 }
2788
2789 #[test]
2790 fn immediate_retry_on_failure() {
2791         // Tests that we can/will retry immediately after a failure
2792         let chanmon_cfgs = create_chanmon_cfgs(2);
2793         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
2794         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None, None]);
2795         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
2796
2797         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2798         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
2799
2800         let amt_msat = 100_000_001;
2801         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2802         #[cfg(feature = "std")]
2803         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2804         #[cfg(not(feature = "std"))]
2805         let payment_expiry_secs = 60 * 60;
2806         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2807         invoice_features.set_variable_length_onion_required();
2808         invoice_features.set_payment_secret_required();
2809         invoice_features.set_basic_mpp_optional();
2810         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2811                 .with_expiry_time(payment_expiry_secs as u64)
2812                 .with_bolt11_features(invoice_features).unwrap();
2813         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2814
2815         let chans = nodes[0].node.list_usable_channels();
2816         let mut route = Route {
2817                 paths: vec![
2818                         Path { hops: vec![RouteHop {
2819                                 pubkey: nodes[1].node.get_our_node_id(),
2820                                 node_features: nodes[1].node.node_features(),
2821                                 short_channel_id: chans[0].short_channel_id.unwrap(),
2822                                 channel_features: nodes[1].node.channel_features(),
2823                                 fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
2824                                 cltv_expiry_delta: 100,
2825                                 maybe_announced_channel: true,
2826                         }], blinded_tail: None },
2827                 ],
2828                 route_params: Some(route_params.clone()),
2829         };
2830         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2831         // On retry, split the payment across both channels.
2832         route.paths.push(route.paths[0].clone());
2833         route.paths[0].hops[0].short_channel_id = chans[1].short_channel_id.unwrap();
2834         route.paths[0].hops[0].fee_msat = 50_000_000;
2835         route.paths[1].hops[0].fee_msat = 50_000_001;
2836         let mut pay_params = route_params.payment_params.clone();
2837         pay_params.previously_failed_channels.push(chans[0].short_channel_id.unwrap());
2838         let retry_params = RouteParameters::from_payment_params_and_value(pay_params, amt_msat);
2839         route.route_params = Some(retry_params.clone());
2840         nodes[0].router.expect_find_route(retry_params, Ok(route.clone()));
2841
2842         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2843                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2844         let events = nodes[0].node.get_and_clear_pending_events();
2845         assert_eq!(events.len(), 1);
2846         match events[0] {
2847                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently: false,
2848                         failure: PathFailure::InitialSend { err: APIError::ChannelUnavailable { .. }},
2849                         short_channel_id: Some(expected_scid), .. } =>
2850                 {
2851                         assert_eq!(payment_hash, ev_payment_hash);
2852                         assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
2853                 },
2854                 _ => panic!("Unexpected event"),
2855         }
2856         let htlc_msgs = nodes[0].node.get_and_clear_pending_msg_events();
2857         assert_eq!(htlc_msgs.len(), 2);
2858         check_added_monitors!(nodes[0], 2);
2859 }
2860
2861 #[test]
2862 fn no_extra_retries_on_back_to_back_fail() {
2863         // In a previous release, we had a race where we may exceed the payment retry count if we
2864         // get two failures in a row with the second indicating that all paths had failed (this field,
2865         // `all_paths_failed`, has since been removed).
2866         // Generally, when we give up trying to retry a payment, we don't know for sure what the
2867         // current state of the ChannelManager event queue is. Specifically, we cannot be sure that
2868         // there are not multiple additional `PaymentPathFailed` or even `PaymentSent` events
2869         // pending which we will see later. Thus, when we previously removed the retry tracking map
2870         // entry after a `all_paths_failed` `PaymentPathFailed` event, we may have dropped the
2871         // retry entry even though more events for the same payment were still pending. This led to
2872         // us retrying a payment again even though we'd already given up on it.
2873         //
2874         // We now have a separate event - `PaymentFailed` which indicates no HTLCs remain and which
2875         // is used to remove the payment retry counter entries instead. This tests for the specific
2876         // excess-retry case while also testing `PaymentFailed` generation.
2877
2878         let chanmon_cfgs = create_chanmon_cfgs(3);
2879         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
2880         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
2881         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
2882
2883         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
2884         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
2885
2886         let amt_msat = 200_000_000;
2887         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], amt_msat);
2888         #[cfg(feature = "std")]
2889         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
2890         #[cfg(not(feature = "std"))]
2891         let payment_expiry_secs = 60 * 60;
2892         let mut invoice_features = Bolt11InvoiceFeatures::empty();
2893         invoice_features.set_variable_length_onion_required();
2894         invoice_features.set_payment_secret_required();
2895         invoice_features.set_basic_mpp_optional();
2896         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
2897                 .with_expiry_time(payment_expiry_secs as u64)
2898                 .with_bolt11_features(invoice_features).unwrap();
2899         let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
2900         route_params.max_total_routing_fee_msat = None;
2901
2902         let mut route = Route {
2903                 paths: vec![
2904                         Path { hops: vec![RouteHop {
2905                                 pubkey: nodes[1].node.get_our_node_id(),
2906                                 node_features: nodes[1].node.node_features(),
2907                                 short_channel_id: chan_1_scid,
2908                                 channel_features: nodes[1].node.channel_features(),
2909                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2910                                 cltv_expiry_delta: 100,
2911                                 maybe_announced_channel: true,
2912                         }, RouteHop {
2913                                 pubkey: nodes[2].node.get_our_node_id(),
2914                                 node_features: nodes[2].node.node_features(),
2915                                 short_channel_id: chan_2_scid,
2916                                 channel_features: nodes[2].node.channel_features(),
2917                                 fee_msat: 100_000_000,
2918                                 cltv_expiry_delta: 100,
2919                                 maybe_announced_channel: true,
2920                         }], blinded_tail: None },
2921                         Path { hops: vec![RouteHop {
2922                                 pubkey: nodes[1].node.get_our_node_id(),
2923                                 node_features: nodes[1].node.node_features(),
2924                                 short_channel_id: chan_1_scid,
2925                                 channel_features: nodes[1].node.channel_features(),
2926                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
2927                                 cltv_expiry_delta: 100,
2928                                 maybe_announced_channel: true,
2929                         }, RouteHop {
2930                                 pubkey: nodes[2].node.get_our_node_id(),
2931                                 node_features: nodes[2].node.node_features(),
2932                                 short_channel_id: chan_2_scid,
2933                                 channel_features: nodes[2].node.channel_features(),
2934                                 fee_msat: 100_000_000,
2935                                 cltv_expiry_delta: 100,
2936                                 maybe_announced_channel: true,
2937                         }], blinded_tail: None }
2938                 ],
2939                 route_params: Some(route_params.clone()),
2940         };
2941         route.route_params.as_mut().unwrap().max_total_routing_fee_msat = None;
2942         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
2943         let mut second_payment_params = route_params.payment_params.clone();
2944         second_payment_params.previously_failed_channels = vec![chan_2_scid, chan_2_scid];
2945         // On retry, we'll only return one path
2946         route.paths.remove(1);
2947         route.paths[0].hops[1].fee_msat = amt_msat;
2948         let mut retry_params = RouteParameters::from_payment_params_and_value(second_payment_params, amt_msat);
2949         retry_params.max_total_routing_fee_msat = None;
2950         route.route_params = Some(retry_params.clone());
2951         nodes[0].router.expect_find_route(retry_params, Ok(route.clone()));
2952
2953         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
2954                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
2955         let htlc_updates = SendEvent::from_node(&nodes[0]);
2956         check_added_monitors!(nodes[0], 1);
2957         assert_eq!(htlc_updates.msgs.len(), 1);
2958
2959         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
2960         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
2961         check_added_monitors!(nodes[1], 1);
2962         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2963
2964         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
2965         check_added_monitors!(nodes[0], 1);
2966         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
2967
2968         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
2969         check_added_monitors!(nodes[0], 1);
2970         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2971
2972         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
2973         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
2974         check_added_monitors!(nodes[1], 1);
2975         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2976
2977         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
2978         check_added_monitors!(nodes[1], 1);
2979         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2980
2981         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
2982         check_added_monitors!(nodes[0], 1);
2983
2984         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
2985         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
2986         check_added_monitors!(nodes[0], 1);
2987         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
2988
2989         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
2990         check_added_monitors!(nodes[1], 1);
2991         let bs_second_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2992
2993         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
2994         check_added_monitors!(nodes[1], 1);
2995         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
2996
2997         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.update_fail_htlcs[0]);
2998         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_fail_update.commitment_signed);
2999         check_added_monitors!(nodes[0], 1);
3000
3001         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
3002         check_added_monitors!(nodes[0], 1);
3003         let (as_third_raa, as_fourth_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
3004
3005         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_third_raa);
3006         check_added_monitors!(nodes[1], 1);
3007         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_fourth_cs);
3008         check_added_monitors!(nodes[1], 1);
3009         let bs_fourth_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
3010
3011         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_fourth_raa);
3012         check_added_monitors!(nodes[0], 1);
3013
3014         // At this point A has sent two HTLCs which both failed due to lack of fee. It now has two
3015         // pending `PaymentPathFailed` events, one with `all_paths_failed` unset, and the second
3016         // with it set.
3017         //
3018         // Previously, we retried payments in an event consumer, which would retry each
3019         // `PaymentPathFailed` individually. In that setup, we had retried the payment in response to
3020         // the first `PaymentPathFailed`, then seen the second `PaymentPathFailed` with
3021         // `all_paths_failed` set and assumed the payment was completely failed. We ultimately fixed it
3022         // by adding the `PaymentFailed` event.
3023         //
3024         // Because we now retry payments as a batch, we simply return a single-path route in the
3025         // second, batched, request, have that fail, ensure the payment was abandoned.
3026         let mut events = nodes[0].node.get_and_clear_pending_events();
3027         assert_eq!(events.len(), 3);
3028         match events[0] {
3029                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
3030                         assert_eq!(payment_hash, ev_payment_hash);
3031                         assert_eq!(payment_failed_permanently, false);
3032                 },
3033                 _ => panic!("Unexpected event"),
3034         }
3035         match events[1] {
3036                 Event::PendingHTLCsForwardable { .. } => {},
3037                 _ => panic!("Unexpected event"),
3038         }
3039         match events[2] {
3040                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
3041                         assert_eq!(payment_hash, ev_payment_hash);
3042                         assert_eq!(payment_failed_permanently, false);
3043                 },
3044                 _ => panic!("Unexpected event"),
3045         }
3046
3047         nodes[0].node.process_pending_htlc_forwards();
3048         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
3049         check_added_monitors!(nodes[0], 1);
3050
3051         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
3052         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
3053         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3054         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
3055         commitment_signed_dance!(nodes[0], nodes[1], &bs_fail_update.commitment_signed, false, true);
3056
3057         let mut events = nodes[0].node.get_and_clear_pending_events();
3058         assert_eq!(events.len(), 2);
3059         match events[0] {
3060                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
3061                         assert_eq!(payment_hash, ev_payment_hash);
3062                         assert_eq!(payment_failed_permanently, false);
3063                 },
3064                 _ => panic!("Unexpected event"),
3065         }
3066         match events[1] {
3067                 Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
3068                         assert_eq!(payment_hash, *ev_payment_hash);
3069                         assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
3070                         assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
3071                 },
3072                 _ => panic!("Unexpected event"),
3073         }
3074 }
3075
3076 #[test]
3077 fn test_simple_partial_retry() {
3078         // In the first version of the in-`ChannelManager` payment retries, retries were sent for the
3079         // full amount of the payment, rather than only the missing amount. Here we simply test for
3080         // this by sending a payment with two parts, failing one, and retrying the second. Note that
3081         // `TestRouter` will check that the `RouteParameters` (which contain the amount) matches the
3082         // request.
3083         let chanmon_cfgs = create_chanmon_cfgs(3);
3084         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
3085         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
3086         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
3087
3088         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
3089         let chan_2_scid = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, 10_000_000, 0).0.contents.short_channel_id;
3090
3091         let amt_msat = 200_000_000;
3092         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
3093         #[cfg(feature = "std")]
3094         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
3095         #[cfg(not(feature = "std"))]
3096         let payment_expiry_secs = 60 * 60;
3097         let mut invoice_features = Bolt11InvoiceFeatures::empty();
3098         invoice_features.set_variable_length_onion_required();
3099         invoice_features.set_payment_secret_required();
3100         invoice_features.set_basic_mpp_optional();
3101         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
3102                 .with_expiry_time(payment_expiry_secs as u64)
3103                 .with_bolt11_features(invoice_features).unwrap();
3104         let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
3105         route_params.max_total_routing_fee_msat = None;
3106
3107         let mut route = Route {
3108                 paths: vec![
3109                         Path { hops: vec![RouteHop {
3110                                 pubkey: nodes[1].node.get_our_node_id(),
3111                                 node_features: nodes[1].node.node_features(),
3112                                 short_channel_id: chan_1_scid,
3113                                 channel_features: nodes[1].node.channel_features(),
3114                                 fee_msat: 0, // nodes[1] will fail the payment as we don't pay its fee
3115                                 cltv_expiry_delta: 100,
3116                                 maybe_announced_channel: true,
3117                         }, RouteHop {
3118                                 pubkey: nodes[2].node.get_our_node_id(),
3119                                 node_features: nodes[2].node.node_features(),
3120                                 short_channel_id: chan_2_scid,
3121                                 channel_features: nodes[2].node.channel_features(),
3122                                 fee_msat: 100_000_000,
3123                                 cltv_expiry_delta: 100,
3124                                 maybe_announced_channel: true,
3125                         }], blinded_tail: None },
3126                         Path { hops: vec![RouteHop {
3127                                 pubkey: nodes[1].node.get_our_node_id(),
3128                                 node_features: nodes[1].node.node_features(),
3129                                 short_channel_id: chan_1_scid,
3130                                 channel_features: nodes[1].node.channel_features(),
3131                                 fee_msat: 100_000,
3132                                 cltv_expiry_delta: 100,
3133                                 maybe_announced_channel: true,
3134                         }, RouteHop {
3135                                 pubkey: nodes[2].node.get_our_node_id(),
3136                                 node_features: nodes[2].node.node_features(),
3137                                 short_channel_id: chan_2_scid,
3138                                 channel_features: nodes[2].node.channel_features(),
3139                                 fee_msat: 100_000_000,
3140                                 cltv_expiry_delta: 100,
3141                                 maybe_announced_channel: true,
3142                         }], blinded_tail: None }
3143                 ],
3144                 route_params: Some(route_params.clone()),
3145         };
3146
3147         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3148
3149         let mut second_payment_params = route_params.payment_params.clone();
3150         second_payment_params.previously_failed_channels = vec![chan_2_scid];
3151         // On retry, we'll only be asked for one path (or 100k sats)
3152         route.paths.remove(0);
3153         let mut retry_params = RouteParameters::from_payment_params_and_value(second_payment_params, amt_msat / 2);
3154         retry_params.max_total_routing_fee_msat = None;
3155         route.route_params = Some(retry_params.clone());
3156         nodes[0].router.expect_find_route(retry_params, Ok(route.clone()));
3157
3158         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3159                 PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
3160         let htlc_updates = SendEvent::from_node(&nodes[0]);
3161         check_added_monitors!(nodes[0], 1);
3162         assert_eq!(htlc_updates.msgs.len(), 1);
3163
3164         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &htlc_updates.msgs[0]);
3165         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &htlc_updates.commitment_msg);
3166         check_added_monitors!(nodes[1], 1);
3167         let (bs_first_raa, bs_first_cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3168
3169         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_first_raa);
3170         check_added_monitors!(nodes[0], 1);
3171         let second_htlc_updates = SendEvent::from_node(&nodes[0]);
3172
3173         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_first_cs);
3174         check_added_monitors!(nodes[0], 1);
3175         let as_first_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
3176
3177         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &second_htlc_updates.msgs[0]);
3178         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &second_htlc_updates.commitment_msg);
3179         check_added_monitors!(nodes[1], 1);
3180         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
3181
3182         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_first_raa);
3183         check_added_monitors!(nodes[1], 1);
3184         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3185
3186         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
3187         check_added_monitors!(nodes[0], 1);
3188
3189         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
3190         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_fail_update.commitment_signed);
3191         check_added_monitors!(nodes[0], 1);
3192         let (as_second_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
3193
3194         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
3195         check_added_monitors!(nodes[1], 1);
3196
3197         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_third_cs);
3198         check_added_monitors!(nodes[1], 1);
3199
3200         let bs_third_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
3201
3202         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
3203         check_added_monitors!(nodes[0], 1);
3204
3205         let mut events = nodes[0].node.get_and_clear_pending_events();
3206         assert_eq!(events.len(), 2);
3207         match events[0] {
3208                 Event::PaymentPathFailed { payment_hash: ev_payment_hash, payment_failed_permanently, ..  } => {
3209                         assert_eq!(payment_hash, ev_payment_hash);
3210                         assert_eq!(payment_failed_permanently, false);
3211                 },
3212                 _ => panic!("Unexpected event"),
3213         }
3214         match events[1] {
3215                 Event::PendingHTLCsForwardable { .. } => {},
3216                 _ => panic!("Unexpected event"),
3217         }
3218
3219         nodes[0].node.process_pending_htlc_forwards();
3220         let retry_htlc_updates = SendEvent::from_node(&nodes[0]);
3221         check_added_monitors!(nodes[0], 1);
3222
3223         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &retry_htlc_updates.msgs[0]);
3224         commitment_signed_dance!(nodes[1], nodes[0], &retry_htlc_updates.commitment_msg, false, true);
3225
3226         expect_pending_htlcs_forwardable!(nodes[1]);
3227         check_added_monitors!(nodes[1], 1);
3228
3229         let bs_forward_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
3230         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[0]);
3231         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_forward_update.update_add_htlcs[1]);
3232         commitment_signed_dance!(nodes[2], nodes[1], &bs_forward_update.commitment_signed, false);
3233
3234         expect_pending_htlcs_forwardable!(nodes[2]);
3235         expect_payment_claimable!(nodes[2], payment_hash, payment_secret, amt_msat);
3236 }
3237
3238 #[test]
3239 #[cfg(feature = "std")]
3240 fn test_threaded_payment_retries() {
3241         // In the first version of the in-`ChannelManager` payment retries, retries weren't limited to
3242         // a single thread and would happily let multiple threads run retries at the same time. Because
3243         // retries are done by first calculating the amount we need to retry, then dropping the
3244         // relevant lock, then actually sending, we would happily let multiple threads retry the same
3245         // amount at the same time, overpaying our original HTLC!
3246         let chanmon_cfgs = create_chanmon_cfgs(4);
3247         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3248         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3249         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3250
3251         // There is one mitigating guardrail when retrying payments - we can never over-pay by more
3252         // than 10% of the original value. Thus, we want all our retries to be below that. In order to
3253         // keep things simple, we route one HTLC for 0.1% of the payment over channel 1 and the rest
3254         // out over channel 3+4. This will let us ignore 99% of the payment value and deal with only
3255         // our channel.
3256         let chan_1_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 10_000_000, 0).0.contents.short_channel_id;
3257         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 10_000_000, 0);
3258         let chan_3_scid = create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 10_000_000, 0).0.contents.short_channel_id;
3259         let chan_4_scid = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 10_000_000, 0).0.contents.short_channel_id;
3260
3261         let amt_msat = 100_000_000;
3262         let (_, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[2], amt_msat);
3263         #[cfg(feature = "std")]
3264         let payment_expiry_secs = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs() + 60 * 60;
3265         #[cfg(not(feature = "std"))]
3266         let payment_expiry_secs = 60 * 60;
3267         let mut invoice_features = Bolt11InvoiceFeatures::empty();
3268         invoice_features.set_variable_length_onion_required();
3269         invoice_features.set_payment_secret_required();
3270         invoice_features.set_basic_mpp_optional();
3271         let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)
3272                 .with_expiry_time(payment_expiry_secs as u64)
3273                 .with_bolt11_features(invoice_features).unwrap();
3274         let mut route_params = RouteParameters {
3275                 payment_params, final_value_msat: amt_msat, max_total_routing_fee_msat: Some(500_000),
3276         };
3277
3278         let mut route = Route {
3279                 paths: vec![
3280                         Path { hops: vec![RouteHop {
3281                                 pubkey: nodes[1].node.get_our_node_id(),
3282                                 node_features: nodes[1].node.node_features(),
3283                                 short_channel_id: chan_1_scid,
3284                                 channel_features: nodes[1].node.channel_features(),
3285                                 fee_msat: 0,
3286                                 cltv_expiry_delta: 100,
3287                                 maybe_announced_channel: true,
3288                         }, RouteHop {
3289                                 pubkey: nodes[3].node.get_our_node_id(),
3290                                 node_features: nodes[2].node.node_features(),
3291                                 short_channel_id: 42, // Set a random SCID which nodes[1] will fail as unknown
3292                                 channel_features: nodes[2].node.channel_features(),
3293                                 fee_msat: amt_msat / 1000,
3294                                 cltv_expiry_delta: 100,
3295                                 maybe_announced_channel: true,
3296                         }], blinded_tail: None },
3297                         Path { hops: vec![RouteHop {
3298                                 pubkey: nodes[2].node.get_our_node_id(),
3299                                 node_features: nodes[2].node.node_features(),
3300                                 short_channel_id: chan_3_scid,
3301                                 channel_features: nodes[2].node.channel_features(),
3302                                 fee_msat: 100_000,
3303                                 cltv_expiry_delta: 100,
3304                                 maybe_announced_channel: true,
3305                         }, RouteHop {
3306                                 pubkey: nodes[3].node.get_our_node_id(),
3307                                 node_features: nodes[3].node.node_features(),
3308                                 short_channel_id: chan_4_scid,
3309                                 channel_features: nodes[3].node.channel_features(),
3310                                 fee_msat: amt_msat - amt_msat / 1000,
3311                                 cltv_expiry_delta: 100,
3312                                 maybe_announced_channel: true,
3313                         }], blinded_tail: None }
3314                 ],
3315                 route_params: Some(route_params.clone()),
3316         };
3317         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3318
3319         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3320                 PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(0xdeadbeef)).unwrap();
3321         check_added_monitors!(nodes[0], 2);
3322         let mut send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
3323         assert_eq!(send_msg_events.len(), 2);
3324         send_msg_events.retain(|msg|
3325                 if let MessageSendEvent::UpdateHTLCs { node_id, .. } = msg {
3326                         // Drop the commitment update for nodes[2], we can just let that one sit pending
3327                         // forever.
3328                         *node_id == nodes[1].node.get_our_node_id()
3329                 } else { panic!(); }
3330         );
3331
3332         // from here on out, the retry `RouteParameters` amount will be amt/1000
3333         route_params.final_value_msat /= 1000;
3334         route.route_params = Some(route_params.clone());
3335         route.paths.pop();
3336
3337         let end_time = Instant::now() + Duration::from_secs(1);
3338         macro_rules! thread_body { () => { {
3339                 // We really want std::thread::scope, but its not stable until 1.63. Until then, we get unsafe.
3340                 let node_ref = NodePtr::from_node(&nodes[0]);
3341                 move || {
3342                         let node_a = unsafe { &*node_ref.0 };
3343                         while Instant::now() < end_time {
3344                                 node_a.node.get_and_clear_pending_events(); // wipe the PendingHTLCsForwardable
3345                                 // Ignore if we have any pending events, just always pretend we just got a
3346                                 // PendingHTLCsForwardable
3347                                 node_a.node.process_pending_htlc_forwards();
3348                         }
3349                 }
3350         } } }
3351         let mut threads = Vec::new();
3352         for _ in 0..16 { threads.push(std::thread::spawn(thread_body!())); }
3353
3354         // Back in the main thread, poll pending messages and make sure that we never have more than
3355         // one HTLC pending at a time. Note that the commitment_signed_dance will fail horribly if
3356         // there are HTLC messages shoved in while its running. This allows us to test that we never
3357         // generate an additional update_add_htlc until we've fully failed the first.
3358         let mut previously_failed_channels = Vec::new();
3359         loop {
3360                 assert_eq!(send_msg_events.len(), 1);
3361                 let send_event = SendEvent::from_event(send_msg_events.pop().unwrap());
3362                 assert_eq!(send_event.msgs.len(), 1);
3363
3364                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
3365                 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
3366
3367                 // Note that we only push one route into `expect_find_route` at a time, because that's all
3368                 // the retries (should) need. If the bug is reintroduced "real" routes may be selected, but
3369                 // we should still ultimately fail for the same reason - because we're trying to send too
3370                 // many HTLCs at once.
3371                 let mut new_route_params = route_params.clone();
3372                 previously_failed_channels.push(route.paths[0].hops[1].short_channel_id);
3373                 new_route_params.payment_params.previously_failed_channels = previously_failed_channels.clone();
3374                 new_route_params.max_total_routing_fee_msat.as_mut().map(|m| *m -= 100_000);
3375                 route.paths[0].hops[1].short_channel_id += 1;
3376                 route.route_params = Some(new_route_params.clone());
3377                 nodes[0].router.expect_find_route(new_route_params, Ok(route.clone()));
3378
3379                 let bs_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3380                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_updates.update_fail_htlcs[0]);
3381                 // The "normal" commitment_signed_dance delivers the final RAA and then calls
3382                 // `check_added_monitors` to ensure only the one RAA-generated monitor update was created.
3383                 // This races with our other threads which may generate an add-HTLCs commitment update via
3384                 // `process_pending_htlc_forwards`. Instead, we defer the monitor update check until after
3385                 // *we've* called `process_pending_htlc_forwards` when its guaranteed to have two updates.
3386                 let last_raa = commitment_signed_dance!(nodes[0], nodes[1], bs_fail_updates.commitment_signed, false, true, false, true);
3387                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &last_raa);
3388
3389                 let cur_time = Instant::now();
3390                 if cur_time > end_time {
3391                         for thread in threads.drain(..) { thread.join().unwrap(); }
3392                 }
3393
3394                 // Make sure we have some events to handle when we go around...
3395                 nodes[0].node.get_and_clear_pending_events(); // wipe the PendingHTLCsForwardable
3396                 nodes[0].node.process_pending_htlc_forwards();
3397                 send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
3398                 check_added_monitors!(nodes[0], 2);
3399
3400                 if cur_time > end_time {
3401                         break;
3402                 }
3403         }
3404 }
3405
3406 fn do_no_missing_sent_on_reload(persist_manager_with_payment: bool, at_midpoint: bool) {
3407         // Test that if we reload in the middle of an HTLC claim commitment signed dance we'll still
3408         // receive the PaymentSent event even if the ChannelManager had no idea about the payment when
3409         // it was last persisted.
3410         let chanmon_cfgs = create_chanmon_cfgs(2);
3411         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
3412         let (persister_a, persister_b, persister_c);
3413         let (chain_monitor_a, chain_monitor_b, chain_monitor_c);
3414         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
3415         let (nodes_0_deserialized, nodes_0_deserialized_b, nodes_0_deserialized_c);
3416         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
3417
3418         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
3419
3420         let mut nodes_0_serialized = Vec::new();
3421         if !persist_manager_with_payment {
3422                 nodes_0_serialized = nodes[0].node.encode();
3423         }
3424
3425         let (our_payment_preimage, our_payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 1_000_000);
3426
3427         if persist_manager_with_payment {
3428                 nodes_0_serialized = nodes[0].node.encode();
3429         }
3430
3431         nodes[1].node.claim_funds(our_payment_preimage);
3432         check_added_monitors!(nodes[1], 1);
3433         expect_payment_claimed!(nodes[1], our_payment_hash, 1_000_000);
3434
3435         if at_midpoint {
3436                 let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3437                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3438                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
3439                 check_added_monitors!(nodes[0], 1);
3440         } else {
3441                 let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3442                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
3443                 commitment_signed_dance!(nodes[0], nodes[1], htlc_fulfill_updates.commitment_signed, false);
3444                 // Ignore the PaymentSent event which is now pending on nodes[0] - if we were to handle it we'd
3445                 // be expected to ignore the eventual conflicting PaymentFailed, but by not looking at it we
3446                 // expect to get the PaymentSent again later.
3447                 check_added_monitors(&nodes[0], 0);
3448         }
3449
3450         // The ChannelMonitor should always be the latest version, as we're required to persist it
3451         // during the commitment signed handling.
3452         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3453         reload_node!(nodes[0], test_default_channel_config(), &nodes_0_serialized, &[&chan_0_monitor_serialized], persister_a, chain_monitor_a, nodes_0_deserialized);
3454
3455         let events = nodes[0].node.get_and_clear_pending_events();
3456         assert_eq!(events.len(), 2);
3457         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[0] {} else { panic!(); }
3458         if let Event::PaymentSent { payment_preimage, .. } = events[1] { assert_eq!(payment_preimage, our_payment_preimage); } else { panic!(); }
3459         // Note that we don't get a PaymentPathSuccessful here as we leave the HTLC pending to avoid
3460         // the double-claim that would otherwise appear at the end of this test.
3461         nodes[0].node.timer_tick_occurred();
3462         let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3463         assert_eq!(as_broadcasted_txn.len(), 1);
3464
3465         // Ensure that, even after some time, if we restart we still include *something* in the current
3466         // `ChannelManager` which prevents a `PaymentFailed` when we restart even if pending resolved
3467         // payments have since been timed out thanks to `IDEMPOTENCY_TIMEOUT_TICKS`.
3468         // A naive implementation of the fix here would wipe the pending payments set, causing a
3469         // failure event when we restart.
3470         for _ in 0..(IDEMPOTENCY_TIMEOUT_TICKS * 2) { nodes[0].node.timer_tick_occurred(); }
3471
3472         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3473         reload_node!(nodes[0], test_default_channel_config(), &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister_b, chain_monitor_b, nodes_0_deserialized_b);
3474         let events = nodes[0].node.get_and_clear_pending_events();
3475         assert!(events.is_empty());
3476
3477         // Ensure that we don't generate any further events even after the channel-closing commitment
3478         // transaction is confirmed on-chain.
3479         confirm_transaction(&nodes[0], &as_broadcasted_txn[0]);
3480         for _ in 0..(IDEMPOTENCY_TIMEOUT_TICKS * 2) { nodes[0].node.timer_tick_occurred(); }
3481
3482         let events = nodes[0].node.get_and_clear_pending_events();
3483         assert!(events.is_empty());
3484
3485         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
3486         reload_node!(nodes[0], test_default_channel_config(), &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister_c, chain_monitor_c, nodes_0_deserialized_c);
3487         let events = nodes[0].node.get_and_clear_pending_events();
3488         assert!(events.is_empty());
3489         check_added_monitors(&nodes[0], 1);
3490 }
3491
3492 #[test]
3493 fn no_missing_sent_on_midpoint_reload() {
3494         do_no_missing_sent_on_reload(false, true);
3495         do_no_missing_sent_on_reload(true, true);
3496 }
3497
3498 #[test]
3499 fn no_missing_sent_on_reload() {
3500         do_no_missing_sent_on_reload(false, false);
3501         do_no_missing_sent_on_reload(true, false);
3502 }
3503
3504 fn do_claim_from_closed_chan(fail_payment: bool) {
3505         // Previously, LDK would refuse to claim a payment if a channel on which the payment was
3506         // received had been closed between when the HTLC was received and when we went to claim it.
3507         // This makes sense in the payment case - why pay an on-chain fee to claim the HTLC when
3508         // presumably the sender may retry later. Long ago it also reduced total code in the claim
3509         // pipeline.
3510         //
3511         // However, this doesn't make sense if you're trying to do an atomic swap or some other
3512         // protocol that requires atomicity with some other action - if your money got claimed
3513         // elsewhere you need to be able to claim the HTLC in lightning no matter what. Further, this
3514         // is an over-optimization - there should be a very, very low likelihood that a channel closes
3515         // between when we receive the last HTLC for a payment and the user goes to claim the payment.
3516         // Since we now have code to handle this anyway we should allow it.
3517
3518         // Build 4 nodes and send an MPP payment across two paths. By building a route manually set the
3519         // CLTVs on the paths to different value resulting in a different claim deadline.
3520         let chanmon_cfgs = create_chanmon_cfgs(4);
3521         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3522         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3523         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3524
3525         create_announced_chan_between_nodes(&nodes, 0, 1);
3526         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
3527         let chan_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
3528         create_announced_chan_between_nodes(&nodes, 2, 3);
3529
3530         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3]);
3531         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3532                 .with_bolt11_features(nodes[1].node.bolt11_invoice_features()).unwrap();
3533         let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, 10_000_000);
3534         let mut route = nodes[0].router.find_route(&nodes[0].node.get_our_node_id(), &route_params,
3535                 None, nodes[0].node.compute_inflight_htlcs()).unwrap();
3536         // Make sure the route is ordered as the B->D path before C->D
3537         route.paths.sort_by(|a, _| if a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
3538                 std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater });
3539
3540         // Note that we add an extra 1 in the send pipeline to compensate for any blocks found while
3541         // the HTLC is being relayed.
3542         route.paths[0].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 8;
3543         route.paths[1].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 12;
3544         let final_cltv = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 8 + 1;
3545
3546         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3547         nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
3548                 PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(1)).unwrap();
3549         check_added_monitors(&nodes[0], 2);
3550         let mut send_msgs = nodes[0].node.get_and_clear_pending_msg_events();
3551         send_msgs.sort_by(|a, _| {
3552                 let a_node_id =
3553                         if let MessageSendEvent::UpdateHTLCs { node_id, .. } = a { node_id } else { panic!() };
3554                 let node_b_id = nodes[1].node.get_our_node_id();
3555                 if *a_node_id == node_b_id { std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater }
3556         });
3557
3558         assert_eq!(send_msgs.len(), 2);
3559         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 10_000_000,
3560                 payment_hash, Some(payment_secret), send_msgs.remove(0), false, None);
3561         let receive_event = pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 10_000_000,
3562                 payment_hash, Some(payment_secret), send_msgs.remove(0), true, None);
3563
3564         match receive_event.unwrap() {
3565                 Event::PaymentClaimable { claim_deadline, .. } => {
3566                         assert_eq!(claim_deadline.unwrap(), final_cltv - HTLC_FAIL_BACK_BUFFER);
3567                 },
3568                 _ => panic!(),
3569         }
3570
3571         // Ensure that the claim_deadline is correct, with the payment failing at exactly the given
3572         // height.
3573         connect_blocks(&nodes[3], final_cltv - HTLC_FAIL_BACK_BUFFER - nodes[3].best_block_info().1
3574                 - if fail_payment { 0 } else { 2 });
3575         if fail_payment {
3576                 // We fail the HTLC on the A->B->D path first as it expires 4 blocks earlier. We go ahead
3577                 // and expire both immediately, though, by connecting another 4 blocks.
3578                 let reason = HTLCDestination::FailedPayment { payment_hash };
3579                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason.clone()]);
3580                 connect_blocks(&nodes[3], 4);
3581                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason]);
3582                 pass_failed_payment_back(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
3583         } else {
3584                 nodes[1].node.force_close_broadcasting_latest_txn(&chan_bd, &nodes[3].node.get_our_node_id()).unwrap();
3585                 check_closed_event!(&nodes[1], 1, ClosureReason::HolderForceClosed, false,
3586                         [nodes[3].node.get_our_node_id()], 1000000);
3587                 check_closed_broadcast(&nodes[1], 1, true);
3588                 let bs_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3589                 assert_eq!(bs_tx.len(), 1);
3590
3591                 mine_transaction(&nodes[3], &bs_tx[0]);
3592                 check_added_monitors(&nodes[3], 1);
3593                 check_closed_broadcast(&nodes[3], 1, true);
3594                 check_closed_event!(&nodes[3], 1, ClosureReason::CommitmentTxConfirmed, false,
3595                         [nodes[1].node.get_our_node_id()], 1000000);
3596
3597                 nodes[3].node.claim_funds(payment_preimage);
3598                 check_added_monitors(&nodes[3], 2);
3599                 expect_payment_claimed!(nodes[3], payment_hash, 10_000_000);
3600
3601                 let ds_tx = nodes[3].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
3602                 assert_eq!(ds_tx.len(), 1);
3603                 check_spends!(&ds_tx[0], &bs_tx[0]);
3604
3605                 mine_transactions(&nodes[1], &[&bs_tx[0], &ds_tx[0]]);
3606                 check_added_monitors(&nodes[1], 1);
3607                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[3], Some(1000), false, true);
3608
3609                 let bs_claims = nodes[1].node.get_and_clear_pending_msg_events();
3610                 check_added_monitors(&nodes[1], 1);
3611                 assert_eq!(bs_claims.len(), 1);
3612                 if let MessageSendEvent::UpdateHTLCs { updates, .. } = &bs_claims[0] {
3613                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3614                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
3615                 } else { panic!(); }
3616
3617                 expect_payment_sent!(nodes[0], payment_preimage);
3618
3619                 let ds_claim_msgs = nodes[3].node.get_and_clear_pending_msg_events();
3620                 assert_eq!(ds_claim_msgs.len(), 1);
3621                 let cs_claim_msgs = if let MessageSendEvent::UpdateHTLCs { updates, .. } = &ds_claim_msgs[0] {
3622                         nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3623                         let cs_claim_msgs = nodes[2].node.get_and_clear_pending_msg_events();
3624                         check_added_monitors(&nodes[2], 1);
3625                         commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
3626                         expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
3627                         cs_claim_msgs
3628                 } else { panic!(); };
3629
3630                 assert_eq!(cs_claim_msgs.len(), 1);
3631                 if let MessageSendEvent::UpdateHTLCs { updates, .. } = &cs_claim_msgs[0] {
3632                         nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
3633                         commitment_signed_dance!(nodes[0], nodes[2], updates.commitment_signed, false, true);
3634                 } else { panic!(); }
3635
3636                 expect_payment_path_successful!(nodes[0]);
3637         }
3638 }
3639
3640 #[test]
3641 fn claim_from_closed_chan() {
3642         do_claim_from_closed_chan(true);
3643         do_claim_from_closed_chan(false);
3644 }
3645
3646 #[test]
3647 fn test_custom_tlvs_basic() {
3648         do_test_custom_tlvs(false, false, false);
3649         do_test_custom_tlvs(true, false, false);
3650 }
3651
3652 #[test]
3653 fn test_custom_tlvs_explicit_claim() {
3654         // Test that when receiving even custom TLVs the user must explicitly accept in case they
3655         // are unknown.
3656         do_test_custom_tlvs(false, true, false);
3657         do_test_custom_tlvs(false, true, true);
3658 }
3659
3660 fn do_test_custom_tlvs(spontaneous: bool, even_tlvs: bool, known_tlvs: bool) {
3661         let chanmon_cfgs = create_chanmon_cfgs(2);
3662         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
3663         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None; 2]);
3664         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
3665
3666         create_announced_chan_between_nodes(&nodes, 0, 1);
3667
3668         let amt_msat = 100_000;
3669         let (mut route, our_payment_hash, our_payment_preimage, our_payment_secret) = get_route_and_payment_hash!(&nodes[0], &nodes[1], amt_msat);
3670         let payment_id = PaymentId(our_payment_hash.0);
3671         let custom_tlvs = vec![
3672                 (if even_tlvs { 5482373482 } else { 5482373483 }, vec![1, 2, 3, 4]),
3673                 (5482373487, vec![0x42u8; 16]),
3674         ];
3675         let onion_fields = RecipientOnionFields {
3676                 payment_secret: if spontaneous { None } else { Some(our_payment_secret) },
3677                 payment_metadata: None,
3678                 custom_tlvs: custom_tlvs.clone()
3679         };
3680         if spontaneous {
3681                 nodes[0].node.send_spontaneous_payment(&route, Some(our_payment_preimage), onion_fields, payment_id).unwrap();
3682         } else {
3683                 nodes[0].node.send_payment_with_route(&route, our_payment_hash, onion_fields, payment_id).unwrap();
3684         }
3685         check_added_monitors(&nodes[0], 1);
3686
3687         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
3688         let ev = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
3689         let mut payment_event = SendEvent::from_event(ev);
3690
3691         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
3692         check_added_monitors!(&nodes[1], 0);
3693         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
3694         expect_pending_htlcs_forwardable!(nodes[1]);
3695
3696         let events = nodes[1].node.get_and_clear_pending_events();
3697         assert_eq!(events.len(), 1);
3698         match events[0] {
3699                 Event::PaymentClaimable { ref onion_fields, .. } => {
3700                         assert_eq!(onion_fields.clone().unwrap().custom_tlvs().clone(), custom_tlvs);
3701                 },
3702                 _ => panic!("Unexpected event"),
3703         }
3704
3705         match (known_tlvs, even_tlvs) {
3706                 (true, _) => {
3707                         nodes[1].node.claim_funds_with_known_custom_tlvs(our_payment_preimage);
3708                         let expected_total_fee_msat = pass_claimed_payment_along_route(&nodes[0], &[&[&nodes[1]]], &[0; 1], false, our_payment_preimage);
3709                         expect_payment_sent!(&nodes[0], our_payment_preimage, Some(expected_total_fee_msat));
3710                 },
3711                 (false, false) => {
3712                         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
3713                 },
3714                 (false, true) => {
3715                         nodes[1].node.claim_funds(our_payment_preimage);
3716                         let expected_destinations = vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }];
3717                         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], expected_destinations);
3718                         pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, our_payment_hash, PaymentFailureReason::RecipientRejected);
3719                 }
3720         }
3721 }
3722
3723 #[test]
3724 fn test_retry_custom_tlvs() {
3725         // Test that custom TLVs are successfully sent on retries
3726         let chanmon_cfgs = create_chanmon_cfgs(3);
3727         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
3728         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
3729         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
3730
3731         create_announced_chan_between_nodes(&nodes, 0, 1);
3732         let (chan_2_update, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 2, 1);
3733
3734         // Rebalance
3735         send_payment(&nodes[2], &vec!(&nodes[1])[..], 1_500_000);
3736
3737         let amt_msat = 1_000_000;
3738         let (mut route, payment_hash, payment_preimage, payment_secret) =
3739                 get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat);
3740
3741         // Initiate the payment
3742         let payment_id = PaymentId(payment_hash.0);
3743         let mut route_params = route.route_params.clone().unwrap();
3744
3745         let custom_tlvs = vec![((1 << 16) + 1, vec![0x42u8; 16])];
3746         let onion_fields = RecipientOnionFields::secret_only(payment_secret);
3747         let onion_fields = onion_fields.with_custom_tlvs(custom_tlvs.clone()).unwrap();
3748
3749         nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
3750         nodes[0].node.send_payment(payment_hash, onion_fields,
3751                 payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
3752         check_added_monitors!(nodes[0], 1); // one monitor per path
3753
3754         // Add the HTLC along the first hop.
3755         let htlc_updates = get_htlc_update_msgs(&nodes[0], &nodes[1].node.get_our_node_id());
3756         let msgs::CommitmentUpdate { update_add_htlcs, commitment_signed, .. } = htlc_updates;
3757         assert_eq!(update_add_htlcs.len(), 1);
3758         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_htlcs[0]);
3759         commitment_signed_dance!(nodes[1], nodes[0], commitment_signed, false);
3760
3761         // Attempt to forward the payment and complete the path's failure.
3762         expect_pending_htlcs_forwardable!(&nodes[1]);
3763         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1],
3764                 vec![HTLCDestination::NextHopChannel {
3765                         node_id: Some(nodes[2].node.get_our_node_id()),
3766                         channel_id: chan_2_id
3767                 }]);
3768         check_added_monitors!(nodes[1], 1);
3769
3770         let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
3771         let msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. } = htlc_updates;
3772         assert_eq!(update_fail_htlcs.len(), 1);
3773         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
3774         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
3775
3776         let mut events = nodes[0].node.get_and_clear_pending_events();
3777         match events[1] {
3778                 Event::PendingHTLCsForwardable { .. } => {},
3779                 _ => panic!("Unexpected event")
3780         }
3781         events.remove(1);
3782         expect_payment_failed_conditions_event(events, payment_hash, false,
3783                 PaymentFailedConditions::new().mpp_parts_remain());
3784
3785         // Rebalance the channel so the retry of the payment can succeed.
3786         send_payment(&nodes[2], &vec!(&nodes[1])[..], 1_500_000);
3787
3788         // Retry the payment and make sure it succeeds
3789         route_params.payment_params.previously_failed_channels.push(chan_2_update.contents.short_channel_id);
3790         route.route_params = Some(route_params.clone());
3791         nodes[0].router.expect_find_route(route_params, Ok(route));
3792         nodes[0].node.process_pending_htlc_forwards();
3793         check_added_monitors!(nodes[0], 1);
3794         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
3795         assert_eq!(events.len(), 1);
3796         let payment_claimable = pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000,
3797                 payment_hash, Some(payment_secret), events.pop().unwrap(), true, None).unwrap();
3798         match payment_claimable {
3799                 Event::PaymentClaimable { onion_fields, .. } => {
3800                         assert_eq!(&onion_fields.unwrap().custom_tlvs()[..], &custom_tlvs[..]);
3801                 },
3802                 _ => panic!("Unexpected event"),
3803         };
3804         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
3805 }
3806
3807 #[test]
3808 fn test_custom_tlvs_consistency() {
3809         let even_type_1 = 1 << 16;
3810         let odd_type_1  = (1 << 16)+ 1;
3811         let even_type_2 = (1 << 16) + 2;
3812         let odd_type_2  = (1 << 16) + 3;
3813         let value_1 = || vec![1, 2, 3, 4];
3814         let differing_value_1 = || vec![1, 2, 3, 5];
3815         let value_2 = || vec![42u8; 16];
3816
3817         // Drop missing odd tlvs
3818         do_test_custom_tlvs_consistency(
3819                 vec![(odd_type_1, value_1()), (odd_type_2, value_2())],
3820                 vec![(odd_type_1, value_1())],
3821                 Some(vec![(odd_type_1, value_1())]),
3822         );
3823         // Drop non-matching odd tlvs
3824         do_test_custom_tlvs_consistency(
3825                 vec![(odd_type_1, value_1()), (odd_type_2, value_2())],
3826                 vec![(odd_type_1, differing_value_1()), (odd_type_2, value_2())],
3827                 Some(vec![(odd_type_2, value_2())]),
3828         );
3829         // Fail missing even tlvs
3830         do_test_custom_tlvs_consistency(
3831                 vec![(odd_type_1, value_1()), (even_type_2, value_2())],
3832                 vec![(odd_type_1, value_1())],
3833                 None,
3834         );
3835         // Fail non-matching even tlvs
3836         do_test_custom_tlvs_consistency(
3837                 vec![(even_type_1, value_1()), (odd_type_2, value_2())],
3838                 vec![(even_type_1, differing_value_1()), (odd_type_2, value_2())],
3839                 None,
3840         );
3841 }
3842
3843 fn do_test_custom_tlvs_consistency(first_tlvs: Vec<(u64, Vec<u8>)>, second_tlvs: Vec<(u64, Vec<u8>)>,
3844         expected_receive_tlvs: Option<Vec<(u64, Vec<u8>)>>) {
3845
3846         let chanmon_cfgs = create_chanmon_cfgs(4);
3847         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3848         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
3849         let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3850
3851         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
3852         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0);
3853         create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0);
3854         let chan_2_3 = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0);
3855
3856         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3857                 .with_bolt11_features(nodes[3].node.bolt11_invoice_features()).unwrap();
3858         let mut route = get_route!(nodes[0], payment_params, 15_000_000).unwrap();
3859         assert_eq!(route.paths.len(), 2);
3860         route.paths.sort_by(|path_a, _| {
3861                 // Sort the path so that the path through nodes[1] comes first
3862                 if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
3863                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
3864         });
3865
3866         let (our_payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(&nodes[3]);
3867         let payment_id = PaymentId([42; 32]);
3868         let amt_msat = 15_000_000;
3869
3870         // Send first part
3871         let onion_fields = RecipientOnionFields {
3872                 payment_secret: Some(our_payment_secret),
3873                 payment_metadata: None,
3874                 custom_tlvs: first_tlvs
3875         };
3876         let session_privs = nodes[0].node.test_add_new_pending_payment(our_payment_hash,
3877                         onion_fields.clone(), payment_id, &route).unwrap();
3878         let cur_height = nodes[0].best_block_info().1;
3879         nodes[0].node.test_send_payment_along_path(&route.paths[0], &our_payment_hash,
3880                 onion_fields.clone(), amt_msat, cur_height, payment_id,
3881                 &None, session_privs[0]).unwrap();
3882         check_added_monitors!(nodes[0], 1);
3883
3884         {
3885                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
3886                 assert_eq!(events.len(), 1);
3887                 pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], amt_msat, our_payment_hash,
3888                         Some(our_payment_secret), events.pop().unwrap(), false, None);
3889         }
3890         assert!(nodes[3].node.get_and_clear_pending_events().is_empty());
3891
3892         // Send second part
3893         let onion_fields = RecipientOnionFields {
3894                 payment_secret: Some(our_payment_secret),
3895                 payment_metadata: None,
3896                 custom_tlvs: second_tlvs
3897         };
3898         nodes[0].node.test_send_payment_along_path(&route.paths[1], &our_payment_hash,
3899                 onion_fields.clone(), amt_msat, cur_height, payment_id, &None, session_privs[1]).unwrap();
3900         check_added_monitors!(nodes[0], 1);
3901
3902         {
3903                 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
3904                 assert_eq!(events.len(), 1);
3905                 let payment_event = SendEvent::from_event(events.pop().unwrap());
3906
3907                 nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
3908                 commitment_signed_dance!(nodes[2], nodes[0], payment_event.commitment_msg, false);
3909
3910                 expect_pending_htlcs_forwardable!(nodes[2]);
3911                 check_added_monitors!(nodes[2], 1);
3912
3913                 let mut events = nodes[2].node.get_and_clear_pending_msg_events();
3914                 assert_eq!(events.len(), 1);
3915                 let payment_event = SendEvent::from_event(events.pop().unwrap());
3916
3917                 nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
3918                 check_added_monitors!(nodes[3], 0);
3919                 commitment_signed_dance!(nodes[3], nodes[2], payment_event.commitment_msg, true, true);
3920         }
3921         expect_pending_htlcs_forwardable_ignore!(nodes[3]);
3922         nodes[3].node.process_pending_htlc_forwards();
3923
3924         if let Some(expected_tlvs) = expected_receive_tlvs {
3925                 // Claim and match expected
3926                 let events = nodes[3].node.get_and_clear_pending_events();
3927                 assert_eq!(events.len(), 1);
3928                 match events[0] {
3929                         Event::PaymentClaimable { ref onion_fields, .. } => {
3930                                 assert_eq!(onion_fields.clone().unwrap().custom_tlvs, expected_tlvs);
3931                         },
3932                         _ => panic!("Unexpected event"),
3933                 }
3934
3935                 do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]],
3936                         false, our_payment_preimage);
3937                 expect_payment_sent(&nodes[0], our_payment_preimage, Some(Some(2000)), true, true);
3938         } else {
3939                 // Expect fail back
3940                 let expected_destinations = vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }];
3941                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], expected_destinations);
3942                 check_added_monitors!(nodes[3], 1);
3943
3944                 let fail_updates_1 = get_htlc_update_msgs!(nodes[3], nodes[2].node.get_our_node_id());
3945                 nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &fail_updates_1.update_fail_htlcs[0]);
3946                 commitment_signed_dance!(nodes[2], nodes[3], fail_updates_1.commitment_signed, false);
3947
3948                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![
3949                         HTLCDestination::NextHopChannel {
3950                                 node_id: Some(nodes[3].node.get_our_node_id()),
3951                                 channel_id: chan_2_3.2
3952                         }]);
3953                 check_added_monitors!(nodes[2], 1);
3954
3955                 let fail_updates_2 = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
3956                 nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &fail_updates_2.update_fail_htlcs[0]);
3957                 commitment_signed_dance!(nodes[0], nodes[2], fail_updates_2.commitment_signed, false);
3958
3959                 expect_payment_failed_conditions(&nodes[0], our_payment_hash, true,
3960                         PaymentFailedConditions::new().mpp_parts_remain());
3961         }
3962 }
3963
3964 fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) {
3965         // Check that a payment metadata received on one HTLC that doesn't match the one received on
3966         // another results in the HTLC being rejected.
3967         //
3968         // We first set up a diamond shaped network, allowing us to split a payment into two HTLCs, the
3969         // first of which we'll deliver and the second of which we'll fail and then re-send with
3970         // modified payment metadata, which will in turn result in it being failed by the recipient.
3971         let chanmon_cfgs = create_chanmon_cfgs(4);
3972         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
3973         let persister;
3974         let new_chain_monitor;
3975
3976         let mut config = test_default_channel_config();
3977         config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 50;
3978         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, Some(config), Some(config), Some(config)]);
3979         let nodes_0_deserialized;
3980
3981         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
3982
3983         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
3984         let chan_id_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
3985         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
3986         let chan_id_cd = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
3987
3988         // Pay more than half of each channel's max, requiring MPP
3989         let amt_msat = 750_000_000;
3990         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3], Some(amt_msat));
3991         let payment_id = PaymentId(payment_hash.0);
3992         let payment_metadata = vec![44, 49, 52, 142];
3993
3994         let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
3995                 .with_bolt11_features(nodes[1].node.bolt11_invoice_features()).unwrap();
3996         let mut route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
3997
3998         // Send the MPP payment, delivering the updated commitment state to nodes[1].
3999         nodes[0].node.send_payment(payment_hash, RecipientOnionFields {
4000                         payment_secret: Some(payment_secret), payment_metadata: Some(payment_metadata), custom_tlvs: vec![],
4001                 }, payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
4002         check_added_monitors!(nodes[0], 2);
4003
4004         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
4005         assert_eq!(send_events.len(), 2);
4006         let first_send = SendEvent::from_event(send_events.pop().unwrap());
4007         let second_send = SendEvent::from_event(send_events.pop().unwrap());
4008
4009         let (b_recv_ev, c_recv_ev) = if first_send.node_id == nodes[1].node.get_our_node_id() {
4010                 (&first_send, &second_send)
4011         } else {
4012                 (&second_send, &first_send)
4013         };
4014         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &b_recv_ev.msgs[0]);
4015         commitment_signed_dance!(nodes[1], nodes[0], b_recv_ev.commitment_msg, false, true);
4016
4017         expect_pending_htlcs_forwardable!(nodes[1]);
4018         check_added_monitors(&nodes[1], 1);
4019         let b_forward_ev = SendEvent::from_node(&nodes[1]);
4020         nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &b_forward_ev.msgs[0]);
4021         commitment_signed_dance!(nodes[3], nodes[1], b_forward_ev.commitment_msg, false, true);
4022
4023         expect_pending_htlcs_forwardable!(nodes[3]);
4024
4025         // Before delivering the second MPP HTLC to nodes[2], disconnect nodes[2] and nodes[3], which
4026         // will result in nodes[2] failing the HTLC back.
4027         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
4028         nodes[3].node.peer_disconnected(&nodes[2].node.get_our_node_id());
4029
4030         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &c_recv_ev.msgs[0]);
4031         commitment_signed_dance!(nodes[2], nodes[0], c_recv_ev.commitment_msg, false, true);
4032
4033         let cs_fail = get_htlc_update_msgs(&nodes[2], &nodes[0].node.get_our_node_id());
4034         nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail.update_fail_htlcs[0]);
4035         commitment_signed_dance!(nodes[0], nodes[2], cs_fail.commitment_signed, false, true);
4036
4037         let payment_fail_retryable_evs = nodes[0].node.get_and_clear_pending_events();
4038         assert_eq!(payment_fail_retryable_evs.len(), 2);
4039         if let Event::PaymentPathFailed { .. } = payment_fail_retryable_evs[0] {} else { panic!(); }
4040         if let Event::PendingHTLCsForwardable { .. } = payment_fail_retryable_evs[1] {} else { panic!(); }
4041
4042         // Before we allow the HTLC to be retried, optionally change the payment_metadata we have
4043         // stored for our payment.
4044         if do_modify {
4045                 nodes[0].node.test_set_payment_metadata(payment_id, Some(Vec::new()));
4046         }
4047
4048         // Optionally reload nodes[3] to check that the payment_metadata is properly serialized with
4049         // the payment state.
4050         if do_reload {
4051                 let mon_bd = get_monitor!(nodes[3], chan_id_bd).encode();
4052                 let mon_cd = get_monitor!(nodes[3], chan_id_cd).encode();
4053                 reload_node!(nodes[3], config, &nodes[3].node.encode(), &[&mon_bd, &mon_cd],
4054                         persister, new_chain_monitor, nodes_0_deserialized);
4055                 nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
4056                 reconnect_nodes(ReconnectArgs::new(&nodes[1], &nodes[3]));
4057         }
4058         let mut reconnect_args = ReconnectArgs::new(&nodes[2], &nodes[3]);
4059         reconnect_args.send_channel_ready = (true, true);
4060         reconnect_nodes(reconnect_args);
4061
4062         // Create a new channel between C and D as A will refuse to retry on the existing one because
4063         // it just failed.
4064         let chan_id_cd_2 = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
4065
4066         // Now retry the failed HTLC.
4067         nodes[0].node.process_pending_htlc_forwards();
4068         check_added_monitors(&nodes[0], 1);
4069         let as_resend = SendEvent::from_node(&nodes[0]);
4070         nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_resend.msgs[0]);
4071         commitment_signed_dance!(nodes[2], nodes[0], as_resend.commitment_msg, false, true);
4072
4073         expect_pending_htlcs_forwardable!(nodes[2]);
4074         check_added_monitors(&nodes[2], 1);
4075         let cs_forward = SendEvent::from_node(&nodes[2]);
4076         nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &cs_forward.msgs[0]);
4077         commitment_signed_dance!(nodes[3], nodes[2], cs_forward.commitment_msg, false, true);
4078
4079         // Finally, check that nodes[3] does the correct thing - either accepting the payment or, if
4080         // the payment metadata was modified, failing only the one modified HTLC and retaining the
4081         // other.
4082         if do_modify {
4083                 expect_pending_htlcs_forwardable_ignore!(nodes[3]);
4084                 nodes[3].node.process_pending_htlc_forwards();
4085                 expect_pending_htlcs_forwardable_conditions(nodes[3].node.get_and_clear_pending_events(),
4086                         &[HTLCDestination::FailedPayment {payment_hash}]);
4087                 nodes[3].node.process_pending_htlc_forwards();
4088
4089                 check_added_monitors(&nodes[3], 1);
4090                 let ds_fail = get_htlc_update_msgs(&nodes[3], &nodes[2].node.get_our_node_id());
4091
4092                 nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &ds_fail.update_fail_htlcs[0]);
4093                 commitment_signed_dance!(nodes[2], nodes[3], ds_fail.commitment_signed, false, true);
4094                 expect_pending_htlcs_forwardable_conditions(nodes[2].node.get_and_clear_pending_events(),
4095                         &[HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_id_cd_2 }]);
4096         } else {
4097                 expect_pending_htlcs_forwardable!(nodes[3]);
4098                 expect_payment_claimable!(nodes[3], payment_hash, payment_secret, amt_msat);
4099                 claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
4100         }
4101 }
4102
4103 #[test]
4104 fn test_payment_metadata_consistency() {
4105         do_test_payment_metadata_consistency(true, true);
4106         do_test_payment_metadata_consistency(true, false);
4107         do_test_payment_metadata_consistency(false, true);
4108         do_test_payment_metadata_consistency(false, false);
4109 }
4110
4111 #[test]
4112 fn  test_htlc_forward_considers_anchor_outputs_value() {
4113         // Tests that:
4114         //
4115         // 1) Forwarding nodes don't forward HTLCs that would cause their balance to dip below the
4116         //    reserve when considering the value of anchor outputs.
4117         //
4118         // 2) Recipients of `update_add_htlc` properly reject HTLCs that would cause the initiator's
4119         //    balance to dip below the reserve when considering the value of anchor outputs.
4120         let mut config = test_default_channel_config();
4121         config.channel_handshake_config.negotiate_anchors_zero_fee_htlc_tx = true;
4122         config.manually_accept_inbound_channels = true;
4123         config.channel_config.forwarding_fee_base_msat = 0;
4124         config.channel_config.forwarding_fee_proportional_millionths = 0;
4125
4126         // Set up a test network of three nodes that replicates a production failure leading to the
4127         // discovery of this bug.
4128         let chanmon_cfgs = create_chanmon_cfgs(3);
4129         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
4130         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[Some(config), Some(config), Some(config)]);
4131         let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
4132
4133         const CHAN_AMT: u64 = 1_000_000;
4134         const PUSH_MSAT: u64 = 900_000_000;
4135         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, CHAN_AMT, 500_000_000);
4136         let (_, _, chan_id_2, _) = create_announced_chan_between_nodes_with_value(&nodes, 1, 2, CHAN_AMT, PUSH_MSAT);
4137
4138         let channel_reserve_msat = get_holder_selected_channel_reserve_satoshis(CHAN_AMT, &config) * 1000;
4139         let commitment_fee_msat = commit_tx_fee_msat(
4140                 *nodes[1].fee_estimator.sat_per_kw.lock().unwrap(), 2, &ChannelTypeFeatures::anchors_zero_htlc_fee_and_dependencies()
4141         );
4142         let anchor_outpus_value_msat = ANCHOR_OUTPUT_VALUE_SATOSHI * 2 * 1000;
4143         let sendable_balance_msat = CHAN_AMT * 1000 - PUSH_MSAT - channel_reserve_msat - commitment_fee_msat - anchor_outpus_value_msat;
4144         let channel_details = nodes[1].node.list_channels().into_iter().find(|channel| channel.channel_id == chan_id_2).unwrap();
4145         assert!(sendable_balance_msat >= channel_details.next_outbound_htlc_minimum_msat);
4146         assert!(sendable_balance_msat <= channel_details.next_outbound_htlc_limit_msat);
4147
4148         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], sendable_balance_msat);
4149         send_payment(&nodes[2], &[&nodes[1], &nodes[0]], sendable_balance_msat);
4150
4151         // Send out an HTLC that would cause the forwarding node to dip below its reserve when
4152         // considering the value of anchor outputs.
4153         let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(
4154                 nodes[0], nodes[2], sendable_balance_msat + anchor_outpus_value_msat
4155         );
4156         nodes[0].node.send_payment_with_route(
4157                 &route, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
4158         ).unwrap();
4159         check_added_monitors!(nodes[0], 1);
4160
4161         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
4162         assert_eq!(events.len(), 1);
4163         let mut update_add_htlc = if let MessageSendEvent::UpdateHTLCs { updates, .. } = events.pop().unwrap() {
4164                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &updates.update_add_htlcs[0]);
4165                 check_added_monitors(&nodes[1], 0);
4166                 commitment_signed_dance!(nodes[1], nodes[0], &updates.commitment_signed, false);
4167                 updates.update_add_htlcs[0].clone()
4168         } else {
4169                 panic!("Unexpected event");
4170         };
4171
4172         // The forwarding node should reject forwarding it as expected.
4173         expect_pending_htlcs_forwardable!(nodes[1]);
4174         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[1], vec![HTLCDestination::NextHopChannel {
4175                 node_id: Some(nodes[2].node.get_our_node_id()),
4176                 channel_id: chan_id_2
4177         }]);
4178         check_added_monitors(&nodes[1], 1);
4179
4180         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
4181         assert_eq!(events.len(), 1);
4182         if let MessageSendEvent::UpdateHTLCs { updates, .. } = events.pop().unwrap() {
4183                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
4184                 check_added_monitors(&nodes[0], 0);
4185                 commitment_signed_dance!(nodes[0], nodes[1], &updates.commitment_signed, false);
4186         } else {
4187                 panic!("Unexpected event");
4188         }
4189
4190         expect_payment_failed!(nodes[0], payment_hash, false);
4191
4192         // Assume that the forwarding node did forward it, and make sure the recipient rejects it as an
4193         // invalid update and closes the channel.
4194         update_add_htlc.channel_id = chan_id_2;
4195         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &update_add_htlc);
4196         check_closed_event(&nodes[2], 1, ClosureReason::ProcessingError {
4197                 err: "Remote HTLC add would put them under remote reserve value".to_owned()
4198         }, false, &[nodes[1].node.get_our_node_id()], 1_000_000);
4199         check_closed_broadcast(&nodes[2], 1, true);
4200         check_added_monitors(&nodes[2], 1);
4201 }
4202
4203 #[test]
4204 fn peel_payment_onion_custom_tlvs() {
4205         let chanmon_cfgs = create_chanmon_cfgs(2);
4206         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
4207         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
4208         let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
4209         create_announced_chan_between_nodes(&nodes, 0, 1);
4210         let secp_ctx = Secp256k1::new();
4211
4212         let amt_msat = 1000;
4213         let payment_params = PaymentParameters::for_keysend(nodes[1].node.get_our_node_id(),
4214                 TEST_FINAL_CLTV, false);
4215         let route_params = RouteParameters::from_payment_params_and_value(payment_params, amt_msat);
4216         let route = functional_test_utils::get_route(&nodes[0], &route_params).unwrap();
4217         let mut recipient_onion = RecipientOnionFields::spontaneous_empty()
4218                 .with_custom_tlvs(vec![(414141, vec![42; 1200])]).unwrap();
4219         let prng_seed = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
4220         let session_priv = SecretKey::from_slice(&prng_seed[..]).expect("RNG is busted");
4221         let keysend_preimage = PaymentPreimage([42; 32]);
4222         let payment_hash = PaymentHash(Sha256::hash(&keysend_preimage.0).to_byte_array());
4223
4224         let (onion_routing_packet, first_hop_msat, cltv_expiry) = onion_utils::create_payment_onion(
4225                 &secp_ctx, &route.paths[0], &session_priv, amt_msat, recipient_onion.clone(),
4226                 nodes[0].best_block_info().1, &payment_hash, &Some(keysend_preimage), prng_seed
4227         ).unwrap();
4228
4229         let update_add = msgs::UpdateAddHTLC {
4230                 channel_id: ChannelId([0; 32]),
4231                 htlc_id: 42,
4232                 amount_msat: first_hop_msat,
4233                 payment_hash,
4234                 cltv_expiry,
4235                 skimmed_fee_msat: None,
4236                 onion_routing_packet,
4237                 blinding_point: None,
4238         };
4239         let peeled_onion = crate::ln::onion_payment::peel_payment_onion(
4240                 &update_add, &&chanmon_cfgs[1].keys_manager, &&chanmon_cfgs[1].logger, &secp_ctx,
4241                 nodes[1].best_block_info().1, true, false
4242         ).unwrap();
4243         assert_eq!(peeled_onion.incoming_amt_msat, Some(amt_msat));
4244         match peeled_onion.routing {
4245                 PendingHTLCRouting::ReceiveKeysend {
4246                         payment_data, payment_metadata, custom_tlvs, ..
4247                 } => {
4248                         #[cfg(not(c_bindings))]
4249                         assert_eq!(&custom_tlvs, recipient_onion.custom_tlvs());
4250                         #[cfg(c_bindings)]
4251                         assert_eq!(custom_tlvs, recipient_onion.custom_tlvs());
4252                         assert!(payment_metadata.is_none());
4253                         assert!(payment_data.is_none());
4254                 },
4255                 _ => panic!()
4256         }
4257 }