Add `TestStore` implementation of `KVStore`
[rust-lightning] / lightning / src / ln / payment_tests.rs
index fc1b181e9e74d91ce4e3a4bf09360a4198432a1d..f0c3fa92d37c4b07f3c40d641e011e70d8b87c2e 100644 (file)
@@ -17,9 +17,9 @@ use crate::sign::EntropySource;
 use crate::chain::transaction::OutPoint;
 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentFailureReason, PaymentPurpose};
 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
-use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, ChannelManager, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS, RecentPaymentDetails, RecipientOnionFields, HTLCForwardInfo, PendingHTLCRouting, PendingAddHTLCInfo};
+use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS, RecentPaymentDetails, RecipientOnionFields, HTLCForwardInfo, PendingHTLCRouting, PendingAddHTLCInfo};
 use crate::ln::features::Bolt11InvoiceFeatures;
-use crate::ln::{msgs, PaymentSecret, PaymentPreimage};
+use crate::ln::{msgs, ChannelId, PaymentSecret, PaymentPreimage};
 use crate::ln::msgs::ChannelMessageHandler;
 use crate::ln::outbound_payment::Retry;
 use crate::routing::gossip::{EffectiveCapacity, RoutingFees};
@@ -274,22 +274,31 @@ fn do_test_keysend_payments(public_node: bool, with_retry: bool) {
                nodes[0].logger, &scorer, &(), &random_seed_bytes
        ).unwrap();
 
-       let test_preimage = PaymentPreimage([42; 32]);
-       let payment_hash = if with_retry {
-               nodes[0].node.send_spontaneous_payment_with_retry(Some(test_preimage),
-                       RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0),
-                       route_params, Retry::Attempts(1)).unwrap()
-       } else {
-               nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage),
-                       RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0)).unwrap()
-       };
+       {
+               let test_preimage = PaymentPreimage([42; 32]);
+               if with_retry {
+                       nodes[0].node.send_spontaneous_payment_with_retry(Some(test_preimage),
+                               RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0),
+                               route_params, Retry::Attempts(1)).unwrap()
+               } else {
+                       nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage),
+                               RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0)).unwrap()
+               };
+       }
        check_added_monitors!(nodes[0], 1);
-       let mut events = nodes[0].node.get_and_clear_pending_msg_events();
-       assert_eq!(events.len(), 1);
-       let event = events.pop().unwrap();
-       let path = vec![&nodes[1]];
-       pass_along_path(&nodes[0], &path, 10000, payment_hash, None, event, true, Some(test_preimage));
-       claim_payment(&nodes[0], &path, test_preimage);
+       let send_event = SendEvent::from_node(&nodes[0]);
+       nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
+       do_commitment_signed_dance(&nodes[1], &nodes[0], &send_event.commitment_msg, false, false);
+       expect_pending_htlcs_forwardable!(nodes[1]);
+       // Previously, a refactor caused us to stop including the payment preimage in the onion which
+       // is sent as a part of keysend payments. Thus, to be extra careful here, we scope the preimage
+       // above to demonstrate that we have no way to get the preimage at this point except by
+       // extracting it from the onion nodes[1] received.
+       let event = nodes[1].node.get_and_clear_pending_events();
+       assert_eq!(event.len(), 1);
+       if let Event::PaymentClaimable { purpose: PaymentPurpose::SpontaneousPayment(preimage), .. } = event[0] {
+               claim_payment(&nodes[0], &[&nodes[1]], preimage);
+       } else { panic!(); }
 }
 
 #[test]
@@ -505,10 +514,10 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
        // which has separate codepaths for "commitment transaction already confirmed" and not.
        let chanmon_cfgs = create_chanmon_cfgs(3);
        let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
+       let persister;
+       let new_chain_monitor;
        let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
-       let persister: test_utils::TestPersister;
-       let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
+       let nodes_0_deserialized;
        let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 
        let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
@@ -561,7 +570,7 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
 
        // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
        // force-close the channel.
-       check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
+       check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[1].node.get_our_node_id()], 100000);
        assert!(nodes[0].node.list_channels().is_empty());
        assert!(nodes[0].node.has_pending_payments());
        nodes[0].node.timer_tick_occurred();
@@ -593,7 +602,8 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
                MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
                        assert_eq!(node_id, nodes[1].node.get_our_node_id());
                        nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
-                       check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) });
+                       check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}",
+                               &nodes[1].node.get_our_node_id())) }, [nodes[0].node.get_our_node_id()], 100000);
                        check_added_monitors!(nodes[1], 1);
                        assert_eq!(nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
                },
@@ -633,7 +643,7 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
                mine_transaction(&nodes[0], &as_commitment_tx);
        }
        mine_transaction(&nodes[0], &bs_htlc_claim_txn[0]);
-       expect_payment_sent!(nodes[0], payment_preimage_1);
+       expect_payment_sent(&nodes[0], payment_preimage_1, None, true, false);
        connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
        let (first_htlc_timeout_tx, second_htlc_timeout_tx) = {
                let mut txn = nodes[0].tx_broadcaster.unique_txn_broadcast();
@@ -704,17 +714,17 @@ fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
        let mut manually_accept_config = test_default_channel_config();
        manually_accept_config.manually_accept_inbound_channels = true;
 
-       let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(manually_accept_config), None]);
+       let first_persister;
+       let first_new_chain_monitor;
+       let second_persister;
+       let second_new_chain_monitor;
+       let third_persister;
+       let third_new_chain_monitor;
 
-       let first_persister: test_utils::TestPersister;
-       let first_new_chain_monitor: test_utils::TestChainMonitor;
-       let first_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
-       let second_persister: test_utils::TestPersister;
-       let second_new_chain_monitor: test_utils::TestChainMonitor;
-       let second_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
-       let third_persister: test_utils::TestPersister;
-       let third_new_chain_monitor: test_utils::TestChainMonitor;
-       let third_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
+       let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(manually_accept_config), None]);
+       let first_nodes_0_deserialized;
+       let second_nodes_0_deserialized;
+       let third_nodes_0_deserialized;
 
        let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 
@@ -742,7 +752,7 @@ fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 
        // On reload, the ChannelManager should realize it is stale compared to the ChannelMonitor and
        // force-close the channel.
-       check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
+       check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[1].node.get_our_node_id()], 100000);
        nodes[0].node.timer_tick_occurred();
        assert!(nodes[0].node.list_channels().is_empty());
        assert!(nodes[0].node.has_pending_payments());
@@ -768,7 +778,8 @@ fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
                MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
                        assert_eq!(node_id, nodes[1].node.get_our_node_id());
                        nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), msg);
-                       check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) });
+                       check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) }
+                               , [nodes[0].node.get_our_node_id()], 100000);
                        check_added_monitors!(nodes[1], 1);
                        bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
                },
@@ -902,10 +913,10 @@ fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, co
        // duplicate HTLC fail/claim (e.g. via a PaymentPathFailed event).
        let chanmon_cfgs = create_chanmon_cfgs(2);
        let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let persister;
+       let new_chain_monitor;
        let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
-       let persister: test_utils::TestPersister;
-       let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
+       let nodes_0_deserialized;
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 
        let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1);
@@ -916,7 +927,7 @@ fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, co
        nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap();
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
-       check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
+       check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 100000);
 
        nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
        nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
@@ -937,7 +948,7 @@ fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, co
        connect_block(&nodes[1], &create_dummy_block(nodes[1].best_block_hash(), 42, vec![node_txn[1].clone()]));
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
-       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed, [nodes[0].node.get_our_node_id()], 100000);
        let claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(claim_txn.len(), 1);
        check_spends!(claim_txn[0], node_txn[1]);
@@ -994,7 +1005,7 @@ fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, co
        if payment_timeout {
                expect_payment_failed!(nodes[0], payment_hash, false);
        } else {
-               expect_payment_sent!(nodes[0], payment_preimage);
+               expect_payment_sent(&nodes[0], payment_preimage, None, true, false);
        }
 
        // If we persist the ChannelManager after we get the PaymentSent event, we shouldn't get it
@@ -1011,7 +1022,7 @@ fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, co
        } else if payment_timeout {
                expect_payment_failed!(nodes[0], payment_hash, false);
        } else {
-               expect_payment_sent!(nodes[0], payment_preimage);
+               expect_payment_sent(&nodes[0], payment_preimage, None, true, false);
        }
 
        // Note that if we re-connect the block which exposed nodes[0] to the payment preimage (but
@@ -1043,10 +1054,10 @@ fn test_fulfill_restart_failure() {
        // handle it, we should test the logic for it anyway. We do that here.
        let chanmon_cfgs = create_chanmon_cfgs(2);
        let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let persister;
+       let new_chain_monitor;
        let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
-       let persister: test_utils::TestPersister;
-       let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
+       let nodes_1_deserialized;
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 
        let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
@@ -1063,7 +1074,7 @@ fn test_fulfill_restart_failure() {
 
        let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
        nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
-       expect_payment_sent_without_paths!(nodes[0], payment_preimage);
+       expect_payment_sent(&nodes[0], payment_preimage, None, false, false);
 
        // Now reload nodes[1]...
        reload_node!(nodes[1], &chan_manager_serialized, &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
@@ -1692,7 +1703,7 @@ fn do_test_intercepted_payment(test: InterceptTest) {
        };
 
        // Check for unknown channel id error.
-       let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &[42; 32], nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
+       let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &ChannelId::from_bytes([42; 32]), nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
        assert_eq!(unknown_chan_id_err , APIError::ChannelUnavailable  {
                err: format!("Funded channel with id {} not found for the passed counterparty node_id {}. Channel may still be opening.",
                        log_bytes!([42; 32]), nodes[2].node.get_our_node_id()) });
@@ -1721,7 +1732,7 @@ fn do_test_intercepted_payment(test: InterceptTest) {
                let unusable_chan_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &temp_chan_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
                assert_eq!(unusable_chan_err , APIError::ChannelUnavailable {
                        err: format!("Funded channel with id {} not found for the passed counterparty node_id {}. Channel may still be opening.",
-                               log_bytes!(temp_chan_id), nodes[2].node.get_our_node_id()) });
+                               &temp_chan_id, nodes[2].node.get_our_node_id()) });
                assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
 
                // Open the just-in-time channel so the payment can then be forwarded.
@@ -1764,6 +1775,7 @@ fn do_test_intercepted_payment(test: InterceptTest) {
                        },
                        _ => panic!("Unexpected event")
                }
+               check_added_monitors(&nodes[0], 1);
        } else if test == InterceptTest::Timeout {
                let mut block = create_dummy_block(nodes[0].best_block_hash(), 42, Vec::new());
                connect_block(&nodes[0], &block);
@@ -1918,7 +1930,7 @@ fn do_accept_underpaying_htlcs_config(num_mpp_parts: usize) {
                payment_preimage);
        // The sender doesn't know that the penultimate hop took an extra fee.
        expect_payment_sent(&nodes[0], payment_preimage,
-               Some(Some(total_fee_msat - skimmed_fee_msat * num_mpp_parts as u64)), true);
+               Some(Some(total_fee_msat - skimmed_fee_msat * num_mpp_parts as u64)), true, true);
 }
 
 #[derive(PartialEq)]
@@ -1945,10 +1957,10 @@ fn do_automatic_retries(test: AutoRetry) {
        // below.
        let chanmon_cfgs = create_chanmon_cfgs(3);
        let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
-       let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
-
        let persister;
        let new_chain_monitor;
+
+       let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
        let node_0_deserialized;
 
        let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
@@ -2342,6 +2354,7 @@ fn auto_retry_partial_failure() {
        assert_eq!(bs_claim_update.update_fulfill_htlcs.len(), 1);
 
        nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_claim_update.update_fulfill_htlcs[0]);
+       expect_payment_sent(&nodes[0], payment_preimage, None, false, false);
        nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_claim_update.commitment_signed);
        check_added_monitors!(nodes[0], 1);
        let (as_third_raa, as_third_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -2356,6 +2369,7 @@ fn auto_retry_partial_failure() {
 
        nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
        check_added_monitors!(nodes[0], 1);
+       expect_payment_path_successful!(nodes[0]);
 
        nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[0]);
        nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_second_claim_update.update_fulfill_htlcs[1]);
@@ -2372,7 +2386,10 @@ fn auto_retry_partial_failure() {
 
        nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
        check_added_monitors!(nodes[0], 1);
-       expect_payment_sent!(nodes[0], payment_preimage);
+       let events = nodes[0].node.get_and_clear_pending_events();
+       assert_eq!(events.len(), 2);
+       if let Event::PaymentPathSuccessful { .. } = events[0] {} else { panic!(); }
+       if let Event::PaymentPathSuccessful { .. } = events[1] {} else { panic!(); }
 }
 
 #[test]
@@ -2527,7 +2544,7 @@ fn retry_multi_path_single_failed_payment() {
                }, Ok(route.clone()));
 
        {
-               let scorer = chanmon_cfgs[0].scorer.lock().unwrap();
+               let scorer = chanmon_cfgs[0].scorer.read().unwrap();
                // The initial send attempt, 2 paths
                scorer.expect_usage(chans[0].short_channel_id.unwrap(), ChannelUsage { amount_msat: 10_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
                scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 100_000_001, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
@@ -3161,15 +3178,15 @@ fn test_threaded_payment_retries() {
        }
 }
 
-fn do_no_missing_sent_on_midpoint_reload(persist_manager_with_payment: bool) {
+fn do_no_missing_sent_on_reload(persist_manager_with_payment: bool, at_midpoint: bool) {
        // Test that if we reload in the middle of an HTLC claim commitment signed dance we'll still
        // receive the PaymentSent event even if the ChannelManager had no idea about the payment when
        // it was last persisted.
        let chanmon_cfgs = create_chanmon_cfgs(2);
        let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
-       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
        let (persister_a, persister_b, persister_c);
        let (chain_monitor_a, chain_monitor_b, chain_monitor_c);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
        let (nodes_0_deserialized, nodes_0_deserialized_b, nodes_0_deserialized_c);
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 
@@ -3190,10 +3207,20 @@ fn do_no_missing_sent_on_midpoint_reload(persist_manager_with_payment: bool) {
        check_added_monitors!(nodes[1], 1);
        expect_payment_claimed!(nodes[1], our_payment_hash, 1_000_000);
 
-       let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
-       nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
-       nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
-       check_added_monitors!(nodes[0], 1);
+       if at_midpoint {
+               let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
+               nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
+               nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
+               check_added_monitors!(nodes[0], 1);
+       } else {
+               let htlc_fulfill_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
+               nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
+               commitment_signed_dance!(nodes[0], nodes[1], htlc_fulfill_updates.commitment_signed, false);
+               // Ignore the PaymentSent event which is now pending on nodes[0] - if we were to handle it we'd
+               // be expected to ignore the eventual conflicting PaymentFailed, but by not looking at it we
+               // expect to get the PaymentSent again later.
+               check_added_monitors(&nodes[0], 0);
+       }
 
        // The ChannelMonitor should always be the latest version, as we're required to persist it
        // during the commitment signed handling.
@@ -3239,8 +3266,14 @@ fn do_no_missing_sent_on_midpoint_reload(persist_manager_with_payment: bool) {
 
 #[test]
 fn no_missing_sent_on_midpoint_reload() {
-       do_no_missing_sent_on_midpoint_reload(false);
-       do_no_missing_sent_on_midpoint_reload(true);
+       do_no_missing_sent_on_reload(false, true);
+       do_no_missing_sent_on_reload(true, true);
+}
+
+#[test]
+fn no_missing_sent_on_reload() {
+       do_no_missing_sent_on_reload(false, false);
+       do_no_missing_sent_on_reload(true, false);
 }
 
 fn do_claim_from_closed_chan(fail_payment: bool) {
@@ -3326,7 +3359,8 @@ fn do_claim_from_closed_chan(fail_payment: bool) {
                pass_failed_payment_back(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
        } else {
                nodes[1].node.force_close_broadcasting_latest_txn(&chan_bd, &nodes[3].node.get_our_node_id()).unwrap();
-               check_closed_event(&nodes[1], 1, ClosureReason::HolderForceClosed, false);
+               check_closed_event!(&nodes[1], 1, ClosureReason::HolderForceClosed, false,
+                       [nodes[3].node.get_our_node_id()], 1000000);
                check_closed_broadcast(&nodes[1], 1, true);
                let bs_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
                assert_eq!(bs_tx.len(), 1);
@@ -3334,7 +3368,8 @@ fn do_claim_from_closed_chan(fail_payment: bool) {
                mine_transaction(&nodes[3], &bs_tx[0]);
                check_added_monitors(&nodes[3], 1);
                check_closed_broadcast(&nodes[3], 1, true);
-               check_closed_event(&nodes[3], 1, ClosureReason::CommitmentTxConfirmed, false);
+               check_closed_event!(&nodes[3], 1, ClosureReason::CommitmentTxConfirmed, false,
+                       [nodes[1].node.get_our_node_id()], 1000000);
 
                nodes[3].node.claim_funds(payment_preimage);
                check_added_monitors(&nodes[3], 2);
@@ -3386,12 +3421,20 @@ fn claim_from_closed_chan() {
 }
 
 #[test]
-fn test_custom_tlvs() {
-       do_test_custom_tlvs(true);
-       do_test_custom_tlvs(false);
+fn test_custom_tlvs_basic() {
+       do_test_custom_tlvs(false, false, false);
+       do_test_custom_tlvs(true, false, false);
+}
+
+#[test]
+fn test_custom_tlvs_explicit_claim() {
+       // Test that when receiving even custom TLVs the user must explicitly accept in case they
+       // are unknown.
+       do_test_custom_tlvs(false, true, false);
+       do_test_custom_tlvs(false, true, true);
 }
 
-fn do_test_custom_tlvs(spontaneous: bool) {
+fn do_test_custom_tlvs(spontaneous: bool, even_tlvs: bool, known_tlvs: bool) {
        let chanmon_cfgs = create_chanmon_cfgs(2);
        let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
        let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None; 2]);
@@ -3403,7 +3446,7 @@ fn do_test_custom_tlvs(spontaneous: bool) {
        let (mut route, our_payment_hash, our_payment_preimage, our_payment_secret) = get_route_and_payment_hash!(&nodes[0], &nodes[1], amt_msat);
        let payment_id = PaymentId(our_payment_hash.0);
        let custom_tlvs = vec![
-               (5482373483, vec![1, 2, 3, 4]),
+               (if even_tlvs { 5482373482 } else { 5482373483 }, vec![1, 2, 3, 4]),
                (5482373487, vec![0x42u8; 16]),
        ];
        let onion_fields = RecipientOnionFields {
@@ -3430,23 +3473,28 @@ fn do_test_custom_tlvs(spontaneous: bool) {
        let events = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
        match events[0] {
-               Event::PaymentClaimable { ref purpose, amount_msat, ref onion_fields, .. } => {
-                       match &purpose {
-                               PaymentPurpose::InvoicePayment { payment_secret, .. } => {
-                                       assert_eq!(our_payment_secret, *payment_secret);
-                                       assert_eq!(Some(*payment_secret), onion_fields.as_ref().unwrap().payment_secret);
-                               },
-                               PaymentPurpose::SpontaneousPayment(payment_preimage) => {
-                                       assert_eq!(our_payment_preimage, *payment_preimage);
-                               },
-                       }
-                       assert_eq!(amount_msat, amt_msat);
+               Event::PaymentClaimable { ref onion_fields, .. } => {
                        assert_eq!(onion_fields.clone().unwrap().custom_tlvs().clone(), custom_tlvs);
                },
                _ => panic!("Unexpected event"),
        }
 
-       claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
+       match (known_tlvs, even_tlvs) {
+               (true, _) => {
+                       nodes[1].node.claim_funds_with_known_custom_tlvs(our_payment_preimage);
+                       let expected_total_fee_msat = pass_claimed_payment_along_route(&nodes[0], &[&[&nodes[1]]], &[0; 1], false, our_payment_preimage);
+                       expect_payment_sent!(&nodes[0], our_payment_preimage, Some(expected_total_fee_msat));
+               },
+               (false, false) => {
+                       claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
+               },
+               (false, true) => {
+                       nodes[1].node.claim_funds(our_payment_preimage);
+                       let expected_destinations = vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }];
+                       expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], expected_destinations);
+                       pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, our_payment_hash, PaymentFailureReason::RecipientRejected);
+               }
+       }
 }
 
 #[test]
@@ -3482,26 +3530,12 @@ fn test_retry_custom_tlvs() {
        nodes[0].node.send_payment(payment_hash, onion_fields,
                payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
        check_added_monitors!(nodes[0], 1); // one monitor per path
-       let mut events = nodes[0].node.get_and_clear_pending_msg_events();
-       assert_eq!(events.len(), 1);
 
        // Add the HTLC along the first hop.
-       let fail_path_msgs_1 = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut events);
-       let (update_add, commitment_signed) = match fail_path_msgs_1 {
-               MessageSendEvent::UpdateHTLCs { node_id: _, updates: msgs::CommitmentUpdate {
-                       ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs,
-                       ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed }
-               } => {
-                       assert_eq!(update_add_htlcs.len(), 1);
-                       assert!(update_fail_htlcs.is_empty());
-                       assert!(update_fulfill_htlcs.is_empty());
-                       assert!(update_fail_malformed_htlcs.is_empty());
-                       assert!(update_fee.is_none());
-                       (update_add_htlcs[0].clone(), commitment_signed.clone())
-               },
-               _ => panic!("Unexpected event"),
-       };
-       nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
+       let htlc_updates = get_htlc_update_msgs(&nodes[0], &nodes[1].node.get_our_node_id());
+       let msgs::CommitmentUpdate { update_add_htlcs, commitment_signed, .. } = htlc_updates;
+       assert_eq!(update_add_htlcs.len(), 1);
+       nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_htlcs[0]);
        commitment_signed_dance!(nodes[1], nodes[0], commitment_signed, false);
 
        // Attempt to forward the payment and complete the path's failure.
@@ -3511,15 +3545,14 @@ fn test_retry_custom_tlvs() {
                        node_id: Some(nodes[2].node.get_our_node_id()),
                        channel_id: chan_2_id
                }]);
-       let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
-       assert!(htlc_updates.update_add_htlcs.is_empty());
-       assert_eq!(htlc_updates.update_fail_htlcs.len(), 1);
-       assert!(htlc_updates.update_fulfill_htlcs.is_empty());
-       assert!(htlc_updates.update_fail_malformed_htlcs.is_empty());
        check_added_monitors!(nodes[1], 1);
-       nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(),
-               &htlc_updates.update_fail_htlcs[0]);
-       commitment_signed_dance!(nodes[0], nodes[1], htlc_updates.commitment_signed, false);
+
+       let htlc_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
+       let msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. } = htlc_updates;
+       assert_eq!(update_fail_htlcs.len(), 1);
+       nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
+       commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
+
        let mut events = nodes[0].node.get_and_clear_pending_events();
        match events[1] {
                Event::PendingHTLCsForwardable { .. } => {},
@@ -3541,11 +3574,12 @@ fn test_retry_custom_tlvs() {
        assert_eq!(events.len(), 1);
        let payment_claimable = pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000,
                payment_hash, Some(payment_secret), events.pop().unwrap(), true, None).unwrap();
-       let onion_fields = match payment_claimable {
-               Event::PaymentClaimable { onion_fields, .. } => onion_fields,
+       match payment_claimable {
+               Event::PaymentClaimable { onion_fields, .. } => {
+                       assert_eq!(onion_fields.unwrap().custom_tlvs(), &custom_tlvs);
+               },
                _ => panic!("Unexpected event"),
        };
-       assert_eq!(onion_fields.unwrap().custom_tlvs(), &custom_tlvs);
        claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
 }
 
@@ -3629,7 +3663,8 @@ fn do_test_custom_tlvs_consistency(first_tlvs: Vec<(u64, Vec<u8>)>, second_tlvs:
        {
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
-               pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], amt_msat, our_payment_hash, Some(our_payment_secret), events.pop().unwrap(), false, None);
+               pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], amt_msat, our_payment_hash,
+                       Some(our_payment_secret), events.pop().unwrap(), false, None);
        }
        assert!(nodes[3].node.get_and_clear_pending_events().is_empty());
 
@@ -3668,27 +3703,17 @@ fn do_test_custom_tlvs_consistency(first_tlvs: Vec<(u64, Vec<u8>)>, second_tlvs:
        if let Some(expected_tlvs) = expected_receive_tlvs {
                // Claim and match expected
                let events = nodes[3].node.get_and_clear_pending_events();
-               println!("events: {:?}", events);
                assert_eq!(events.len(), 1);
                match events[0] {
-                       Event::PaymentClaimable { ref purpose, amount_msat, ref onion_fields, .. } => {
-                               match &purpose {
-                                       PaymentPurpose::InvoicePayment { payment_secret, .. } => {
-                                               assert_eq!(our_payment_secret, *payment_secret);
-                                               assert_eq!(Some(*payment_secret), onion_fields.as_ref().unwrap().payment_secret);
-                                       },
-                                       PaymentPurpose::SpontaneousPayment(payment_preimage) => {
-                                               assert_eq!(our_payment_preimage, *payment_preimage);
-                                       },
-                               }
-                               assert_eq!(amount_msat, amt_msat);
+                       Event::PaymentClaimable { ref onion_fields, .. } => {
                                assert_eq!(onion_fields.clone().unwrap().custom_tlvs, expected_tlvs);
                        },
                        _ => panic!("Unexpected event"),
                }
 
-               do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, our_payment_preimage);
-               expect_payment_sent(&nodes[0], our_payment_preimage, Some(Some(2000)), true);
+               do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]],
+                       false, our_payment_preimage);
+               expect_payment_sent(&nodes[0], our_payment_preimage, Some(Some(2000)), true, true);
        } else {
                // Expect fail back
                let expected_destinations = vec![HTLCDestination::FailedPayment { payment_hash: our_payment_hash }];
@@ -3699,14 +3724,19 @@ fn do_test_custom_tlvs_consistency(first_tlvs: Vec<(u64, Vec<u8>)>, second_tlvs:
                nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &fail_updates_1.update_fail_htlcs[0]);
                commitment_signed_dance!(nodes[2], nodes[3], fail_updates_1.commitment_signed, false);
 
-               expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_2_3.2 }]);
+               expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![
+                       HTLCDestination::NextHopChannel {
+                               node_id: Some(nodes[3].node.get_our_node_id()),
+                               channel_id: chan_2_3.2
+                       }]);
                check_added_monitors!(nodes[2], 1);
 
                let fail_updates_2 = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
                nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &fail_updates_2.update_fail_htlcs[0]);
                commitment_signed_dance!(nodes[0], nodes[2], fail_updates_2.commitment_signed, false);
 
-               expect_payment_failed_conditions(&nodes[0], our_payment_hash, true, PaymentFailedConditions::new().mpp_parts_remain());
+               expect_payment_failed_conditions(&nodes[0], our_payment_hash, true,
+                       PaymentFailedConditions::new().mpp_parts_remain());
        }
 }
 
@@ -3719,12 +3749,12 @@ fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) {
        // modified payment metadata, which will in turn result in it being failed by the recipient.
        let chanmon_cfgs = create_chanmon_cfgs(4);
        let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
+       let persister;
+       let new_chain_monitor;
+
        let mut config = test_default_channel_config();
        config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 50;
        let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, Some(config), Some(config), Some(config)]);
-
-       let persister;
-       let new_chain_monitor;
        let nodes_0_deserialized;
 
        let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);