Merge pull request #1738 from jkczyz/2022-09-invoice-request
authorvalentinewallace <valentinewallace@users.noreply.github.com>
Mon, 12 Dec 2022 16:25:07 +0000 (11:25 -0500)
committerGitHub <noreply@github.com>
Mon, 12 Dec 2022 16:25:07 +0000 (11:25 -0500)
BOLT 12 `invoice_request` encoding and building

37 files changed:
README.md
fuzz/src/chanmon_consistency.rs
fuzz/src/full_stack.rs
fuzz/src/onion_message.rs
fuzz/src/router.rs
lightning-background-processor/src/lib.rs
lightning-invoice/src/payment.rs
lightning-invoice/src/utils.rs
lightning/src/chain/channelmonitor.rs
lightning/src/chain/keysinterface.rs
lightning/src/chain/onchaintx.rs
lightning/src/chain/package.rs
lightning/src/ln/chan_utils.rs
lightning/src/ln/chanmon_update_fail_tests.rs
lightning/src/ln/channel.rs
lightning/src/ln/channelmanager.rs
lightning/src/ln/functional_test_utils.rs
lightning/src/ln/functional_tests.rs
lightning/src/ln/monitor_tests.rs
lightning/src/ln/onion_route_tests.rs
lightning/src/ln/onion_utils.rs
lightning/src/ln/payment_tests.rs
lightning/src/ln/priv_short_conf_tests.rs
lightning/src/ln/reload_tests.rs
lightning/src/ln/reorg_tests.rs
lightning/src/routing/router.rs
lightning/src/util/byte_utils.rs
lightning/src/util/config.rs
lightning/src/util/enforcing_trait_impls.rs
lightning/src/util/errors.rs
lightning/src/util/events.rs
lightning/src/util/persist.rs
lightning/src/util/scid_utils.rs
lightning/src/util/ser_macros.rs
lightning/src/util/test_utils.rs
pending_changelog/1856.txt [new file with mode: 0644]
pending_changelog/elias-event-renaming.txt [new file with mode: 0644]

index 824a2875b6331a51d293f47aaaa2f83c3fada7ee..7f95427a6ab13bfb29d1fe1614e5238fb28b1bcb 100644 (file)
--- a/README.md
+++ b/README.md
@@ -105,8 +105,8 @@ https://vimeo.com/showcase/8372504/video/412818125
 
 Design Goal
 -----------
-The goal is to provide a full-featured but also incredibly flexible Lightning
-implementation, allowing the user to decide how they wish to use it. With that
+The goal is to provide a fully-featured and incredibly flexible Lightning
+implementation, allowing users to decide how they wish to use it. With that
 in mind, everything should be exposed via simple, composable APIs. More
 information about `rust-lightning`'s flexibility is provided in the `About`
 section above.
index 783bbc06aa9fe58e4de90e5ce13650aeb0458b9d..605a8c29c291690cacff642d78454d00950517ee 100644 (file)
@@ -193,9 +193,14 @@ impl KeysInterface for KeyProvider {
                ShutdownScript::new_p2wpkh(&pubkey_hash)
        }
 
-       fn get_channel_signer(&self, _inbound: bool, channel_value_satoshis: u64) -> EnforcingSigner {
+       fn generate_channel_keys_id(&self, _inbound: bool, _channel_value_satoshis: u64, _user_channel_id: u128) -> [u8; 32] {
+               let id = self.rand_bytes_id.fetch_add(1, atomic::Ordering::Relaxed) as u8;
+               [id; 32]
+       }
+
+       fn derive_channel_signer(&self, channel_value_satoshis: u64, channel_keys_id: [u8; 32]) -> Self::Signer {
                let secp_ctx = Secp256k1::signing_only();
-               let id = self.rand_bytes_id.fetch_add(1, atomic::Ordering::Relaxed);
+               let id = channel_keys_id[0];
                let keys = InMemorySigner::new(
                        &secp_ctx,
                        self.get_node_secret(Recipient::Node).unwrap(),
@@ -204,9 +209,9 @@ impl KeysInterface for KeyProvider {
                        SecretKey::from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, self.node_id]).unwrap(),
                        SecretKey::from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, self.node_id]).unwrap(),
                        SecretKey::from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, self.node_id]).unwrap(),
-                       [id as u8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, self.node_id],
+                       [id, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, self.node_id],
                        channel_value_satoshis,
-                       [0; 32],
+                       channel_keys_id,
                );
                let revoked_commitment = self.make_enforcement_state_cell(keys.commitment_seed);
                EnforcingSigner::new_with_revoked(keys, revoked_commitment, false)
@@ -253,7 +258,7 @@ fn check_api_err(api_err: APIError) {
        match api_err {
                APIError::APIMisuseError { .. } => panic!("We can't misuse the API"),
                APIError::FeeRateTooHigh { .. } => panic!("We can't send too much fee?"),
-               APIError::RouteError { .. } => panic!("Our routes should work"),
+               APIError::InvalidRoute { .. } => panic!("Our routes should work"),
                APIError::ChannelUnavailable { err } => {
                        // Test the error against a list of errors we can hit, and reject
                        // all others. If you hit this panic, the list of acceptable errors
@@ -843,16 +848,16 @@ pub fn do_test<Out: Output>(data: &[u8], underlying_out: Out) {
                                let mut events = nodes[$node].get_and_clear_pending_events();
                                // Sort events so that PendingHTLCsForwardable get processed last. This avoids a
                                // case where we first process a PendingHTLCsForwardable, then claim/fail on a
-                               // PaymentReceived, claiming/failing two HTLCs, but leaving a just-generated
-                               // PaymentReceived event for the second HTLC in our pending_events (and breaking
+                               // PaymentClaimable, claiming/failing two HTLCs, but leaving a just-generated
+                               // PaymentClaimable event for the second HTLC in our pending_events (and breaking
                                // our claim_set deduplication).
                                events.sort_by(|a, b| {
-                                       if let events::Event::PaymentReceived { .. } = a {
+                                       if let events::Event::PaymentClaimable { .. } = a {
                                                if let events::Event::PendingHTLCsForwardable { .. } = b {
                                                        Ordering::Less
                                                } else { Ordering::Equal }
                                        } else if let events::Event::PendingHTLCsForwardable { .. } = a {
-                                               if let events::Event::PaymentReceived { .. } = b {
+                                               if let events::Event::PaymentClaimable { .. } = b {
                                                        Ordering::Greater
                                                } else { Ordering::Equal }
                                        } else { Ordering::Equal }
@@ -860,7 +865,7 @@ pub fn do_test<Out: Output>(data: &[u8], underlying_out: Out) {
                                let had_events = !events.is_empty();
                                for event in events.drain(..) {
                                        match event {
-                                               events::Event::PaymentReceived { payment_hash, .. } => {
+                                               events::Event::PaymentClaimable { payment_hash, .. } => {
                                                        if claim_set.insert(payment_hash.0) {
                                                                if $fail {
                                                                        nodes[$node].fail_htlc_backwards(&payment_hash);
index 193085676cdf8267ecf1e570abcf93b87904ddea..6374328238892ad3727933e64075d6561c88d07e 100644 (file)
@@ -263,6 +263,7 @@ struct KeyProvider {
        node_secret: SecretKey,
        inbound_payment_key: KeyMaterial,
        counter: AtomicU64,
+       signer_state: RefCell<HashMap<u8, (bool, Arc<Mutex<EnforcementState>>)>>
 }
 impl KeysInterface for KeyProvider {
        type Signer = EnforcingSigner;
@@ -297,10 +298,17 @@ impl KeysInterface for KeyProvider {
                ShutdownScript::new_p2wpkh(&pubkey_hash)
        }
 
-       fn get_channel_signer(&self, inbound: bool, channel_value_satoshis: u64) -> EnforcingSigner {
+       fn generate_channel_keys_id(&self, inbound: bool, _channel_value_satoshis: u64, _user_channel_id: u128) -> [u8; 32] {
                let ctr = self.counter.fetch_add(1, Ordering::Relaxed) as u8;
+               self.signer_state.borrow_mut().insert(ctr, (inbound, Arc::new(Mutex::new(EnforcementState::new()))));
+               [ctr; 32]
+       }
+
+       fn derive_channel_signer(&self, channel_value_satoshis: u64, channel_keys_id: [u8; 32]) -> Self::Signer {
                let secp_ctx = Secp256k1::signing_only();
-               EnforcingSigner::new(if inbound {
+               let ctr = channel_keys_id[0];
+               let (inbound, state) = self.signer_state.borrow().get(&ctr).unwrap().clone();
+               EnforcingSigner::new_with_revoked(if inbound {
                        InMemorySigner::new(
                                &secp_ctx,
                                self.node_secret.clone(),
@@ -311,7 +319,7 @@ impl KeysInterface for KeyProvider {
                                SecretKey::from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, ctr]).unwrap(),
                                [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, ctr],
                                channel_value_satoshis,
-                               [0; 32]
+                               channel_keys_id,
                        )
                } else {
                        InMemorySigner::new(
@@ -324,9 +332,9 @@ impl KeysInterface for KeyProvider {
                                SecretKey::from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, ctr]).unwrap(),
                                [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, ctr],
                                channel_value_satoshis,
-                               [0; 32]
+                               channel_keys_id,
                        )
-               })
+               }, state, false)
        }
 
        fn get_secure_random_bytes(&self) -> [u8; 32] {
@@ -390,7 +398,12 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
        let monitor = Arc::new(chainmonitor::ChainMonitor::new(None, broadcast.clone(), Arc::clone(&logger), fee_est.clone(),
                Arc::new(TestPersister { update_ret: Mutex::new(ChannelMonitorUpdateStatus::Completed) })));
 
-       let keys_manager = Arc::new(KeyProvider { node_secret: our_network_key.clone(), inbound_payment_key: KeyMaterial(inbound_payment_key.try_into().unwrap()), counter: AtomicU64::new(0) });
+       let keys_manager = Arc::new(KeyProvider {
+               node_secret: our_network_key.clone(),
+               inbound_payment_key: KeyMaterial(inbound_payment_key.try_into().unwrap()),
+               counter: AtomicU64::new(0),
+               signer_state: RefCell::new(HashMap::new())
+       });
        let mut config = UserConfig::default();
        config.channel_config.forwarding_fee_proportional_millionths =  slice_to_be32(get_slice!(4));
        config.channel_handshake_config.announced_channel = get_slice!(1)[0] != 0;
@@ -646,7 +659,7 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
                                Event::FundingGenerationReady { temporary_channel_id, counterparty_node_id, channel_value_satoshis, output_script, .. } => {
                                        pending_funding_generation.push((temporary_channel_id, counterparty_node_id, channel_value_satoshis, output_script));
                                },
-                               Event::PaymentReceived { payment_hash, .. } => {
+                               Event::PaymentClaimable { payment_hash, .. } => {
                                        //TODO: enhance by fetching random amounts from fuzz input?
                                        payments_received.push(payment_hash);
                                },
index 8fa6e4231be499a0e1ba989b68ed4c0c7dcc8e0c..b1f8fc33637f948cc0a6cf1c8b30ccfeaa45a0a1 100644 (file)
@@ -111,7 +111,9 @@ impl KeysInterface for KeyProvider {
 
        fn get_shutdown_scriptpubkey(&self) -> ShutdownScript { unreachable!() }
 
-       fn get_channel_signer(&self, _inbound: bool, _channel_value_satoshis: u64) -> EnforcingSigner {
+       fn generate_channel_keys_id(&self, _inbound: bool, _channel_value_satoshis: u64, _user_channel_id: u128) -> [u8; 32] { unreachable!() }
+
+       fn derive_channel_signer(&self, _channel_value_satoshis: u64, _channel_keys_id: [u8; 32]) -> Self::Signer {
                unreachable!()
        }
 
index a5cd91f2884a0a4e78eae3aa09114f3d417141b4..761f7bddc864706d39ebb44739f3dd2082cd365c 100644 (file)
@@ -225,6 +225,7 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
                                                                user_channel_id: 0, inbound_capacity_msat: 0,
                                                                unspendable_punishment_reserve: None,
                                                                confirmations_required: None,
+                                                               confirmations: None,
                                                                force_close_spend_delay: None,
                                                                is_outbound: true, is_channel_ready: true,
                                                                is_usable: true, is_public: true,
index ece82bcaa55a2299fd51f738050f55c52926ad78..1c720921970095d4211b4b8113b113926db3854f 100644 (file)
@@ -283,8 +283,8 @@ macro_rules! define_run_body {
                        // continuing our normal cadence.
                        if last_prune_call.elapsed().as_secs() > if have_pruned { NETWORK_PRUNE_TIMER } else { FIRST_NETWORK_PRUNE_TIMER } {
                                // The network graph must not be pruned while rapid sync completion is pending
-                               log_trace!($logger, "Assessing prunability of network graph");
                                if let Some(network_graph) = $gossip_sync.prunable_network_graph() {
+                                       log_trace!($logger, "Pruning and persisting network graph.");
                                        network_graph.remove_stale_channels_and_tracking();
 
                                        if let Err(e) = $persister.persist_graph(network_graph) {
@@ -293,8 +293,6 @@ macro_rules! define_run_body {
 
                                        last_prune_call = Instant::now();
                                        have_pruned = true;
-                               } else {
-                                       log_trace!($logger, "Not pruning network graph, either due to pending rapid gossip sync or absence of a prunable graph.");
                                }
                        }
 
@@ -580,13 +578,13 @@ mod tests {
        use lightning::ln::msgs::{ChannelMessageHandler, Init};
        use lightning::ln::peer_handler::{PeerManager, MessageHandler, SocketDescriptor, IgnoringMessageHandler};
        use lightning::routing::gossip::{NetworkGraph, P2PGossipSync};
+       use lightning::routing::router::DefaultRouter;
        use lightning::util::config::UserConfig;
        use lightning::util::events::{Event, MessageSendEventsProvider, MessageSendEvent};
        use lightning::util::ser::Writeable;
        use lightning::util::test_utils;
        use lightning::util::persist::KVStorePersister;
        use lightning_invoice::payment::{InvoicePayer, Retry};
-       use lightning_invoice::utils::DefaultRouter;
        use lightning_persister::FilesystemPersister;
        use std::fs;
        use std::path::PathBuf;
@@ -1072,10 +1070,11 @@ mod tests {
 
                loop {
                        let log_entries = nodes[0].logger.lines.lock().unwrap();
-                       let expected_log_a = "Assessing prunability of network graph".to_string();
-                       let expected_log_b = "Not pruning network graph, either due to pending rapid gossip sync or absence of a prunable graph.".to_string();
-                       if log_entries.get(&("lightning_background_processor".to_string(), expected_log_a)).is_some() &&
-                               log_entries.get(&("lightning_background_processor".to_string(), expected_log_b)).is_some() {
+                       let loop_counter = "Calling ChannelManager's timer_tick_occurred".to_string();
+                       if *log_entries.get(&("lightning_background_processor".to_string(), loop_counter))
+                               .unwrap_or(&0) > 1
+                       {
+                               // Wait until the loop has gone around at least twice.
                                break
                        }
                }
index a1c187915b98a81cb6f78d582458bbc3e4524dd6..4fddedc0c0749e908889fcdc82db028c877c4612 100644 (file)
@@ -44,7 +44,7 @@
 //! # use lightning::util::logger::{Logger, Record};
 //! # use lightning::util::ser::{Writeable, Writer};
 //! # use lightning_invoice::Invoice;
-//! # use lightning_invoice::payment::{InvoicePayer, Payer, Retry, ScoringRouter};
+//! # use lightning_invoice::payment::{InvoicePayer, Payer, Retry};
 //! # use secp256k1::PublicKey;
 //! # use std::cell::RefCell;
 //! # use std::ops::Deref;
@@ -69,6 +69,7 @@
 //! #         &self, route: &Route, payment_id: PaymentId
 //! #     ) -> Result<(), PaymentSendFailure> { unimplemented!() }
 //! #     fn abandon_payment(&self, payment_id: PaymentId) { unimplemented!() }
+//! #     fn inflight_htlcs(&self) -> InFlightHtlcs { unimplemented!() }
 //! # }
 //! #
 //! # struct FakeRouter {}
@@ -77,8 +78,6 @@
 //! #         &self, payer: &PublicKey, params: &RouteParameters,
 //! #         first_hops: Option<&[&ChannelDetails]>, _inflight_htlcs: InFlightHtlcs
 //! #     ) -> Result<Route, LightningError> { unimplemented!() }
-//! # }
-//! # impl ScoringRouter for FakeRouter {
 //! #     fn notify_payment_path_failed(&self, path: &[&RouteHop], short_channel_id: u64) {  unimplemented!() }
 //! #     fn notify_payment_path_successful(&self, path: &[&RouteHop]) {  unimplemented!() }
 //! #     fn notify_payment_probe_successful(&self, path: &[&RouteHop]) {  unimplemented!() }
@@ -145,9 +144,7 @@ use crate::prelude::*;
 use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
 use lightning::ln::channelmanager::{ChannelDetails, PaymentId, PaymentSendFailure};
 use lightning::ln::msgs::LightningError;
-use lightning::routing::gossip::NodeId;
-use lightning::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, RouteParameters, Router};
-use lightning::util::errors::APIError;
+use lightning::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteParameters, Router};
 use lightning::util::events::{Event, EventHandler};
 use lightning::util::logger::Logger;
 use crate::time_utils::Time;
@@ -187,7 +184,7 @@ mod sealed {
 /// (C-not exported) generally all users should use the [`InvoicePayer`] type alias.
 pub struct InvoicePayerUsingTime<
        P: Deref,
-       R: ScoringRouter,
+       R: Router,
        L: Deref,
        E: sealed::BaseEventHandler,
        T: Time
@@ -200,26 +197,10 @@ pub struct InvoicePayerUsingTime<
        logger: L,
        event_handler: E,
        /// Caches the overall attempts at making a payment, which is updated prior to retrying.
-       payment_cache: Mutex<HashMap<PaymentHash, PaymentInfo<T>>>,
+       payment_cache: Mutex<HashMap<PaymentHash, PaymentAttempts<T>>>,
        retry: Retry,
 }
 
-/// Used by [`InvoicePayerUsingTime::payment_cache`] to track the payments that are either
-/// currently being made, or have outstanding paths that need retrying.
-struct PaymentInfo<T: Time> {
-       attempts: PaymentAttempts<T>,
-       paths: Vec<Vec<RouteHop>>,
-}
-
-impl<T: Time> PaymentInfo<T> {
-       fn new() -> Self {
-               PaymentInfo {
-                       attempts: PaymentAttempts::new(),
-                       paths: vec![],
-               }
-       }
-}
-
 /// Storing minimal payment attempts information required for determining if a outbound payment can
 /// be retried.
 #[derive(Clone, Copy)]
@@ -290,30 +271,10 @@ pub trait Payer {
 
        /// Signals that no further retries for the given payment will occur.
        fn abandon_payment(&self, payment_id: PaymentId);
-}
 
-/// A trait defining behavior for a [`Router`] implementation that also supports scoring channels
-/// based on payment and probe success/failure.
-///
-/// [`Router`]: lightning::routing::router::Router
-pub trait ScoringRouter: Router {
-       /// Finds a [`Route`] between `payer` and `payee` for a payment with the given values. Includes
-       /// `PaymentHash` and `PaymentId` to be able to correlate the request with a specific payment.
-       fn find_route_with_id(
-               &self, payer: &PublicKey, route_params: &RouteParameters,
-               first_hops: Option<&[&ChannelDetails]>, inflight_htlcs: InFlightHtlcs,
-               _payment_hash: PaymentHash, _payment_id: PaymentId
-       ) -> Result<Route, LightningError> {
-               self.find_route(payer, route_params, first_hops, inflight_htlcs)
-       }
-       /// Lets the router know that payment through a specific path has failed.
-       fn notify_payment_path_failed(&self, path: &[&RouteHop], short_channel_id: u64);
-       /// Lets the router know that payment through a specific path was successful.
-       fn notify_payment_path_successful(&self, path: &[&RouteHop]);
-       /// Lets the router know that a payment probe was successful.
-       fn notify_payment_probe_successful(&self, path: &[&RouteHop]);
-       /// Lets the router know that a payment probe failed.
-       fn notify_payment_probe_failed(&self, path: &[&RouteHop], short_channel_id: u64);
+       /// Construct an [`InFlightHtlcs`] containing information about currently used up liquidity
+       /// across payments.
+       fn inflight_htlcs(&self) -> InFlightHtlcs;
 }
 
 /// Strategies available to retry payment path failures for an [`Invoice`].
@@ -355,7 +316,7 @@ pub enum PaymentError {
        Sending(PaymentSendFailure),
 }
 
-impl<P: Deref, R: ScoringRouter, L: Deref, E: sealed::BaseEventHandler, T: Time>
+impl<P: Deref, R: Router, L: Deref, E: sealed::BaseEventHandler, T: Time>
        InvoicePayerUsingTime<P, R, L, E, T>
 where
        P::Target: Payer,
@@ -455,7 +416,7 @@ where
                let payment_hash = PaymentHash(invoice.payment_hash().clone().into_inner());
                match self.payment_cache.lock().unwrap().entry(payment_hash) {
                        hash_map::Entry::Occupied(_) => return Err(PaymentError::Invoice("payment pending")),
-                       hash_map::Entry::Vacant(entry) => entry.insert(PaymentInfo::new()),
+                       hash_map::Entry::Vacant(entry) => entry.insert(PaymentAttempts::new()),
                };
 
                let payment_secret = Some(invoice.payment_secret().clone());
@@ -519,7 +480,7 @@ where
        ) -> Result<(), PaymentError> {
                match self.payment_cache.lock().unwrap().entry(payment_hash) {
                        hash_map::Entry::Occupied(_) => return Err(PaymentError::Invoice("payment pending")),
-                       hash_map::Entry::Vacant(entry) => entry.insert(PaymentInfo::new()),
+                       hash_map::Entry::Vacant(entry) => entry.insert(PaymentAttempts::new()),
                };
 
                let route_params = RouteParameters {
@@ -547,46 +508,29 @@ where
 
                let payer = self.payer.node_id();
                let first_hops = self.payer.first_hops();
-               let inflight_htlcs = self.create_inflight_map();
+               let inflight_htlcs = self.payer.inflight_htlcs();
                let route = self.router.find_route(
                        &payer, &params, Some(&first_hops.iter().collect::<Vec<_>>()), inflight_htlcs
                ).map_err(|e| PaymentError::Routing(e))?;
 
                match send_payment(&route) {
-                       Ok(()) => {
-                               for path in route.paths {
-                                       self.process_path_inflight_htlcs(payment_hash, path);
-                               }
-                               Ok(())
-                       },
+                       Ok(()) => Ok(()),
                        Err(e) => match e {
                                PaymentSendFailure::ParameterError(_) => Err(e),
                                PaymentSendFailure::PathParameterError(_) => Err(e),
                                PaymentSendFailure::DuplicatePayment => Err(e),
                                PaymentSendFailure::AllFailedResendSafe(_) => {
                                        let mut payment_cache = self.payment_cache.lock().unwrap();
-                                       let payment_info = payment_cache.get_mut(&payment_hash).unwrap();
-                                       payment_info.attempts.count += 1;
-                                       if self.retry.is_retryable_now(&payment_info.attempts) {
+                                       let payment_attempts = payment_cache.get_mut(&payment_hash).unwrap();
+                                       payment_attempts.count += 1;
+                                       if self.retry.is_retryable_now(payment_attempts) {
                                                core::mem::drop(payment_cache);
                                                Ok(self.pay_internal(params, payment_hash, send_payment)?)
                                        } else {
                                                Err(e)
                                        }
                                },
-                               PaymentSendFailure::PartialFailure { failed_paths_retry, payment_id, results } => {
-                                       // If a `PartialFailure` event returns a result that is an `Ok()`, it means that
-                                       // part of our payment is retried. When we receive `MonitorUpdateInProgress`, it
-                                       // means that we are still waiting for our channel monitor update to be completed.
-                                       for (result, path) in results.iter().zip(route.paths.into_iter()) {
-                                               match result {
-                                                       Ok(_) | Err(APIError::MonitorUpdateInProgress) => {
-                                                               self.process_path_inflight_htlcs(payment_hash, path);
-                                                       },
-                                                       _ => {},
-                                               }
-                                       }
-
+                               PaymentSendFailure::PartialFailure { failed_paths_retry, payment_id, .. } => {
                                        if let Some(retry_data) = failed_paths_retry {
                                                // Some paths were sent, even if we failed to send the full MPP value our
                                                // recipient may misbehave and claim the funds, at which point we have to
@@ -606,36 +550,16 @@ where
                }.map_err(|e| PaymentError::Sending(e))
        }
 
-       // Takes in a path to have its information stored in `payment_cache`. This is done for paths
-       // that are pending retry.
-       fn process_path_inflight_htlcs(&self, payment_hash: PaymentHash, path: Vec<RouteHop>) {
-               self.payment_cache.lock().unwrap().entry(payment_hash)
-                       .or_insert_with(|| PaymentInfo::new())
-                       .paths.push(path);
-       }
-
-       // Find the path we want to remove in `payment_cache`. If it doesn't exist, do nothing.
-       fn remove_path_inflight_htlcs(&self, payment_hash: PaymentHash, path: &Vec<RouteHop>) {
-               self.payment_cache.lock().unwrap().entry(payment_hash)
-                       .and_modify(|payment_info| {
-                               if let Some(idx) = payment_info.paths.iter().position(|p| p == path) {
-                                       payment_info.paths.swap_remove(idx);
-                               }
-                       });
-       }
-
        fn retry_payment(
                &self, payment_id: PaymentId, payment_hash: PaymentHash, params: &RouteParameters
        ) -> Result<(), ()> {
-               let attempts = self.payment_cache.lock().unwrap().entry(payment_hash)
-                       .and_modify(|info| info.attempts.count += 1 )
-                       .or_insert_with(|| PaymentInfo {
-                               attempts: PaymentAttempts {
+               let attempts =
+                       *self.payment_cache.lock().unwrap().entry(payment_hash)
+                               .and_modify(|attempts| attempts.count += 1)
+                               .or_insert(PaymentAttempts {
                                        count: 1,
-                                       first_attempted_at: T::now(),
-                               },
-                               paths: vec![],
-                       }).attempts;
+                                       first_attempted_at: T::now()
+                               });
 
                if !self.retry.is_retryable_now(&attempts) {
                        log_trace!(self.logger, "Payment {} exceeded maximum attempts; not retrying ({})", log_bytes!(payment_hash.0), attempts);
@@ -651,7 +575,7 @@ where
 
                let payer = self.payer.node_id();
                let first_hops = self.payer.first_hops();
-               let inflight_htlcs = self.create_inflight_map();
+               let inflight_htlcs = self.payer.inflight_htlcs();
 
                let route = self.router.find_route(
                        &payer, &params, Some(&first_hops.iter().collect::<Vec<_>>()), inflight_htlcs
@@ -663,12 +587,7 @@ where
                }
 
                match self.payer.retry_payment(&route.as_ref().unwrap(), payment_id) {
-                       Ok(()) => {
-                               for path in route.unwrap().paths.into_iter() {
-                                       self.process_path_inflight_htlcs(payment_hash, path);
-                               }
-                               Ok(())
-                       },
+                       Ok(()) => Ok(()),
                        Err(PaymentSendFailure::ParameterError(_)) |
                        Err(PaymentSendFailure::PathParameterError(_)) => {
                                log_trace!(self.logger, "Failed to retry for payment {} due to bogus route/payment data, not retrying.", log_bytes!(payment_hash.0));
@@ -681,19 +600,7 @@ where
                                log_error!(self.logger, "Got a DuplicatePayment error when attempting to retry a payment, this shouldn't happen.");
                                Err(())
                        }
-                       Err(PaymentSendFailure::PartialFailure { failed_paths_retry, results, .. }) => {
-                               // If a `PartialFailure` error contains a result that is an `Ok()`, it means that
-                               // part of our payment is retried. When we receive `MonitorUpdateInProgress`, it
-                               // means that we are still waiting for our channel monitor update to complete.
-                               for (result, path) in results.iter().zip(route.unwrap().paths.into_iter()) {
-                                       match result {
-                                               Ok(_) | Err(APIError::MonitorUpdateInProgress) => {
-                                                       self.process_path_inflight_htlcs(payment_hash, path);
-                                               },
-                                               _ => {},
-                                       }
-                               }
-
+                       Err(PaymentSendFailure::PartialFailure { failed_paths_retry, .. }) => {
                                if let Some(retry) = failed_paths_retry {
                                        // Always return Ok for the same reason as noted in pay_internal.
                                        let _ = self.retry_payment(payment_id, payment_hash, &retry);
@@ -710,46 +617,6 @@ where
        pub fn remove_cached_payment(&self, payment_hash: &PaymentHash) {
                self.payment_cache.lock().unwrap().remove(payment_hash);
        }
-
-       /// Use path information in the payment_cache to construct a HashMap mapping a channel's short
-       /// channel id and direction to the amount being sent through it.
-       ///
-       /// This function should be called whenever we need information about currently used up liquidity
-       /// across payments.
-       fn create_inflight_map(&self) -> InFlightHtlcs {
-               let mut total_inflight_map: HashMap<(u64, bool), u64> = HashMap::new();
-               // Make an attempt at finding existing payment information from `payment_cache`. If it
-               // does not exist, it probably is a fresh payment and we can just return an empty
-               // HashMap.
-               for payment_info in self.payment_cache.lock().unwrap().values() {
-                       for path in &payment_info.paths {
-                               if path.is_empty() { break };
-                               // total_inflight_map needs to be direction-sensitive when keeping track of the HTLC value
-                               // that is held up. However, the `hops` array, which is a path returned by `find_route` in
-                               // the router excludes the payer node. In the following lines, the payer's information is
-                               // hardcoded with an inflight value of 0 so that we can correctly represent the first hop
-                               // in our sliding window of two.
-                               let our_node_id: PublicKey = self.payer.node_id();
-                               let reversed_hops_with_payer = path.iter().rev().skip(1)
-                                       .map(|hop| hop.pubkey)
-                                       .chain(core::iter::once(our_node_id));
-                               let mut cumulative_msat = 0;
-
-                               // Taking the reversed vector from above, we zip it with just the reversed hops list to
-                               // work "backwards" of the given path, since the last hop's `fee_msat` actually represents
-                               // the total amount sent.
-                               for (next_hop, prev_hop) in path.iter().rev().zip(reversed_hops_with_payer) {
-                                       cumulative_msat += next_hop.fee_msat;
-                                       total_inflight_map
-                                               .entry((next_hop.short_channel_id, NodeId::from_pubkey(&prev_hop) < NodeId::from_pubkey(&next_hop.pubkey)))
-                                               .and_modify(|used_liquidity_msat| *used_liquidity_msat += cumulative_msat)
-                                               .or_insert(cumulative_msat);
-                               }
-                       }
-               }
-
-               InFlightHtlcs::new(total_inflight_map)
-       }
 }
 
 fn expiry_time_from_unix_epoch(invoice: &Invoice) -> Duration {
@@ -763,7 +630,7 @@ fn has_expired(route_params: &RouteParameters) -> bool {
        } else { false }
 }
 
-impl<P: Deref, R: ScoringRouter, L: Deref, E: sealed::BaseEventHandler, T: Time>
+impl<P: Deref, R: Router, L: Deref, E: sealed::BaseEventHandler, T: Time>
        InvoicePayerUsingTime<P, R, L, E, T>
 where
        P::Target: Payer,
@@ -772,16 +639,6 @@ where
        /// Returns a bool indicating whether the processed event should be forwarded to a user-provided
        /// event handler.
        fn handle_event_internal(&self, event: &Event) -> bool {
-               match event {
-                       Event::PaymentPathFailed { payment_hash, path, ..  }
-                       | Event::PaymentPathSuccessful { path, payment_hash: Some(payment_hash), .. }
-                       | Event::ProbeSuccessful { payment_hash, path, .. }
-                       | Event::ProbeFailed { payment_hash, path, .. } => {
-                               self.remove_path_inflight_htlcs(*payment_hash, path);
-                       },
-                       _ => {},
-               }
-
                match event {
                        Event::PaymentPathFailed {
                                payment_id, payment_hash, payment_failed_permanently, path, short_channel_id, retry, ..
@@ -817,7 +674,7 @@ where
                                let mut payment_cache = self.payment_cache.lock().unwrap();
                                let attempts = payment_cache
                                        .remove(payment_hash)
-                                       .map_or(1, |payment_info| payment_info.attempts.count + 1);
+                                       .map_or(1, |attempts| attempts.count + 1);
                                log_trace!(self.logger, "Payment {} succeeded (attempts: {})", log_bytes!(payment_hash.0), attempts);
                        },
                        Event::ProbeSuccessful { payment_hash, path, .. } => {
@@ -840,7 +697,7 @@ where
        }
 }
 
-impl<P: Deref, R: ScoringRouter, L: Deref, E: EventHandler, T: Time>
+impl<P: Deref, R: Router, L: Deref, E: EventHandler, T: Time>
        EventHandler for InvoicePayerUsingTime<P, R, L, E, T>
 where
        P::Target: Payer,
@@ -854,7 +711,7 @@ where
        }
 }
 
-impl<P: Deref, R: ScoringRouter, L: Deref, T: Time, F: Future, H: Fn(Event) -> F>
+impl<P: Deref, R: Router, L: Deref, T: Time, F: Future, H: Fn(Event) -> F>
        InvoicePayerUsingTime<P, R, L, H, T>
 where
        P::Target: Payer,
@@ -874,7 +731,7 @@ where
 mod tests {
        use super::*;
        use crate::{InvoiceBuilder, Currency};
-       use crate::utils::{ScorerAccountingForInFlightHtlcs, create_invoice_from_channelmanager_and_duration_since_epoch};
+       use crate::utils::create_invoice_from_channelmanager_and_duration_since_epoch;
        use bitcoin_hashes::sha256::Hash as Sha256;
        use lightning::ln::PaymentPreimage;
        use lightning::ln::channelmanager;
@@ -882,7 +739,7 @@ mod tests {
        use lightning::ln::functional_test_utils::*;
        use lightning::ln::msgs::{ChannelMessageHandler, ErrorAction, LightningError};
        use lightning::routing::gossip::{EffectiveCapacity, NodeId};
-       use lightning::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, Router};
+       use lightning::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, Router, ScorerAccountingForInFlightHtlcs};
        use lightning::routing::scoring::{ChannelUsage, LockableScore, Score};
        use lightning::util::test_utils::TestLogger;
        use lightning::util::errors::APIError;
@@ -894,7 +751,6 @@ mod tests {
        use std::time::{SystemTime, Duration};
        use crate::time_utils::tests::SinceEpoch;
        use crate::DEFAULT_EXPIRY_TIME;
-       use lightning::util::errors::APIError::{ChannelUnavailable, MonitorUpdateInProgress};
 
        fn invoice(payment_preimage: PaymentPreimage) -> Invoice {
                let payment_hash = Sha256::hash(&payment_preimage.0);
@@ -1614,66 +1470,17 @@ mod tests {
        }
 
        #[test]
-       fn generates_correct_inflight_map_data() {
-               let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
-
-               let payment_preimage = PaymentPreimage([1; 32]);
-               let invoice = invoice(payment_preimage);
-               let payment_hash = Some(PaymentHash(invoice.payment_hash().clone().into_inner()));
-               let final_value_msat = invoice.amount_milli_satoshis().unwrap();
-
-               let payer = TestPayer::new().expect_send(Amount::ForInvoice(final_value_msat));
-               let final_value_msat = invoice.amount_milli_satoshis().unwrap();
-               let route = TestRouter::route_for_value(final_value_msat);
-               let router = TestRouter::new(TestScorer::new());
-               let logger = TestLogger::new();
-               let invoice_payer =
-                       InvoicePayer::new(&payer, router, &logger, event_handler, Retry::Attempts(0));
-
-               let payment_id = invoice_payer.pay_invoice(&invoice).unwrap();
-
-               let inflight_map = invoice_payer.create_inflight_map();
-               // First path check
-               assert_eq!(inflight_map.0.get(&(0, false)).unwrap().clone(), 94);
-               assert_eq!(inflight_map.0.get(&(1, true)).unwrap().clone(), 84);
-               assert_eq!(inflight_map.0.get(&(2, false)).unwrap().clone(), 64);
-
-               // Second path check
-               assert_eq!(inflight_map.0.get(&(3, false)).unwrap().clone(), 74);
-               assert_eq!(inflight_map.0.get(&(4, false)).unwrap().clone(), 64);
-
-               invoice_payer.handle_event(Event::PaymentPathSuccessful {
-                       payment_id, payment_hash, path: route.paths[0].clone()
-               });
-
-               let inflight_map = invoice_payer.create_inflight_map();
-
-               assert_eq!(inflight_map.0.get(&(0, false)), None);
-               assert_eq!(inflight_map.0.get(&(1, true)), None);
-               assert_eq!(inflight_map.0.get(&(2, false)), None);
-
-               // Second path should still be inflight
-               assert_eq!(inflight_map.0.get(&(3, false)).unwrap().clone(), 74);
-               assert_eq!(inflight_map.0.get(&(4, false)).unwrap().clone(), 64)
-       }
-
-       #[test]
-       fn considers_inflight_htlcs_between_invoice_payments_when_path_succeeds() {
-               // First, let's just send a payment through, but only make sure one of the path completes
+       fn considers_inflight_htlcs_between_invoice_payments() {
                let event_handled = core::cell::RefCell::new(false);
                let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let payment_invoice = invoice(payment_preimage);
-               let payment_hash = Some(PaymentHash(payment_invoice.payment_hash().clone().into_inner()));
                let final_value_msat = payment_invoice.amount_milli_satoshis().unwrap();
 
                let payer = TestPayer::new()
                        .expect_send(Amount::ForInvoice(final_value_msat))
                        .expect_send(Amount::ForInvoice(final_value_msat));
-               let final_value_msat = payment_invoice.amount_milli_satoshis().unwrap();
-               let route = TestRouter::route_for_value(final_value_msat);
                let scorer = TestScorer::new()
                        // 1st invoice, 1st path
                        .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } )
@@ -1683,9 +1490,9 @@ mod tests {
                        .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } )
                        .expect_usage(ChannelUsage { amount_msat: 74, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } )
                        // 2nd invoice, 1st path
-                       .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } )
-                       .expect_usage(ChannelUsage { amount_msat: 84, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } )
-                       .expect_usage(ChannelUsage { amount_msat: 94, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown } )
+                       .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 64, effective_capacity: EffectiveCapacity::Unknown } )
+                       .expect_usage(ChannelUsage { amount_msat: 84, inflight_htlc_msat: 84, effective_capacity: EffectiveCapacity::Unknown } )
+                       .expect_usage(ChannelUsage { amount_msat: 94, inflight_htlc_msat: 94, effective_capacity: EffectiveCapacity::Unknown } )
                        // 2nd invoice, 2nd path
                        .expect_usage(ChannelUsage { amount_msat: 64, inflight_htlc_msat: 64, effective_capacity: EffectiveCapacity::Unknown } )
                        .expect_usage(ChannelUsage { amount_msat: 74, inflight_htlc_msat: 74, effective_capacity: EffectiveCapacity::Unknown } );
@@ -1694,16 +1501,12 @@ mod tests {
                let invoice_payer =
                        InvoicePayer::new(&payer, router, &logger, event_handler, Retry::Attempts(0));
 
-               // Succeed 1st path, leave 2nd path inflight
-               let payment_id = invoice_payer.pay_invoice(&payment_invoice).unwrap();
-               invoice_payer.handle_event(Event::PaymentPathSuccessful {
-                       payment_id, payment_hash, path: route.paths[0].clone()
-               });
+               // Make first invoice payment.
+               invoice_payer.pay_invoice(&payment_invoice).unwrap();
 
                // Let's pay a second invoice that will be using the same path. This should trigger the
-               // assertions that expect the last 4 ChannelUsage values above where TestScorer is initialized.
-               // Particularly, the 2nd path of the 1st payment, since it is not yet complete, should still
-               // have 64 msats inflight for paths considering the channel with scid of 1.
+               // assertions that expect `ChannelUsage` values of the first invoice payment that is still
+               // in-flight.
                let payment_preimage_2 = PaymentPreimage([2; 32]);
                let payment_invoice_2 = invoice(payment_preimage_2);
                invoice_payer.pay_invoice(&payment_invoice_2).unwrap();
@@ -1754,6 +1557,7 @@ mod tests {
 
                // Fail 1st path, leave 2nd path inflight
                let payment_id = Some(invoice_payer.pay_invoice(&payment_invoice).unwrap());
+               invoice_payer.payer.fail_path(&TestRouter::path_for_value(final_value_msat));
                invoice_payer.handle_event(Event::PaymentPathFailed {
                        payment_id,
                        payment_hash,
@@ -1766,6 +1570,7 @@ mod tests {
                });
 
                // Fails again the 1st path of our retry
+               invoice_payer.payer.fail_path(&TestRouter::path_for_value(final_value_msat / 2));
                invoice_payer.handle_event(Event::PaymentPathFailed {
                        payment_id,
                        payment_hash,
@@ -1781,67 +1586,6 @@ mod tests {
                });
        }
 
-       #[test]
-       fn accounts_for_some_inflight_htlcs_sent_during_partial_failure() {
-               let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
-
-               let payment_preimage = PaymentPreimage([1; 32]);
-               let invoice_to_pay = invoice(payment_preimage);
-               let final_value_msat = invoice_to_pay.amount_milli_satoshis().unwrap();
-
-               let retry = TestRouter::retry_for_invoice(&invoice_to_pay);
-               let payer = TestPayer::new()
-                       .fails_with_partial_failure(
-                               retry.clone(), OnAttempt(1),
-                               Some(vec![
-                                       Err(ChannelUnavailable { err: "abc".to_string() }), Err(MonitorUpdateInProgress)
-                               ]))
-                       .expect_send(Amount::ForInvoice(final_value_msat));
-
-               let router = TestRouter::new(TestScorer::new());
-               let logger = TestLogger::new();
-               let invoice_payer =
-                       InvoicePayer::new(&payer, router, &logger, event_handler, Retry::Attempts(0));
-
-               invoice_payer.pay_invoice(&invoice_to_pay).unwrap();
-               let inflight_map = invoice_payer.create_inflight_map();
-
-               // Only the second path, which failed with `MonitorUpdateInProgress` should be added to our
-               // inflight map because retries are disabled.
-               assert_eq!(inflight_map.0.len(), 2);
-       }
-
-       #[test]
-       fn accounts_for_all_inflight_htlcs_sent_during_partial_failure() {
-               let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
-
-               let payment_preimage = PaymentPreimage([1; 32]);
-               let invoice_to_pay = invoice(payment_preimage);
-               let final_value_msat = invoice_to_pay.amount_milli_satoshis().unwrap();
-
-               let retry = TestRouter::retry_for_invoice(&invoice_to_pay);
-               let payer = TestPayer::new()
-                       .fails_with_partial_failure(
-                               retry.clone(), OnAttempt(1),
-                               Some(vec![
-                                       Ok(()), Err(MonitorUpdateInProgress)
-                               ]))
-                       .expect_send(Amount::ForInvoice(final_value_msat));
-
-               let router = TestRouter::new(TestScorer::new());
-               let logger = TestLogger::new();
-               let invoice_payer =
-                       InvoicePayer::new(&payer, router, &logger, event_handler, Retry::Attempts(0));
-
-               invoice_payer.pay_invoice(&invoice_to_pay).unwrap();
-               let inflight_map = invoice_payer.create_inflight_map();
-
-               // All paths successful, hence we check of the existence of all 5 hops.
-               assert_eq!(inflight_map.0.len(), 5);
-       }
-
        struct TestRouter {
                scorer: RefCell<TestScorer>,
        }
@@ -1956,9 +1700,7 @@ mod tests {
                                payment_params: Some(route_params.payment_params.clone()), ..Self::route_for_value(route_params.final_value_msat)
                        })
                }
-       }
 
-       impl ScoringRouter for TestRouter {
                fn notify_payment_path_failed(&self, path: &[&RouteHop], short_channel_id: u64) {
                        self.scorer.lock().payment_path_failed(path, short_channel_id);
                }
@@ -1985,9 +1727,7 @@ mod tests {
                ) -> Result<Route, LightningError> {
                        Err(LightningError { err: String::new(), action: ErrorAction::IgnoreError })
                }
-       }
 
-       impl ScoringRouter for FailingRouter {
                fn notify_payment_path_failed(&self, _path: &[&RouteHop], _short_channel_id: u64) {}
 
                fn notify_payment_path_successful(&self, _path: &[&RouteHop]) {}
@@ -2129,6 +1869,7 @@ mod tests {
                expectations: core::cell::RefCell<VecDeque<Amount>>,
                attempts: core::cell::RefCell<usize>,
                failing_on_attempt: core::cell::RefCell<HashMap<usize, PaymentSendFailure>>,
+               inflight_htlcs_paths: core::cell::RefCell<Vec<Vec<RouteHop>>>,
        }
 
        #[derive(Clone, Debug, PartialEq, Eq)]
@@ -2146,6 +1887,7 @@ mod tests {
                                expectations: core::cell::RefCell::new(VecDeque::new()),
                                attempts: core::cell::RefCell::new(0),
                                failing_on_attempt: core::cell::RefCell::new(HashMap::new()),
+                               inflight_htlcs_paths: core::cell::RefCell::new(Vec::new()),
                        }
                }
 
@@ -2190,6 +1932,20 @@ mod tests {
                                panic!("Unexpected amount: {:?}", actual_value_msats);
                        }
                }
+
+               fn track_inflight_htlcs(&self, route: &Route) {
+                       for path in &route.paths {
+                               self.inflight_htlcs_paths.borrow_mut().push(path.clone());
+                       }
+               }
+
+               fn fail_path(&self, path: &Vec<RouteHop>) {
+                       let path_idx = self.inflight_htlcs_paths.borrow().iter().position(|p| p == path);
+
+                       if let Some(idx) = path_idx {
+                               self.inflight_htlcs_paths.borrow_mut().swap_remove(idx);
+                       }
+               }
        }
 
        impl Drop for TestPayer {
@@ -2219,6 +1975,7 @@ mod tests {
                        _payment_secret: &Option<PaymentSecret>, _payment_id: PaymentId,
                ) -> Result<(), PaymentSendFailure> {
                        self.check_value_msats(Amount::ForInvoice(route.get_total_amount()));
+                       self.track_inflight_htlcs(route);
                        self.check_attempts()
                }
 
@@ -2233,10 +1990,19 @@ mod tests {
                        &self, route: &Route, _payment_id: PaymentId
                ) -> Result<(), PaymentSendFailure> {
                        self.check_value_msats(Amount::OnRetry(route.get_total_amount()));
+                       self.track_inflight_htlcs(route);
                        self.check_attempts()
                }
 
                fn abandon_payment(&self, _payment_id: PaymentId) { }
+
+               fn inflight_htlcs(&self) -> InFlightHtlcs {
+                       let mut inflight_htlcs = InFlightHtlcs::new();
+                       for path in self.inflight_htlcs_paths.clone().into_inner() {
+                               inflight_htlcs.process_path(&path, self.node_id());
+                       }
+                       inflight_htlcs
+               }
        }
 
        // *** Full Featured Functional Tests with a Real ChannelManager ***
@@ -2249,8 +2015,7 @@ mod tests {
                ) -> Result<Route, LightningError> {
                        self.0.borrow_mut().pop_front().unwrap()
                }
-       }
-       impl ScoringRouter for ManualRouter {
+
                fn notify_payment_path_failed(&self, _path: &[&RouteHop], _short_channel_id: u64) {}
 
                fn notify_payment_path_successful(&self, _path: &[&RouteHop]) {}
index b6442934da78898d56594a2f90951f47c9d35b50..d19254c84649e78ddcab64516c6fceaf39e803b9 100644 (file)
@@ -1,11 +1,11 @@
 //! Convenient utilities to create an invoice.
 
 use crate::{CreationError, Currency, Invoice, InvoiceBuilder, SignOrCreationError};
-use crate::payment::{Payer, ScoringRouter};
+use crate::payment::Payer;
 
 use crate::{prelude::*, Description, InvoiceDescription, Sha256};
 use bech32::ToBase32;
-use bitcoin_hashes::{Hash, sha256};
+use bitcoin_hashes::Hash;
 use lightning::chain;
 use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator};
 use lightning::chain::keysinterface::{Recipient, KeysInterface};
@@ -14,15 +14,12 @@ use lightning::ln::channelmanager::{ChannelDetails, ChannelManager, PaymentId, P
 #[cfg(feature = "std")]
 use lightning::ln::channelmanager::{PhantomRouteHints, MIN_CLTV_EXPIRY_DELTA};
 use lightning::ln::inbound_payment::{create, create_from_hash, ExpandedKey};
-use lightning::ln::msgs::LightningError;
-use lightning::routing::gossip::{NetworkGraph, NodeId, RoutingFees};
-use lightning::routing::router::{InFlightHtlcs, Route, RouteHint, RouteHintHop, RouteParameters, find_route, RouteHop, Router};
-use lightning::routing::scoring::{ChannelUsage, LockableScore, Score};
+use lightning::routing::gossip::RoutingFees;
+use lightning::routing::router::{InFlightHtlcs, Route, RouteHint, RouteHintHop};
 use lightning::util::logger::Logger;
 use secp256k1::PublicKey;
 use core::ops::Deref;
 use core::time::Duration;
-use crate::sync::Mutex;
 
 #[cfg(feature = "std")]
 /// Utility to create an invoice that can be paid to one of multiple nodes, or a "phantom invoice."
@@ -524,72 +521,6 @@ fn filter_channels<L: Deref>(
                .collect::<Vec<RouteHint>>()
 }
 
-/// A [`Router`] implemented using [`find_route`].
-pub struct DefaultRouter<G: Deref<Target = NetworkGraph<L>>, L: Deref, S: Deref> where
-       L::Target: Logger,
-       S::Target: for <'a> LockableScore<'a>,
-{
-       network_graph: G,
-       logger: L,
-       random_seed_bytes: Mutex<[u8; 32]>,
-       scorer: S
-}
-
-impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, S: Deref> DefaultRouter<G, L, S> where
-       L::Target: Logger,
-       S::Target: for <'a> LockableScore<'a>,
-{
-       /// Creates a new router using the given [`NetworkGraph`], a [`Logger`], and a randomness source
-       /// `random_seed_bytes`.
-       pub fn new(network_graph: G, logger: L, random_seed_bytes: [u8; 32], scorer: S) -> Self {
-               let random_seed_bytes = Mutex::new(random_seed_bytes);
-               Self { network_graph, logger, random_seed_bytes, scorer }
-       }
-}
-
-impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, S: Deref> Router for DefaultRouter<G, L, S> where
-       L::Target: Logger,
-       S::Target: for <'a> LockableScore<'a>,
-{
-       fn find_route(
-               &self, payer: &PublicKey, params: &RouteParameters, first_hops: Option<&[&ChannelDetails]>,
-               inflight_htlcs: InFlightHtlcs
-       ) -> Result<Route, LightningError> {
-               let random_seed_bytes = {
-                       let mut locked_random_seed_bytes = self.random_seed_bytes.lock().unwrap();
-                       *locked_random_seed_bytes = sha256::Hash::hash(&*locked_random_seed_bytes).into_inner();
-                       *locked_random_seed_bytes
-               };
-
-               find_route(
-                       payer, params, &self.network_graph, first_hops, &*self.logger,
-                       &ScorerAccountingForInFlightHtlcs::new(&mut self.scorer.lock(), inflight_htlcs),
-                       &random_seed_bytes
-               )
-       }
-}
-
-impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, S: Deref> ScoringRouter for DefaultRouter<G, L, S> where
-       L::Target: Logger,
-       S::Target: for <'a> LockableScore<'a>,
-{
-       fn notify_payment_path_failed(&self, path: &[&RouteHop], short_channel_id: u64) {
-               self.scorer.lock().payment_path_failed(path, short_channel_id);
-       }
-
-       fn notify_payment_path_successful(&self, path: &[&RouteHop]) {
-               self.scorer.lock().payment_path_successful(path);
-       }
-
-       fn notify_payment_probe_successful(&self, path: &[&RouteHop]) {
-               self.scorer.lock().probe_successful(path);
-       }
-
-       fn notify_payment_probe_failed(&self, path: &[&RouteHop], short_channel_id: u64) {
-               self.scorer.lock().probe_failed(path, short_channel_id);
-       }
-}
-
 impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Payer for ChannelManager<M, T, K, F, L>
 where
        M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
@@ -628,56 +559,10 @@ where
        fn abandon_payment(&self, payment_id: PaymentId) {
                self.abandon_payment(payment_id)
        }
-}
-
-
-/// Used to store information about all the HTLCs that are inflight across all payment attempts.
-pub(crate) struct ScorerAccountingForInFlightHtlcs<'a, S: Score> {
-       scorer: &'a mut S,
-       /// Maps a channel's short channel id and its direction to the liquidity used up.
-       inflight_htlcs: InFlightHtlcs,
-}
-
-impl<'a, S: Score> ScorerAccountingForInFlightHtlcs<'a, S> {
-       pub(crate) fn new(scorer: &'a mut S, inflight_htlcs: InFlightHtlcs) -> Self {
-               ScorerAccountingForInFlightHtlcs {
-                       scorer,
-                       inflight_htlcs
-               }
-       }
-}
-
-#[cfg(c_bindings)]
-impl<'a, S:Score> lightning::util::ser::Writeable for ScorerAccountingForInFlightHtlcs<'a, S> {
-       fn write<W: lightning::util::ser::Writer>(&self, writer: &mut W) -> Result<(), lightning::io::Error> { self.scorer.write(writer) }
-}
-
-impl<'a, S: Score> Score for ScorerAccountingForInFlightHtlcs<'a, S> {
-       fn channel_penalty_msat(&self, short_channel_id: u64, source: &NodeId, target: &NodeId, usage: ChannelUsage) -> u64 {
-               if let Some(used_liqudity) = self.inflight_htlcs.used_liquidity_msat(
-                       source, target, short_channel_id
-               ) {
-                       let usage = ChannelUsage {
-                               inflight_htlc_msat: usage.inflight_htlc_msat + used_liqudity,
-                               ..usage
-                       };
-
-                       self.scorer.channel_penalty_msat(short_channel_id, source, target, usage)
-               } else {
-                       self.scorer.channel_penalty_msat(short_channel_id, source, target, usage)
-               }
-       }
-
-       fn payment_path_failed(&mut self, _path: &[&RouteHop], _short_channel_id: u64) { unreachable!() }
 
-       fn payment_path_successful(&mut self, _path: &[&RouteHop]) { unreachable!() }
-
-       fn probe_failed(&mut self, _path: &[&RouteHop], _short_channel_id: u64) { unreachable!() }
-
-       fn probe_successful(&mut self, _path: &[&RouteHop]) { unreachable!() }
+       fn inflight_htlcs(&self) -> InFlightHtlcs { self.compute_inflight_htlcs() }
 }
 
-
 #[cfg(test)]
 mod test {
        use core::time::Duration;
@@ -1072,7 +957,7 @@ mod test {
                nodes[fwd_idx].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
                commitment_signed_dance!(nodes[fwd_idx], nodes[0], &payment_event.commitment_msg, false, true);
 
-               // Note that we have to "forward pending HTLCs" twice before we see the PaymentReceived as
+               // Note that we have to "forward pending HTLCs" twice before we see the PaymentClaimable as
                // this "emulates" the payment taking two hops, providing some privacy to make phantom node
                // payments "look real" by taking more time.
                expect_pending_htlcs_forwardable_ignore!(nodes[fwd_idx]);
@@ -1081,7 +966,7 @@ mod test {
                nodes[fwd_idx].node.process_pending_htlc_forwards();
 
                let payment_preimage_opt = if user_generated_pmt_hash { None } else { Some(payment_preimage) };
-               expect_payment_received!(&nodes[fwd_idx], payment_hash, payment_secret, payment_amt, payment_preimage_opt);
+               expect_payment_claimable!(&nodes[fwd_idx], payment_hash, payment_secret, payment_amt, payment_preimage_opt, route.paths[0].last().unwrap().pubkey);
                do_claim_payment_along_route(&nodes[0], &vec!(&vec!(&nodes[fwd_idx])[..]), false, payment_preimage);
                let events = nodes[0].node.get_and_clear_pending_events();
                assert_eq!(events.len(), 2);
index c0d0affcce5d2fad616ab1ae48e49feaea2a76d7..5c1e102aefc8e2cc44350acc052f99ffdab5e224 100644 (file)
@@ -53,7 +53,7 @@ use crate::util::ser::{Readable, ReadableArgs, MaybeReadable, Writer, Writeable,
 use crate::util::byte_utils;
 use crate::util::events::Event;
 #[cfg(anchors)]
-use crate::util::events::{AnchorDescriptor, BumpTransactionEvent};
+use crate::util::events::{AnchorDescriptor, HTLCDescriptor, BumpTransactionEvent};
 
 use crate::prelude::*;
 use core::{cmp, mem};
@@ -291,7 +291,7 @@ struct CounterpartyCommitmentParameters {
 
 impl Writeable for CounterpartyCommitmentParameters {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.write_all(&byte_utils::be64_to_array(0))?;
+               w.write_all(&(0 as u64).to_be_bytes())?;
                write_tlv_fields!(w, {
                        (0, self.counterparty_delayed_payment_base_key, required),
                        (2, self.counterparty_htlc_base_key, required),
@@ -647,6 +647,7 @@ struct IrrevocablyResolvedHTLC {
        /// was not present in the confirmed commitment transaction), HTLC-Success, or HTLC-Timeout
        /// transaction.
        resolving_txid: Option<Txid>, // Added as optional, but always filled in, in 0.0.110
+       resolving_tx: Option<Transaction>,
        /// Only set if the HTLC claim was ours using a payment preimage
        payment_preimage: Option<PaymentPreimage>,
 }
@@ -662,6 +663,7 @@ impl Writeable for IrrevocablyResolvedHTLC {
                        (0, mapped_commitment_tx_output_idx, required),
                        (1, self.resolving_txid, option),
                        (2, self.payment_preimage, option),
+                       (3, self.resolving_tx, option),
                });
                Ok(())
        }
@@ -672,15 +674,18 @@ impl Readable for IrrevocablyResolvedHTLC {
                let mut mapped_commitment_tx_output_idx = 0;
                let mut resolving_txid = None;
                let mut payment_preimage = None;
+               let mut resolving_tx = None;
                read_tlv_fields!(reader, {
                        (0, mapped_commitment_tx_output_idx, required),
                        (1, resolving_txid, option),
                        (2, payment_preimage, option),
+                       (3, resolving_tx, option),
                });
                Ok(Self {
                        commitment_tx_output_idx: if mapped_commitment_tx_output_idx == u32::max_value() { None } else { Some(mapped_commitment_tx_output_idx) },
                        resolving_txid,
                        payment_preimage,
+                       resolving_tx,
                })
        }
 }
@@ -826,6 +831,13 @@ pub(crate) struct ChannelMonitorImpl<Signer: Sign> {
        /// spending CSV for revocable outputs).
        htlcs_resolved_on_chain: Vec<IrrevocablyResolvedHTLC>,
 
+       /// The set of `SpendableOutput` events which we have already passed upstream to be claimed.
+       /// These are tracked explicitly to ensure that we don't generate the same events redundantly
+       /// if users duplicatively confirm old transactions. Specifically for transactions claiming a
+       /// revoked remote outpoint we otherwise have no tracking at all once they've reached
+       /// [`ANTI_REORG_DELAY`], so we have to track them here.
+       spendable_txids_confirmed: Vec<Txid>,
+
        // We simply modify best_block in Channel's block_connected so that serialization is
        // consistent but hopefully the users' copy handles block_connected in a consistent way.
        // (we do *not*, however, update them in update_monitor to ensure any local user copies keep
@@ -938,7 +950,7 @@ impl<Signer: Sign> Writeable for ChannelMonitorImpl<Signer> {
                self.channel_keys_id.write(writer)?;
                self.holder_revocation_basepoint.write(writer)?;
                writer.write_all(&self.funding_info.0.txid[..])?;
-               writer.write_all(&byte_utils::be16_to_array(self.funding_info.0.index))?;
+               writer.write_all(&self.funding_info.0.index.to_be_bytes())?;
                self.funding_info.1.write(writer)?;
                self.current_counterparty_commitment_txid.write(writer)?;
                self.prev_counterparty_commitment_txid.write(writer)?;
@@ -965,24 +977,24 @@ impl<Signer: Sign> Writeable for ChannelMonitorImpl<Signer> {
                        },
                }
 
-               writer.write_all(&byte_utils::be16_to_array(self.on_holder_tx_csv))?;
+               writer.write_all(&self.on_holder_tx_csv.to_be_bytes())?;
 
                self.commitment_secrets.write(writer)?;
 
                macro_rules! serialize_htlc_in_commitment {
                        ($htlc_output: expr) => {
                                writer.write_all(&[$htlc_output.offered as u8; 1])?;
-                               writer.write_all(&byte_utils::be64_to_array($htlc_output.amount_msat))?;
-                               writer.write_all(&byte_utils::be32_to_array($htlc_output.cltv_expiry))?;
+                               writer.write_all(&$htlc_output.amount_msat.to_be_bytes())?;
+                               writer.write_all(&$htlc_output.cltv_expiry.to_be_bytes())?;
                                writer.write_all(&$htlc_output.payment_hash.0[..])?;
                                $htlc_output.transaction_output_index.write(writer)?;
                        }
                }
 
-               writer.write_all(&byte_utils::be64_to_array(self.counterparty_claimable_outpoints.len() as u64))?;
+               writer.write_all(&(self.counterparty_claimable_outpoints.len() as u64).to_be_bytes())?;
                for (ref txid, ref htlc_infos) in self.counterparty_claimable_outpoints.iter() {
                        writer.write_all(&txid[..])?;
-                       writer.write_all(&byte_utils::be64_to_array(htlc_infos.len() as u64))?;
+                       writer.write_all(&(htlc_infos.len() as u64).to_be_bytes())?;
                        for &(ref htlc_output, ref htlc_source) in htlc_infos.iter() {
                                debug_assert!(htlc_source.is_none() || Some(**txid) == self.current_counterparty_commitment_txid
                                                || Some(**txid) == self.prev_counterparty_commitment_txid,
@@ -992,13 +1004,13 @@ impl<Signer: Sign> Writeable for ChannelMonitorImpl<Signer> {
                        }
                }
 
-               writer.write_all(&byte_utils::be64_to_array(self.counterparty_commitment_txn_on_chain.len() as u64))?;
+               writer.write_all(&(self.counterparty_commitment_txn_on_chain.len() as u64).to_be_bytes())?;
                for (ref txid, commitment_number) in self.counterparty_commitment_txn_on_chain.iter() {
                        writer.write_all(&txid[..])?;
                        writer.write_all(&byte_utils::be48_to_array(*commitment_number))?;
                }
 
-               writer.write_all(&byte_utils::be64_to_array(self.counterparty_hash_commitment_number.len() as u64))?;
+               writer.write_all(&(self.counterparty_hash_commitment_number.len() as u64).to_be_bytes())?;
                for (ref payment_hash, commitment_number) in self.counterparty_hash_commitment_number.iter() {
                        writer.write_all(&payment_hash.0[..])?;
                        writer.write_all(&byte_utils::be48_to_array(*commitment_number))?;
@@ -1016,7 +1028,7 @@ impl<Signer: Sign> Writeable for ChannelMonitorImpl<Signer> {
                writer.write_all(&byte_utils::be48_to_array(self.current_counterparty_commitment_number))?;
                writer.write_all(&byte_utils::be48_to_array(self.current_holder_commitment_number))?;
 
-               writer.write_all(&byte_utils::be64_to_array(self.payment_preimages.len() as u64))?;
+               writer.write_all(&(self.payment_preimages.len() as u64).to_be_bytes())?;
                for payment_preimage in self.payment_preimages.values() {
                        writer.write_all(&payment_preimage.0[..])?;
                }
@@ -1037,15 +1049,15 @@ impl<Signer: Sign> Writeable for ChannelMonitorImpl<Signer> {
                        }
                }
 
-               writer.write_all(&byte_utils::be64_to_array(self.pending_events.len() as u64))?;
+               writer.write_all(&(self.pending_events.len() as u64).to_be_bytes())?;
                for event in self.pending_events.iter() {
                        event.write(writer)?;
                }
 
                self.best_block.block_hash().write(writer)?;
-               writer.write_all(&byte_utils::be32_to_array(self.best_block.height()))?;
+               writer.write_all(&self.best_block.height().to_be_bytes())?;
 
-               writer.write_all(&byte_utils::be64_to_array(self.onchain_events_awaiting_threshold_conf.len() as u64))?;
+               writer.write_all(&(self.onchain_events_awaiting_threshold_conf.len() as u64).to_be_bytes())?;
                for ref entry in self.onchain_events_awaiting_threshold_conf.iter() {
                        entry.write(writer)?;
                }
@@ -1071,6 +1083,7 @@ impl<Signer: Sign> Writeable for ChannelMonitorImpl<Signer> {
                        (7, self.funding_spend_seen, required),
                        (9, self.counterparty_node_id, option),
                        (11, self.confirmed_commitment_tx_counterparty_output, option),
+                       (13, self.spendable_txids_confirmed, vec_type),
                });
 
                Ok(())
@@ -1179,6 +1192,7 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
                        funding_spend_confirmed: None,
                        confirmed_commitment_tx_counterparty_output: None,
                        htlcs_resolved_on_chain: Vec::new(),
+                       spendable_txids_confirmed: Vec::new(),
 
                        best_block,
                        counterparty_node_id: Some(counterparty_node_id),
@@ -1517,6 +1531,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        if let Some(v) = htlc.transaction_output_index { v } else { return None; };
 
                let mut htlc_spend_txid_opt = None;
+               let mut htlc_spend_tx_opt = None;
                let mut holder_timeout_spend_pending = None;
                let mut htlc_spend_pending = None;
                let mut holder_delayed_output_pending = None;
@@ -1525,7 +1540,9 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                OnchainEvent::HTLCUpdate { commitment_tx_output_idx, htlc_value_satoshis, .. }
                                if commitment_tx_output_idx == Some(htlc_commitment_tx_output_idx) => {
                                        debug_assert!(htlc_spend_txid_opt.is_none());
-                                       htlc_spend_txid_opt = event.transaction.as_ref().map(|tx| tx.txid());
+                                       htlc_spend_txid_opt = Some(&event.txid);
+                                       debug_assert!(htlc_spend_tx_opt.is_none());
+                                       htlc_spend_tx_opt = event.transaction.as_ref();
                                        debug_assert!(holder_timeout_spend_pending.is_none());
                                        debug_assert_eq!(htlc_value_satoshis.unwrap(), htlc.amount_msat / 1000);
                                        holder_timeout_spend_pending = Some(event.confirmation_threshold());
@@ -1533,7 +1550,9 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx, preimage, .. }
                                if commitment_tx_output_idx == htlc_commitment_tx_output_idx => {
                                        debug_assert!(htlc_spend_txid_opt.is_none());
-                                       htlc_spend_txid_opt = event.transaction.as_ref().map(|tx| tx.txid());
+                                       htlc_spend_txid_opt = Some(&event.txid);
+                                       debug_assert!(htlc_spend_tx_opt.is_none());
+                                       htlc_spend_tx_opt = event.transaction.as_ref();
                                        debug_assert!(htlc_spend_pending.is_none());
                                        htlc_spend_pending = Some((event.confirmation_threshold(), preimage.is_some()));
                                },
@@ -1549,19 +1568,32 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                let htlc_resolved = self.htlcs_resolved_on_chain.iter()
                        .find(|v| if v.commitment_tx_output_idx == Some(htlc_commitment_tx_output_idx) {
                                debug_assert!(htlc_spend_txid_opt.is_none());
-                               htlc_spend_txid_opt = v.resolving_txid;
+                               htlc_spend_txid_opt = v.resolving_txid.as_ref();
+                               debug_assert!(htlc_spend_tx_opt.is_none());
+                               htlc_spend_tx_opt = v.resolving_tx.as_ref();
                                true
                        } else { false });
                debug_assert!(holder_timeout_spend_pending.is_some() as u8 + htlc_spend_pending.is_some() as u8 + htlc_resolved.is_some() as u8 <= 1);
 
+               let htlc_commitment_outpoint = BitcoinOutPoint::new(confirmed_txid.unwrap(), htlc_commitment_tx_output_idx);
                let htlc_output_to_spend =
                        if let Some(txid) = htlc_spend_txid_opt {
-                               debug_assert!(
-                                       self.onchain_tx_handler.channel_transaction_parameters.opt_anchors.is_none(),
-                                       "This code needs updating for anchors");
-                               BitcoinOutPoint::new(txid, 0)
+                               // Because HTLC transactions either only have 1 input and 1 output (pre-anchors) or
+                               // are signed with SIGHASH_SINGLE|ANYONECANPAY under BIP-0143 (post-anchors), we can
+                               // locate the correct output by ensuring its adjacent input spends the HTLC output
+                               // in the commitment.
+                               if let Some(ref tx) = htlc_spend_tx_opt {
+                                       let htlc_input_idx_opt = tx.input.iter().enumerate()
+                                               .find(|(_, input)| input.previous_output == htlc_commitment_outpoint)
+                                               .map(|(idx, _)| idx as u32);
+                                       debug_assert!(htlc_input_idx_opt.is_some());
+                                       BitcoinOutPoint::new(*txid, htlc_input_idx_opt.unwrap_or(0))
+                               } else {
+                                       debug_assert!(!self.onchain_tx_handler.opt_anchors());
+                                       BitcoinOutPoint::new(*txid, 0)
+                               }
                        } else {
-                               BitcoinOutPoint::new(confirmed_txid.unwrap(), htlc_commitment_tx_output_idx)
+                               htlc_commitment_outpoint
                        };
                let htlc_output_spend_pending = self.onchain_tx_handler.is_output_spend_pending(&htlc_output_to_spend);
 
@@ -1585,8 +1617,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                } = &event.event {
                                        if event.transaction.as_ref().map(|tx| tx.input.iter().any(|inp| {
                                                if let Some(htlc_spend_txid) = htlc_spend_txid_opt {
-                                                       Some(tx.txid()) == htlc_spend_txid_opt ||
-                                                               inp.previous_output.txid == htlc_spend_txid
+                                                       tx.txid() == *htlc_spend_txid || inp.previous_output.txid == *htlc_spend_txid
                                                } else {
                                                        Some(inp.previous_output.txid) == confirmed_txid &&
                                                                inp.previous_output.vout == htlc_commitment_tx_output_idx
@@ -1828,12 +1859,60 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
                res
        }
 
+       /// Gets the set of outbound HTLCs which can be (or have been) resolved by this
+       /// `ChannelMonitor`. This is used to determine if an HTLC was removed from the channel prior
+       /// to the `ChannelManager` having been persisted.
+       ///
+       /// This is similar to [`Self::get_pending_outbound_htlcs`] except it includes HTLCs which were
+       /// resolved by this `ChannelMonitor`.
+       pub(crate) fn get_all_current_outbound_htlcs(&self) -> HashMap<HTLCSource, HTLCOutputInCommitment> {
+               let mut res = HashMap::new();
+               // Just examine the available counterparty commitment transactions. See docs on
+               // `fail_unbroadcast_htlcs`, below, for justification.
+               let us = self.inner.lock().unwrap();
+               macro_rules! walk_counterparty_commitment {
+                       ($txid: expr) => {
+                               if let Some(ref latest_outpoints) = us.counterparty_claimable_outpoints.get($txid) {
+                                       for &(ref htlc, ref source_option) in latest_outpoints.iter() {
+                                               if let &Some(ref source) = source_option {
+                                                       res.insert((**source).clone(), htlc.clone());
+                                               }
+                                       }
+                               }
+                       }
+               }
+               if let Some(ref txid) = us.current_counterparty_commitment_txid {
+                       walk_counterparty_commitment!(txid);
+               }
+               if let Some(ref txid) = us.prev_counterparty_commitment_txid {
+                       walk_counterparty_commitment!(txid);
+               }
+               res
+       }
+
        /// Gets the set of outbound HTLCs which are pending resolution in this channel.
        /// This is used to reconstruct pending outbound payments on restart in the ChannelManager.
        pub(crate) fn get_pending_outbound_htlcs(&self) -> HashMap<HTLCSource, HTLCOutputInCommitment> {
-               let mut res = HashMap::new();
                let us = self.inner.lock().unwrap();
+               // We're only concerned with the confirmation count of HTLC transactions, and don't
+               // actually care how many confirmations a commitment transaction may or may not have. Thus,
+               // we look for either a FundingSpendConfirmation event or a funding_spend_confirmed.
+               let confirmed_txid = us.funding_spend_confirmed.or_else(|| {
+                       us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| {
+                               if let OnchainEvent::FundingSpendConfirmation { .. } = event.event {
+                                       Some(event.txid)
+                               } else { None }
+                       })
+               });
 
+               if confirmed_txid.is_none() {
+                       // If we have not seen a commitment transaction on-chain (ie the channel is not yet
+                       // closed), just get the full set.
+                       mem::drop(us);
+                       return self.get_all_current_outbound_htlcs();
+               }
+
+               let mut res = HashMap::new();
                macro_rules! walk_htlcs {
                        ($holder_commitment: expr, $htlc_iter: expr) => {
                                for (htlc, source) in $htlc_iter {
@@ -1869,54 +1948,22 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
                        }
                }
 
-               // We're only concerned with the confirmation count of HTLC transactions, and don't
-               // actually care how many confirmations a commitment transaction may or may not have. Thus,
-               // we look for either a FundingSpendConfirmation event or a funding_spend_confirmed.
-               let confirmed_txid = us.funding_spend_confirmed.or_else(|| {
-                       us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| {
-                               if let OnchainEvent::FundingSpendConfirmation { .. } = event.event {
-                                       Some(event.txid)
+               let txid = confirmed_txid.unwrap();
+               if Some(txid) == us.current_counterparty_commitment_txid || Some(txid) == us.prev_counterparty_commitment_txid {
+                       walk_htlcs!(false, us.counterparty_claimable_outpoints.get(&txid).unwrap().iter().filter_map(|(a, b)| {
+                               if let &Some(ref source) = b {
+                                       Some((a, &**source))
                                } else { None }
-                       })
-               });
-               if let Some(txid) = confirmed_txid {
-                       if Some(txid) == us.current_counterparty_commitment_txid || Some(txid) == us.prev_counterparty_commitment_txid {
-                               walk_htlcs!(false, us.counterparty_claimable_outpoints.get(&txid).unwrap().iter().filter_map(|(a, b)| {
-                                       if let &Some(ref source) = b {
-                                               Some((a, &**source))
-                                       } else { None }
-                               }));
-                       } else if txid == us.current_holder_commitment_tx.txid {
-                               walk_htlcs!(true, us.current_holder_commitment_tx.htlc_outputs.iter().filter_map(|(a, _, c)| {
+                       }));
+               } else if txid == us.current_holder_commitment_tx.txid {
+                       walk_htlcs!(true, us.current_holder_commitment_tx.htlc_outputs.iter().filter_map(|(a, _, c)| {
+                               if let Some(source) = c { Some((a, source)) } else { None }
+                       }));
+               } else if let Some(prev_commitment) = &us.prev_holder_signed_commitment_tx {
+                       if txid == prev_commitment.txid {
+                               walk_htlcs!(true, prev_commitment.htlc_outputs.iter().filter_map(|(a, _, c)| {
                                        if let Some(source) = c { Some((a, source)) } else { None }
                                }));
-                       } else if let Some(prev_commitment) = &us.prev_holder_signed_commitment_tx {
-                               if txid == prev_commitment.txid {
-                                       walk_htlcs!(true, prev_commitment.htlc_outputs.iter().filter_map(|(a, _, c)| {
-                                               if let Some(source) = c { Some((a, source)) } else { None }
-                                       }));
-                               }
-                       }
-               } else {
-                       // If we have not seen a commitment transaction on-chain (ie the channel is not yet
-                       // closed), just examine the available counterparty commitment transactions. See docs
-                       // on `fail_unbroadcast_htlcs`, below, for justification.
-                       macro_rules! walk_counterparty_commitment {
-                               ($txid: expr) => {
-                                       if let Some(ref latest_outpoints) = us.counterparty_claimable_outpoints.get($txid) {
-                                               for &(ref htlc, ref source_option) in latest_outpoints.iter() {
-                                                       if let &Some(ref source) = source_option {
-                                                               res.insert((**source).clone(), htlc.clone());
-                                                       }
-                                               }
-                                       }
-                               }
-                       }
-                       if let Some(ref txid) = us.current_counterparty_commitment_txid {
-                               walk_counterparty_commitment!(txid);
-                       }
-                       if let Some(ref txid) = us.prev_counterparty_commitment_txid {
-                               walk_counterparty_commitment!(txid);
                        }
                }
 
@@ -2378,6 +2425,27 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                pending_htlcs,
                                        }));
                                },
+                               ClaimEvent::BumpHTLC {
+                                       target_feerate_sat_per_1000_weight, htlcs,
+                               } => {
+                                       let mut htlc_descriptors = Vec::with_capacity(htlcs.len());
+                                       for htlc in htlcs {
+                                               htlc_descriptors.push(HTLCDescriptor {
+                                                       channel_keys_id: self.channel_keys_id,
+                                                       channel_value_satoshis: self.channel_value_satoshis,
+                                                       channel_parameters: self.onchain_tx_handler.channel_transaction_parameters.clone(),
+                                                       commitment_txid: htlc.commitment_txid,
+                                                       per_commitment_number: htlc.per_commitment_number,
+                                                       htlc: htlc.htlc,
+                                                       preimage: htlc.preimage,
+                                                       counterparty_sig: htlc.counterparty_sig,
+                                               });
+                                       }
+                                       ret.push(Event::BumpTransaction(BumpTransactionEvent::HTLCResolution {
+                                               target_feerate_sat_per_1000_weight,
+                                               htlc_descriptors,
+                                       }));
+                               }
                        }
                }
                ret
@@ -2434,8 +2502,8 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        let secret = self.get_secret(commitment_number).unwrap();
                        let per_commitment_key = ignore_error!(SecretKey::from_slice(&secret));
                        let per_commitment_point = PublicKey::from_secret_key(&self.secp_ctx, &per_commitment_key);
-                       let revocation_pubkey = ignore_error!(chan_utils::derive_public_revocation_key(&self.secp_ctx, &per_commitment_point, &self.holder_revocation_basepoint));
-                       let delayed_key = ignore_error!(chan_utils::derive_public_key(&self.secp_ctx, &PublicKey::from_secret_key(&self.secp_ctx, &per_commitment_key), &self.counterparty_commitment_params.counterparty_delayed_payment_base_key));
+                       let revocation_pubkey = chan_utils::derive_public_revocation_key(&self.secp_ctx, &per_commitment_point, &self.holder_revocation_basepoint);
+                       let delayed_key = chan_utils::derive_public_key(&self.secp_ctx, &PublicKey::from_secret_key(&self.secp_ctx, &per_commitment_key), &self.counterparty_commitment_params.counterparty_delayed_payment_base_key);
 
                        let revokeable_redeemscript = chan_utils::get_revokeable_redeemscript(&revocation_pubkey, self.counterparty_commitment_params.on_counterparty_tx_csv, &delayed_key);
                        let revokeable_p2wsh = revokeable_redeemscript.to_v0_p2wsh();
@@ -2547,31 +2615,18 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        } else { return (claimable_outpoints, to_counterparty_output_info); };
 
                if let Some(transaction) = tx {
-                       let revokeable_p2wsh_opt =
-                               if let Ok(revocation_pubkey) = chan_utils::derive_public_revocation_key(
-                                       &self.secp_ctx, &per_commitment_point, &self.holder_revocation_basepoint)
-                               {
-                                       if let Ok(delayed_key) = chan_utils::derive_public_key(&self.secp_ctx,
-                                               &per_commitment_point,
-                                               &self.counterparty_commitment_params.counterparty_delayed_payment_base_key)
-                                       {
-                                               Some(chan_utils::get_revokeable_redeemscript(&revocation_pubkey,
-                                                       self.counterparty_commitment_params.on_counterparty_tx_csv,
-                                                       &delayed_key).to_v0_p2wsh())
-                                       } else {
-                                               debug_assert!(false, "Failed to derive a delayed payment key for a commitment state we accepted");
-                                               None
-                                       }
-                               } else {
-                                       debug_assert!(false, "Failed to derive a revocation pubkey key for a commitment state we accepted");
-                                       None
-                               };
-                       if let Some(revokeable_p2wsh) = revokeable_p2wsh_opt {
-                               for (idx, outp) in transaction.output.iter().enumerate() {
-                                       if outp.script_pubkey == revokeable_p2wsh {
-                                               to_counterparty_output_info =
-                                                       Some((idx.try_into().expect("Can't have > 2^32 outputs"), outp.value));
-                                       }
+                       let revocation_pubkey = chan_utils::derive_public_revocation_key(
+                               &self.secp_ctx, &per_commitment_point, &self.holder_revocation_basepoint);
+                       let delayed_key = chan_utils::derive_public_key(&self.secp_ctx,
+                               &per_commitment_point,
+                               &self.counterparty_commitment_params.counterparty_delayed_payment_base_key);
+                       let revokeable_p2wsh = chan_utils::get_revokeable_redeemscript(&revocation_pubkey,
+                               self.counterparty_commitment_params.on_counterparty_tx_csv,
+                               &delayed_key).to_v0_p2wsh();
+                       for (idx, outp) in transaction.output.iter().enumerate() {
+                               if outp.script_pubkey == revokeable_p2wsh {
+                                       to_counterparty_output_info =
+                                               Some((idx.try_into().expect("Can't have > 2^32 outputs"), outp.value));
                                }
                        }
                }
@@ -2611,31 +2666,49 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
        }
 
        /// Attempts to claim a counterparty HTLC-Success/HTLC-Timeout's outputs using the revocation key
-       fn check_spend_counterparty_htlc<L: Deref>(&mut self, tx: &Transaction, commitment_number: u64, height: u32, logger: &L) -> (Vec<PackageTemplate>, Option<TransactionOutputs>) where L::Target: Logger {
-               let htlc_txid = tx.txid();
-               if tx.input.len() != 1 || tx.output.len() != 1 || tx.input[0].witness.len() != 5 {
-                       return (Vec::new(), None)
-               }
-
-               macro_rules! ignore_error {
-                       ( $thing : expr ) => {
-                               match $thing {
-                                       Ok(a) => a,
-                                       Err(_) => return (Vec::new(), None)
-                               }
-                       };
-               }
-
+       fn check_spend_counterparty_htlc<L: Deref>(
+               &mut self, tx: &Transaction, commitment_number: u64, commitment_txid: &Txid, height: u32, logger: &L
+       ) -> (Vec<PackageTemplate>, Option<TransactionOutputs>) where L::Target: Logger {
                let secret = if let Some(secret) = self.get_secret(commitment_number) { secret } else { return (Vec::new(), None); };
-               let per_commitment_key = ignore_error!(SecretKey::from_slice(&secret));
+               let per_commitment_key = match SecretKey::from_slice(&secret) {
+                       Ok(key) => key,
+                       Err(_) => return (Vec::new(), None)
+               };
                let per_commitment_point = PublicKey::from_secret_key(&self.secp_ctx, &per_commitment_key);
 
-               log_error!(logger, "Got broadcast of revoked counterparty HTLC transaction, spending {}:{}", htlc_txid, 0);
-               let revk_outp = RevokedOutput::build(per_commitment_point, self.counterparty_commitment_params.counterparty_delayed_payment_base_key, self.counterparty_commitment_params.counterparty_htlc_base_key, per_commitment_key, tx.output[0].value, self.counterparty_commitment_params.on_counterparty_tx_csv);
-               let justice_package = PackageTemplate::build_package(htlc_txid, 0, PackageSolvingData::RevokedOutput(revk_outp), height + self.counterparty_commitment_params.on_counterparty_tx_csv as u32, true, height);
-               let claimable_outpoints = vec!(justice_package);
-               let outputs = vec![(0, tx.output[0].clone())];
-               (claimable_outpoints, Some((htlc_txid, outputs)))
+               let htlc_txid = tx.txid();
+               let mut claimable_outpoints = vec![];
+               let mut outputs_to_watch = None;
+               // Previously, we would only claim HTLCs from revoked HTLC transactions if they had 1 input
+               // with a witness of 5 elements and 1 output. This wasn't enough for anchor outputs, as the
+               // counterparty can now aggregate multiple HTLCs into a single transaction thanks to
+               // `SIGHASH_SINGLE` remote signatures, leading us to not claim any HTLCs upon seeing a
+               // confirmed revoked HTLC transaction (for more details, see
+               // https://lists.linuxfoundation.org/pipermail/lightning-dev/2022-April/003561.html).
+               //
+               // We make sure we're not vulnerable to this case by checking all inputs of the transaction,
+               // and claim those which spend the commitment transaction, have a witness of 5 elements, and
+               // have a corresponding output at the same index within the transaction.
+               for (idx, input) in tx.input.iter().enumerate() {
+                       if input.previous_output.txid == *commitment_txid && input.witness.len() == 5 && tx.output.get(idx).is_some() {
+                               log_error!(logger, "Got broadcast of revoked counterparty HTLC transaction, spending {}:{}", htlc_txid, idx);
+                               let revk_outp = RevokedOutput::build(
+                                       per_commitment_point, self.counterparty_commitment_params.counterparty_delayed_payment_base_key,
+                                       self.counterparty_commitment_params.counterparty_htlc_base_key, per_commitment_key,
+                                       tx.output[idx].value, self.counterparty_commitment_params.on_counterparty_tx_csv
+                               );
+                               let justice_package = PackageTemplate::build_package(
+                                       htlc_txid, idx as u32, PackageSolvingData::RevokedOutput(revk_outp),
+                                       height + self.counterparty_commitment_params.on_counterparty_tx_csv as u32, true, height
+                               );
+                               claimable_outpoints.push(justice_package);
+                               if outputs_to_watch.is_none() {
+                                       outputs_to_watch = Some((htlc_txid, vec![]));
+                               }
+                               outputs_to_watch.as_mut().unwrap().1.push((idx as u32, tx.output[idx].clone()));
+                       }
+               }
+               (claimable_outpoints, outputs_to_watch)
        }
 
        // Returns (1) `PackageTemplate`s that can be given to the OnchainTxHandler, so that the handler can
@@ -2649,18 +2722,28 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
 
                for &(ref htlc, _, _) in holder_tx.htlc_outputs.iter() {
                        if let Some(transaction_output_index) = htlc.transaction_output_index {
-                               let htlc_output = if htlc.offered {
-                                               HolderHTLCOutput::build_offered(htlc.amount_msat, htlc.cltv_expiry)
+                               let (htlc_output, aggregable) = if htlc.offered {
+                                       let htlc_output = HolderHTLCOutput::build_offered(
+                                               htlc.amount_msat, htlc.cltv_expiry, self.onchain_tx_handler.opt_anchors()
+                                       );
+                                       (htlc_output, false)
+                               } else {
+                                       let payment_preimage = if let Some(preimage) = self.payment_preimages.get(&htlc.payment_hash) {
+                                               preimage.clone()
                                        } else {
-                                               let payment_preimage = if let Some(preimage) = self.payment_preimages.get(&htlc.payment_hash) {
-                                                       preimage.clone()
-                                               } else {
-                                                       // We can't build an HTLC-Success transaction without the preimage
-                                                       continue;
-                                               };
-                                               HolderHTLCOutput::build_accepted(payment_preimage, htlc.amount_msat)
+                                               // We can't build an HTLC-Success transaction without the preimage
+                                               continue;
                                        };
-                               let htlc_package = PackageTemplate::build_package(holder_tx.txid, transaction_output_index, PackageSolvingData::HolderHTLCOutput(htlc_output), htlc.cltv_expiry, false, conf_height);
+                                       let htlc_output = HolderHTLCOutput::build_accepted(
+                                               payment_preimage, htlc.amount_msat, self.onchain_tx_handler.opt_anchors()
+                                       );
+                                       (htlc_output, self.onchain_tx_handler.opt_anchors())
+                               };
+                               let htlc_package = PackageTemplate::build_package(
+                                       holder_tx.txid, transaction_output_index,
+                                       PackageSolvingData::HolderHTLCOutput(htlc_output),
+                                       htlc.cltv_expiry, aggregable, conf_height
+                               );
                                claim_requests.push(htlc_package);
                        }
                }
@@ -2860,17 +2943,47 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
 
                let mut watch_outputs = Vec::new();
                let mut claimable_outpoints = Vec::new();
-               for tx in &txn_matched {
+               'tx_iter: for tx in &txn_matched {
+                       let txid = tx.txid();
+                       // If a transaction has already been confirmed, ensure we don't bother processing it duplicatively.
+                       if Some(txid) == self.funding_spend_confirmed {
+                               log_debug!(logger, "Skipping redundant processing of funding-spend tx {} as it was previously confirmed", txid);
+                               continue 'tx_iter;
+                       }
+                       for ev in self.onchain_events_awaiting_threshold_conf.iter() {
+                               if ev.txid == txid {
+                                       if let Some(conf_hash) = ev.block_hash {
+                                               assert_eq!(header.block_hash(), conf_hash,
+                                                       "Transaction {} was already confirmed and is being re-confirmed in a different block.\n\
+                                                       This indicates a severe bug in the transaction connection logic - a reorg should have been processed first!", ev.txid);
+                                       }
+                                       log_debug!(logger, "Skipping redundant processing of confirming tx {} as it was previously confirmed", txid);
+                                       continue 'tx_iter;
+                               }
+                       }
+                       for htlc in self.htlcs_resolved_on_chain.iter() {
+                               if Some(txid) == htlc.resolving_txid {
+                                       log_debug!(logger, "Skipping redundant processing of HTLC resolution tx {} as it was previously confirmed", txid);
+                                       continue 'tx_iter;
+                               }
+                       }
+                       for spendable_txid in self.spendable_txids_confirmed.iter() {
+                               if txid == *spendable_txid {
+                                       log_debug!(logger, "Skipping redundant processing of spendable tx {} as it was previously confirmed", txid);
+                                       continue 'tx_iter;
+                               }
+                       }
+
                        if tx.input.len() == 1 {
                                // Assuming our keys were not leaked (in which case we're screwed no matter what),
-                               // commitment transactions and HTLC transactions will all only ever have one input,
-                               // which is an easy way to filter out any potential non-matching txn for lazy
-                               // filters.
+                               // commitment transactions and HTLC transactions will all only ever have one input
+                               // (except for HTLC transactions for channels with anchor outputs), which is an easy
+                               // way to filter out any potential non-matching txn for lazy filters.
                                let prevout = &tx.input[0].previous_output;
                                if prevout.txid == self.funding_info.0.txid && prevout.vout == self.funding_info.0.index as u32 {
                                        let mut balance_spendable_csv = None;
                                        log_info!(logger, "Channel {} closed by funding output spend in txid {}.",
-                                               log_bytes!(self.funding_info.0.to_channel_id()), tx.txid());
+                                               log_bytes!(self.funding_info.0.to_channel_id()), txid);
                                        self.funding_spend_seen = true;
                                        let mut commitment_tx_to_counterparty_output = None;
                                        if (tx.input[0].sequence.0 >> 8*3) as u8 == 0x80 && (tx.lock_time.0 >> 8*3) as u8 == 0x20 {
@@ -2893,7 +3006,6 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                        }
                                                }
                                        }
-                                       let txid = tx.txid();
                                        self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry {
                                                txid,
                                                transaction: Some((*tx).clone()),
@@ -2904,22 +3016,33 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                        commitment_tx_to_counterparty_output,
                                                },
                                        });
-                               } else {
-                                       if let Some(&commitment_number) = self.counterparty_commitment_txn_on_chain.get(&prevout.txid) {
-                                               let (mut new_outpoints, new_outputs_option) = self.check_spend_counterparty_htlc(&tx, commitment_number, height, &logger);
+                               }
+                       }
+                       if tx.input.len() >= 1 {
+                               // While all commitment transactions have one input, HTLC transactions may have more
+                               // if the HTLC was present in an anchor channel. HTLCs can also be resolved in a few
+                               // other ways which can have more than one output.
+                               for tx_input in &tx.input {
+                                       let commitment_txid = tx_input.previous_output.txid;
+                                       if let Some(&commitment_number) = self.counterparty_commitment_txn_on_chain.get(&commitment_txid) {
+                                               let (mut new_outpoints, new_outputs_option) = self.check_spend_counterparty_htlc(
+                                                       &tx, commitment_number, &commitment_txid, height, &logger
+                                               );
                                                claimable_outpoints.append(&mut new_outpoints);
                                                if let Some(new_outputs) = new_outputs_option {
                                                        watch_outputs.push(new_outputs);
                                                }
+                                               // Since there may be multiple HTLCs (all from the same commitment) being
+                                               // claimed by the counterparty within the same transaction, and
+                                               // `check_spend_counterparty_htlc` already checks for all of them, we can
+                                               // safely break from our loop.
+                                               break;
                                        }
                                }
-                       }
-                       // While all commitment/HTLC-Success/HTLC-Timeout transactions have one input, HTLCs
-                       // can also be resolved in a few other ways which can have more than one output. Thus,
-                       // we call is_resolving_htlc_output here outside of the tx.input.len() == 1 check.
-                       self.is_resolving_htlc_output(&tx, height, &block_hash, &logger);
+                               self.is_resolving_htlc_output(&tx, height, &block_hash, &logger);
 
-                       self.is_paying_spendable_output(&tx, height, &block_hash, &logger);
+                               self.is_paying_spendable_output(&tx, height, &block_hash, &logger);
+                       }
                }
 
                if height > self.best_block.height() {
@@ -3033,7 +3156,9 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                htlc_value_satoshis,
                                        }));
                                        self.htlcs_resolved_on_chain.push(IrrevocablyResolvedHTLC {
-                                               commitment_tx_output_idx, resolving_txid: Some(entry.txid),
+                                               commitment_tx_output_idx,
+                                               resolving_txid: Some(entry.txid),
+                                               resolving_tx: entry.transaction,
                                                payment_preimage: None,
                                        });
                                },
@@ -3042,10 +3167,13 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                        self.pending_events.push(Event::SpendableOutputs {
                                                outputs: vec![descriptor]
                                        });
+                                       self.spendable_txids_confirmed.push(entry.txid);
                                },
                                OnchainEvent::HTLCSpendConfirmation { commitment_tx_output_idx, preimage, .. } => {
                                        self.htlcs_resolved_on_chain.push(IrrevocablyResolvedHTLC {
-                                               commitment_tx_output_idx: Some(commitment_tx_output_idx), resolving_txid: Some(entry.txid),
+                                               commitment_tx_output_idx: Some(commitment_tx_output_idx),
+                                               resolving_txid: Some(entry.txid),
+                                               resolving_tx: entry.transaction,
                                                payment_preimage: preimage,
                                        });
                                },
@@ -3111,10 +3239,24 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                F::Target: FeeEstimator,
                L::Target: Logger,
        {
-               self.onchain_events_awaiting_threshold_conf.retain(|ref entry| if entry.txid == *txid {
-                       log_info!(logger, "Removing onchain event with txid {}", txid);
-                       false
-               } else { true });
+               let mut removed_height = None;
+               for entry in self.onchain_events_awaiting_threshold_conf.iter() {
+                       if entry.txid == *txid {
+                               removed_height = Some(entry.height);
+                               break;
+                       }
+               }
+
+               if let Some(removed_height) = removed_height {
+                       log_info!(logger, "transaction_unconfirmed of txid {} implies height {} was reorg'd out", txid, removed_height);
+                       self.onchain_events_awaiting_threshold_conf.retain(|ref entry| if entry.height >= removed_height {
+                               log_info!(logger, "Transaction {} reorg'd out", entry.txid);
+                               false
+                       } else { true });
+               }
+
+               debug_assert!(!self.onchain_events_awaiting_threshold_conf.iter().any(|ref entry| entry.txid == *txid));
+
                self.onchain_tx_handler.transaction_unconfirmed(txid, broadcaster, fee_estimator, logger);
        }
 
@@ -3736,7 +3878,9 @@ impl<'a, K: KeysInterface> ReadableArgs<&'a K>
                                return Err(DecodeError::InvalidValue);
                        }
                }
-               let onchain_tx_handler: OnchainTxHandler<K::Signer> = ReadableArgs::read(reader, keys_manager)?;
+               let onchain_tx_handler: OnchainTxHandler<K::Signer> = ReadableArgs::read(
+                       reader, (keys_manager, channel_value_satoshis, channel_keys_id)
+               )?;
 
                let lockdown_from_offchain = Readable::read(reader)?;
                let holder_tx_signed = Readable::read(reader)?;
@@ -3763,6 +3907,7 @@ impl<'a, K: KeysInterface> ReadableArgs<&'a K>
                let mut funding_spend_seen = Some(false);
                let mut counterparty_node_id = None;
                let mut confirmed_commitment_tx_counterparty_output = None;
+               let mut spendable_txids_confirmed = Some(Vec::new());
                read_tlv_fields!(reader, {
                        (1, funding_spend_confirmed, option),
                        (3, htlcs_resolved_on_chain, vec_type),
@@ -3770,6 +3915,7 @@ impl<'a, K: KeysInterface> ReadableArgs<&'a K>
                        (7, funding_spend_seen, option),
                        (9, counterparty_node_id, option),
                        (11, confirmed_commitment_tx_counterparty_output, option),
+                       (13, spendable_txids_confirmed, vec_type),
                });
 
                let mut secp_ctx = Secp256k1::new();
@@ -3822,6 +3968,7 @@ impl<'a, K: KeysInterface> ReadableArgs<&'a K>
                        funding_spend_confirmed,
                        confirmed_commitment_tx_counterparty_output,
                        htlcs_resolved_on_chain: htlcs_resolved_on_chain.unwrap(),
+                       spendable_txids_confirmed: spendable_txids_confirmed.unwrap(),
 
                        best_block,
                        counterparty_node_id,
@@ -4030,7 +4177,7 @@ mod tests {
                        SecretKey::from_slice(&[41; 32]).unwrap(),
                        [41; 32],
                        0,
-                       [0; 32]
+                       [0; 32],
                );
 
                let counterparty_pubkeys = ChannelPublicKeys {
@@ -4051,6 +4198,7 @@ mod tests {
                        }),
                        funding_outpoint: Some(funding_outpoint),
                        opt_anchors: None,
+                       opt_non_zero_fee_anchors: None,
                };
                // Prune with one old state and a holder commitment tx holding a few overlaps with the
                // old state.
index b12eee017aa93b81a080bf342a267c76aa309f5e..f2d198bbfa101d97d129c251381a637a76aaf348 100644 (file)
@@ -31,10 +31,11 @@ use bitcoin::secp256k1::ecdh::SharedSecret;
 use bitcoin::secp256k1::ecdsa::RecoverableSignature;
 use bitcoin::{PackedLockTime, secp256k1, Sequence, Witness};
 
-use crate::util::{byte_utils, transaction_utils};
+use crate::util::transaction_utils;
 use crate::util::crypto::{hkdf_extract_expand_twice, sign};
 use crate::util::ser::{Writeable, Writer, Readable, ReadableArgs};
-
+#[cfg(anchors)]
+use crate::util::events::HTLCDescriptor;
 use crate::chain::transaction::OutPoint;
 use crate::ln::channel::ANCHOR_OUTPUT_VALUE_SATOSHI;
 use crate::ln::{chan_utils, PaymentPreimage};
@@ -43,6 +44,7 @@ use crate::ln::msgs::UnsignedChannelAnnouncement;
 use crate::ln::script::ShutdownScript;
 
 use crate::prelude::*;
+use core::convert::TryInto;
 use core::sync::atomic::{AtomicUsize, Ordering};
 use crate::io::{self, Error};
 use crate::ln::msgs::{DecodeError, MAX_VALUE_MSAT};
@@ -161,7 +163,7 @@ pub enum SpendableOutputDescriptor {
        ///
        /// To derive the revocation_pubkey provided here (which is used in the witness
        /// script generation), you must pass the counterparty revocation_basepoint (which appears in the
-       /// call to Sign::ready_channel) and the provided per_commitment point
+       /// call to Sign::provide_channel_parameters) and the provided per_commitment point
        /// to chan_utils::derive_public_revocation_key.
        ///
        /// The witness script which is hashed and included in the output script_pubkey may be
@@ -324,6 +326,19 @@ pub trait BaseSign {
        /// (which is committed to in the BIP 143 signatures).
        fn sign_justice_revoked_htlc(&self, justice_tx: &Transaction, input: usize, amount: u64, per_commitment_key: &SecretKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()>;
 
+       #[cfg(anchors)]
+       /// Computes the signature for a commitment transaction's HTLC output used as an input within
+       /// `htlc_tx`, which spends the commitment transaction, at index `input`. The signature returned
+       /// must be be computed using [`EcdsaSighashType::All`]. Note that this should only be used to
+       /// sign HTLC transactions from channels supporting anchor outputs after all additional
+       /// inputs/outputs have been added to the transaction.
+       ///
+       /// [`EcdsaSighashType::All`]: bitcoin::blockdata::transaction::EcdsaSighashType::All
+       fn sign_holder_htlc_transaction(
+               &self, htlc_tx: &Transaction, input: usize, htlc_descriptor: &HTLCDescriptor,
+               secp_ctx: &Secp256k1<secp256k1::All>
+       ) -> Result<Signature, ()>;
+
        /// Create a signature for a claiming transaction for a HTLC output on a counterparty's commitment
        /// transaction, either offered or received.
        ///
@@ -368,24 +383,22 @@ pub trait BaseSign {
                -> Result<(Signature, Signature), ()>;
 
        /// Set the counterparty static channel data, including basepoints,
-       /// counterparty_selected/holder_selected_contest_delay and funding outpoint.
-       /// This is done as soon as the funding outpoint is known.  Since these are static channel data,
-       /// they MUST NOT be allowed to change to different values once set.
+       /// counterparty_selected/holder_selected_contest_delay and funding outpoint. Since these are
+       /// static channel data, they MUST NOT be allowed to change to different values once set, as LDK
+       /// may call this method more than once.
        ///
        /// channel_parameters.is_populated() MUST be true.
-       ///
-       /// We bind holder_selected_contest_delay late here for API convenience.
-       ///
-       /// Will be called before any signatures are applied.
-       fn ready_channel(&mut self, channel_parameters: &ChannelTransactionParameters);
+       fn provide_channel_parameters(&mut self, channel_parameters: &ChannelTransactionParameters);
 }
 
-/// A cloneable signer.
+/// A writeable signer.
 ///
-/// Although we require signers to be cloneable, it may be useful for developers to be able to use
-/// signers in an un-sized way, for example as `dyn BaseSign`. Therefore we separate the Clone trait,
-/// which implies Sized, into this derived trait.
-pub trait Sign: BaseSign + Writeable + Clone {
+/// There will always be two instances of a signer per channel, one occupied by the
+/// [`ChannelManager`] and another by the channel's [`ChannelMonitor`].
+///
+/// [`ChannelManager`]: crate::ln::channelmanager::ChannelManager
+/// [`ChannelMonitor`]: crate::chain::channelmonitor::ChannelMonitor
+pub trait Sign: BaseSign + Writeable {
 }
 
 /// Specifies the recipient of an invoice, to indicate to [`KeysInterface::sign_invoice`] what node
@@ -402,7 +415,7 @@ pub enum Recipient {
 
 /// A trait to describe an object which can get user secrets and key material.
 pub trait KeysInterface {
-       /// A type which implements Sign which will be returned by get_channel_signer.
+       /// A type which implements Sign which will be returned by derive_channel_signer.
        type Signer : Sign;
 
        /// Get node secret key based on the provided [`Recipient`].
@@ -445,11 +458,20 @@ pub trait KeysInterface {
        /// This method should return a different value each time it is called, to avoid linking
        /// on-chain funds across channels as controlled to the same user.
        fn get_shutdown_scriptpubkey(&self) -> ShutdownScript;
-       /// Get a new set of Sign for per-channel secrets. These MUST be unique even if you
-       /// restarted with some stale data!
+       /// Generates a unique `channel_keys_id` that can be used to obtain a `Signer` through
+       /// [`KeysInterface::derive_channel_signer`]. The `user_channel_id` is provided to allow
+       /// implementations of `KeysInterface` to maintain a mapping between it and the generated
+       /// `channel_keys_id`.
        ///
        /// This method must return a different value each time it is called.
-       fn get_channel_signer(&self, inbound: bool, channel_value_satoshis: u64) -> Self::Signer;
+       fn generate_channel_keys_id(&self, inbound: bool, channel_value_satoshis: u64, user_channel_id: u128) -> [u8; 32];
+       /// Derives the private key material backing a `Signer`.
+       ///
+       /// To derive a new `Signer`, a fresh `channel_keys_id` should be obtained through
+       /// [`KeysInterface::generate_channel_keys_id`]. Otherwise, an existing `Signer` can be
+       /// re-derived from its `channel_keys_id`, which can be obtained through its trait method
+       /// [`BaseSign::channel_keys_id`].
+       fn derive_channel_signer(&self, channel_value_satoshis: u64, channel_keys_id: [u8; 32]) -> Self::Signer;
        /// Gets a unique, cryptographically-secure, random 32 byte value. This is used for encrypting
        /// onion packets and for temporary channel IDs. There is no requirement that these be
        /// persisted anywhere, though they must be unique across restarts.
@@ -463,6 +485,9 @@ pub trait KeysInterface {
        /// The bytes are exactly those which `<Self::Signer as Writeable>::write()` writes, and
        /// contain no versioning scheme. You may wish to include your own version prefix and ensure
        /// you've read all of the provided bytes to ensure no corruption occurred.
+       ///
+       /// This method is slowly being phased out -- it will only be called when reading objects
+       /// written by LDK versions prior to 0.0.113.
        fn read_chan_signer(&self, reader: &[u8]) -> Result<Self::Signer, DecodeError>;
 
        /// Sign an invoice.
@@ -533,7 +558,8 @@ impl InMemorySigner {
                htlc_base_key: SecretKey,
                commitment_seed: [u8; 32],
                channel_value_satoshis: u64,
-               channel_keys_id: [u8; 32]) -> InMemorySigner {
+               channel_keys_id: [u8; 32],
+       ) -> InMemorySigner {
                let holder_channel_pubkeys =
                        InMemorySigner::make_holder_keys(secp_ctx, &funding_key, &revocation_base_key,
                                                             &payment_key, &delayed_payment_base_key,
@@ -570,39 +596,39 @@ impl InMemorySigner {
        }
 
        /// Counterparty pubkeys.
-       /// Will panic if ready_channel wasn't called.
+       /// Will panic if provide_channel_parameters wasn't called.
        pub fn counterparty_pubkeys(&self) -> &ChannelPublicKeys { &self.get_channel_parameters().counterparty_parameters.as_ref().unwrap().pubkeys }
 
        /// The contest_delay value specified by our counterparty and applied on holder-broadcastable
        /// transactions, ie the amount of time that we have to wait to recover our funds if we
        /// broadcast a transaction.
-       /// Will panic if ready_channel wasn't called.
+       /// Will panic if provide_channel_parameters wasn't called.
        pub fn counterparty_selected_contest_delay(&self) -> u16 { self.get_channel_parameters().counterparty_parameters.as_ref().unwrap().selected_contest_delay }
 
        /// The contest_delay value specified by us and applied on transactions broadcastable
        /// by our counterparty, ie the amount of time that they have to wait to recover their funds
        /// if they broadcast a transaction.
-       /// Will panic if ready_channel wasn't called.
+       /// Will panic if provide_channel_parameters wasn't called.
        pub fn holder_selected_contest_delay(&self) -> u16 { self.get_channel_parameters().holder_selected_contest_delay }
 
        /// Whether the holder is the initiator
-       /// Will panic if ready_channel wasn't called.
+       /// Will panic if provide_channel_parameters wasn't called.
        pub fn is_outbound(&self) -> bool { self.get_channel_parameters().is_outbound_from_holder }
 
        /// Funding outpoint
-       /// Will panic if ready_channel wasn't called.
+       /// Will panic if provide_channel_parameters wasn't called.
        pub fn funding_outpoint(&self) -> &OutPoint { self.get_channel_parameters().funding_outpoint.as_ref().unwrap() }
 
        /// Obtain a ChannelTransactionParameters for this channel, to be used when verifying or
        /// building transactions.
        ///
-       /// Will panic if ready_channel wasn't called.
+       /// Will panic if provide_channel_parameters wasn't called.
        pub fn get_channel_parameters(&self) -> &ChannelTransactionParameters {
                self.channel_parameters.as_ref().unwrap()
        }
 
        /// Whether anchors should be used.
-       /// Will panic if ready_channel wasn't called.
+       /// Will panic if provide_channel_parameters wasn't called.
        pub fn opt_anchors(&self) -> bool {
                self.get_channel_parameters().opt_anchors.is_some()
        }
@@ -654,8 +680,7 @@ impl InMemorySigner {
                if spend_tx.input[input_idx].previous_output != descriptor.outpoint.into_bitcoin_outpoint() { return Err(()); }
                if spend_tx.input[input_idx].sequence.0 != descriptor.to_self_delay as u32 { return Err(()); }
 
-               let delayed_payment_key = chan_utils::derive_private_key(&secp_ctx, &descriptor.per_commitment_point, &self.delayed_payment_base_key)
-                       .expect("We constructed the payment_base_key, so we can only fail here if the RNG is busted.");
+               let delayed_payment_key = chan_utils::derive_private_key(&secp_ctx, &descriptor.per_commitment_point, &self.delayed_payment_base_key);
                let delayed_payment_pubkey = PublicKey::from_secret_key(&secp_ctx, &delayed_payment_key);
                let witness_script = chan_utils::get_revokeable_redeemscript(&descriptor.revocation_pubkey, descriptor.to_self_delay, &delayed_payment_pubkey);
                let sighash = hash_to_message!(&sighash::SighashCache::new(spend_tx).segwit_signature_hash(input_idx, &witness_script, descriptor.output.value, EcdsaSighashType::All).unwrap()[..]);
@@ -671,7 +696,6 @@ impl InMemorySigner {
                witness.push(witness_script.clone().into_bytes());
                Ok(witness)
        }
-
 }
 
 impl BaseSign for InMemorySigner {
@@ -704,11 +728,12 @@ impl BaseSign for InMemorySigner {
 
                let mut htlc_sigs = Vec::with_capacity(commitment_tx.htlcs().len());
                for htlc in commitment_tx.htlcs() {
-                       let htlc_tx = chan_utils::build_htlc_transaction(&commitment_txid, commitment_tx.feerate_per_kw(), self.holder_selected_contest_delay(), htlc, self.opt_anchors(), &keys.broadcaster_delayed_payment_key, &keys.revocation_key);
+                       let channel_parameters = self.get_channel_parameters();
+                       let htlc_tx = chan_utils::build_htlc_transaction(&commitment_txid, commitment_tx.feerate_per_kw(), self.holder_selected_contest_delay(), htlc, self.opt_anchors(), channel_parameters.opt_non_zero_fee_anchors.is_some(), &keys.broadcaster_delayed_payment_key, &keys.revocation_key);
                        let htlc_redeemscript = chan_utils::get_htlc_redeemscript(&htlc, self.opt_anchors(), &keys);
                        let htlc_sighashtype = if self.opt_anchors() { EcdsaSighashType::SinglePlusAnyoneCanPay } else { EcdsaSighashType::All };
                        let htlc_sighash = hash_to_message!(&sighash::SighashCache::new(&htlc_tx).segwit_signature_hash(0, &htlc_redeemscript, htlc.amount_msat / 1000, htlc_sighashtype).unwrap()[..]);
-                       let holder_htlc_key = chan_utils::derive_private_key(&secp_ctx, &keys.per_commitment_point, &self.htlc_base_key).map_err(|_| ())?;
+                       let holder_htlc_key = chan_utils::derive_private_key(&secp_ctx, &keys.per_commitment_point, &self.htlc_base_key);
                        htlc_sigs.push(sign(secp_ctx, &htlc_sighash, &holder_htlc_key));
                }
 
@@ -741,11 +766,11 @@ impl BaseSign for InMemorySigner {
        }
 
        fn sign_justice_revoked_output(&self, justice_tx: &Transaction, input: usize, amount: u64, per_commitment_key: &SecretKey, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
-               let revocation_key = chan_utils::derive_private_revocation_key(&secp_ctx, &per_commitment_key, &self.revocation_base_key).map_err(|_| ())?;
+               let revocation_key = chan_utils::derive_private_revocation_key(&secp_ctx, &per_commitment_key, &self.revocation_base_key);
                let per_commitment_point = PublicKey::from_secret_key(secp_ctx, &per_commitment_key);
-               let revocation_pubkey = chan_utils::derive_public_revocation_key(&secp_ctx, &per_commitment_point, &self.pubkeys().revocation_basepoint).map_err(|_| ())?;
+               let revocation_pubkey = chan_utils::derive_public_revocation_key(&secp_ctx, &per_commitment_point, &self.pubkeys().revocation_basepoint);
                let witness_script = {
-                       let counterparty_delayedpubkey = chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &self.counterparty_pubkeys().delayed_payment_basepoint).map_err(|_| ())?;
+                       let counterparty_delayedpubkey = chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &self.counterparty_pubkeys().delayed_payment_basepoint);
                        chan_utils::get_revokeable_redeemscript(&revocation_pubkey, self.holder_selected_contest_delay(), &counterparty_delayedpubkey)
                };
                let mut sighash_parts = sighash::SighashCache::new(justice_tx);
@@ -754,12 +779,12 @@ impl BaseSign for InMemorySigner {
        }
 
        fn sign_justice_revoked_htlc(&self, justice_tx: &Transaction, input: usize, amount: u64, per_commitment_key: &SecretKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
-               let revocation_key = chan_utils::derive_private_revocation_key(&secp_ctx, &per_commitment_key, &self.revocation_base_key).map_err(|_| ())?;
+               let revocation_key = chan_utils::derive_private_revocation_key(&secp_ctx, &per_commitment_key, &self.revocation_base_key);
                let per_commitment_point = PublicKey::from_secret_key(secp_ctx, &per_commitment_key);
-               let revocation_pubkey = chan_utils::derive_public_revocation_key(&secp_ctx, &per_commitment_point, &self.pubkeys().revocation_basepoint).map_err(|_| ())?;
+               let revocation_pubkey = chan_utils::derive_public_revocation_key(&secp_ctx, &per_commitment_point, &self.pubkeys().revocation_basepoint);
                let witness_script = {
-                       let counterparty_htlcpubkey = chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &self.counterparty_pubkeys().htlc_basepoint).map_err(|_| ())?;
-                       let holder_htlcpubkey = chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &self.pubkeys().htlc_basepoint).map_err(|_| ())?;
+                       let counterparty_htlcpubkey = chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &self.counterparty_pubkeys().htlc_basepoint);
+                       let holder_htlcpubkey = chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &self.pubkeys().htlc_basepoint);
                        chan_utils::get_htlc_redeemscript_with_explicit_keys(&htlc, self.opt_anchors(), &counterparty_htlcpubkey, &holder_htlcpubkey, &revocation_pubkey)
                };
                let mut sighash_parts = sighash::SighashCache::new(justice_tx);
@@ -767,20 +792,33 @@ impl BaseSign for InMemorySigner {
                return Ok(sign(secp_ctx, &sighash, &revocation_key))
        }
 
+       #[cfg(anchors)]
+       fn sign_holder_htlc_transaction(
+               &self, htlc_tx: &Transaction, input: usize, htlc_descriptor: &HTLCDescriptor,
+               secp_ctx: &Secp256k1<secp256k1::All>
+       ) -> Result<Signature, ()> {
+               let per_commitment_point = self.get_per_commitment_point(
+                       htlc_descriptor.per_commitment_number, &secp_ctx
+               );
+               let witness_script = htlc_descriptor.witness_script(&per_commitment_point, secp_ctx);
+               let sighash = &sighash::SighashCache::new(&*htlc_tx).segwit_signature_hash(
+                       input, &witness_script, htlc_descriptor.htlc.amount_msat / 1000, EcdsaSighashType::All
+               ).map_err(|_| ())?;
+               let our_htlc_private_key = chan_utils::derive_private_key(
+                       &secp_ctx, &per_commitment_point, &self.htlc_base_key
+               );
+               Ok(sign(&secp_ctx, &hash_to_message!(sighash), &our_htlc_private_key))
+       }
+
        fn sign_counterparty_htlc_transaction(&self, htlc_tx: &Transaction, input: usize, amount: u64, per_commitment_point: &PublicKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
-               if let Ok(htlc_key) = chan_utils::derive_private_key(&secp_ctx, &per_commitment_point, &self.htlc_base_key) {
-                       let witness_script = if let Ok(revocation_pubkey) = chan_utils::derive_public_revocation_key(&secp_ctx, &per_commitment_point, &self.pubkeys().revocation_basepoint) {
-                               if let Ok(counterparty_htlcpubkey) = chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &self.counterparty_pubkeys().htlc_basepoint) {
-                                       if let Ok(htlcpubkey) = chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &self.pubkeys().htlc_basepoint) {
-                                               chan_utils::get_htlc_redeemscript_with_explicit_keys(&htlc, self.opt_anchors(), &counterparty_htlcpubkey, &htlcpubkey, &revocation_pubkey)
-                                       } else { return Err(()) }
-                               } else { return Err(()) }
-                       } else { return Err(()) };
-                       let mut sighash_parts = sighash::SighashCache::new(htlc_tx);
-                       let sighash = hash_to_message!(&sighash_parts.segwit_signature_hash(input, &witness_script, amount, EcdsaSighashType::All).unwrap()[..]);
-                       return Ok(sign(secp_ctx, &sighash, &htlc_key))
-               }
-               Err(())
+               let htlc_key = chan_utils::derive_private_key(&secp_ctx, &per_commitment_point, &self.htlc_base_key);
+               let revocation_pubkey = chan_utils::derive_public_revocation_key(&secp_ctx, &per_commitment_point, &self.pubkeys().revocation_basepoint);
+               let counterparty_htlcpubkey = chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &self.counterparty_pubkeys().htlc_basepoint);
+               let htlcpubkey = chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &self.pubkeys().htlc_basepoint);
+               let witness_script = chan_utils::get_htlc_redeemscript_with_explicit_keys(&htlc, self.opt_anchors(), &counterparty_htlcpubkey, &htlcpubkey, &revocation_pubkey);
+               let mut sighash_parts = sighash::SighashCache::new(htlc_tx);
+               let sighash = hash_to_message!(&sighash_parts.segwit_signature_hash(input, &witness_script, amount, EcdsaSighashType::All).unwrap()[..]);
+               Ok(sign(secp_ctx, &sighash, &htlc_key))
        }
 
        fn sign_closing_transaction(&self, closing_tx: &ClosingTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
@@ -805,8 +843,12 @@ impl BaseSign for InMemorySigner {
                Ok((sign(secp_ctx, &msghash, &self.node_secret), sign(secp_ctx, &msghash, &self.funding_key)))
        }
 
-       fn ready_channel(&mut self, channel_parameters: &ChannelTransactionParameters) {
-               assert!(self.channel_parameters.is_none(), "Acceptance already noted");
+       fn provide_channel_parameters(&mut self, channel_parameters: &ChannelTransactionParameters) {
+               assert!(self.channel_parameters.is_none() || self.channel_parameters.as_ref().unwrap() == channel_parameters);
+               if self.channel_parameters.is_some() {
+                       // The channel parameters were already set and they match, return early.
+                       return;
+               }
                assert!(channel_parameters.is_populated(), "Channel parameters must be fully populated");
                self.channel_parameters = Some(channel_parameters.clone());
        }
@@ -953,8 +995,8 @@ impl KeysManager {
                                inbound_pmt_key_bytes.copy_from_slice(&inbound_payment_key[..]);
 
                                let mut rand_bytes_unique_start = Sha256::engine();
-                               rand_bytes_unique_start.input(&byte_utils::be64_to_array(starting_time_secs));
-                               rand_bytes_unique_start.input(&byte_utils::be32_to_array(starting_time_nanos));
+                               rand_bytes_unique_start.input(&starting_time_secs.to_be_bytes());
+                               rand_bytes_unique_start.input(&starting_time_nanos.to_be_bytes());
                                rand_bytes_unique_start.input(seed);
 
                                let mut res = KeysManager {
@@ -985,13 +1027,8 @@ impl KeysManager {
                }
        }
        /// Derive an old Sign containing per-channel secrets based on a key derivation parameters.
-       ///
-       /// Key derivation parameters are accessible through a per-channel secrets
-       /// Sign::channel_keys_id and is provided inside DynamicOuputP2WSH in case of
-       /// onchain output detection for which a corresponding delayed_payment_key must be derived.
        pub fn derive_channel_keys(&self, channel_value_satoshis: u64, params: &[u8; 32]) -> InMemorySigner {
-               let chan_id = byte_utils::slice_to_be64(&params[0..8]);
-               assert!(chan_id <= core::u32::MAX as u64); // Otherwise the params field wasn't created by us
+               let chan_id = u64::from_be_bytes(params[0..8].try_into().unwrap());
                let mut unique_start = Sha256::engine();
                unique_start.input(params);
                unique_start.input(&self.seed);
@@ -1035,7 +1072,7 @@ impl KeysManager {
                        htlc_base_key,
                        commitment_seed,
                        channel_value_satoshis,
-                       params.clone()
+                       params.clone(),
                )
        }
 
@@ -1207,14 +1244,19 @@ impl KeysInterface for KeysManager {
                ShutdownScript::new_p2wpkh_from_pubkey(self.shutdown_pubkey.clone())
        }
 
-       fn get_channel_signer(&self, _inbound: bool, channel_value_satoshis: u64) -> Self::Signer {
-               let child_ix = self.channel_child_index.fetch_add(1, Ordering::AcqRel);
-               assert!(child_ix <= core::u32::MAX as usize);
+       fn generate_channel_keys_id(&self, _inbound: bool, _channel_value_satoshis: u64, user_channel_id: u128) -> [u8; 32] {
+               let child_idx = self.channel_child_index.fetch_add(1, Ordering::AcqRel);
+               assert!(child_idx <= core::u32::MAX as usize);
                let mut id = [0; 32];
-               id[0..8].copy_from_slice(&byte_utils::be64_to_array(child_ix as u64));
-               id[8..16].copy_from_slice(&byte_utils::be64_to_array(self.starting_time_nanos as u64));
-               id[16..24].copy_from_slice(&byte_utils::be64_to_array(self.starting_time_secs));
-               self.derive_channel_keys(channel_value_satoshis, &id)
+               id[0..4].copy_from_slice(&(child_idx as u32).to_be_bytes());
+               id[4..8].copy_from_slice(&self.starting_time_nanos.to_be_bytes());
+               id[8..16].copy_from_slice(&self.starting_time_secs.to_be_bytes());
+               id[16..32].copy_from_slice(&user_channel_id.to_be_bytes());
+               id
+       }
+
+       fn derive_channel_signer(&self, channel_value_satoshis: u64, channel_keys_id: [u8; 32]) -> Self::Signer {
+               self.derive_channel_keys(channel_value_satoshis, &channel_keys_id)
        }
 
        fn get_secure_random_bytes(&self) -> [u8; 32] {
@@ -1307,8 +1349,12 @@ impl KeysInterface for PhantomKeysManager {
                self.inner.get_shutdown_scriptpubkey()
        }
 
-       fn get_channel_signer(&self, inbound: bool, channel_value_satoshis: u64) -> Self::Signer {
-               self.inner.get_channel_signer(inbound, channel_value_satoshis)
+       fn generate_channel_keys_id(&self, inbound: bool, channel_value_satoshis: u64, user_channel_id: u128) -> [u8; 32] {
+               self.inner.generate_channel_keys_id(inbound, channel_value_satoshis, user_channel_id)
+       }
+
+       fn derive_channel_signer(&self, channel_value_satoshis: u64, channel_keys_id: [u8; 32]) -> Self::Signer {
+               self.inner.derive_channel_signer(channel_value_satoshis, channel_keys_id)
        }
 
        fn get_secure_random_bytes(&self) -> [u8; 32] {
index 2ce2ed41ba1fcf0e00325aec5005e8757d10c76b..5bf3a8fd4823369902fba99752ef4b546aebaad3 100644 (file)
@@ -21,10 +21,11 @@ use bitcoin::hash_types::{Txid, BlockHash};
 use bitcoin::secp256k1::{Secp256k1, ecdsa::Signature};
 use bitcoin::secp256k1;
 
+use crate::chain::keysinterface::BaseSign;
 use crate::ln::msgs::DecodeError;
 use crate::ln::PaymentPreimage;
 #[cfg(anchors)]
-use crate::ln::chan_utils;
+use crate::ln::chan_utils::{self, HTLCOutputInCommitment};
 use crate::ln::chan_utils::{ChannelTransactionParameters, HolderCommitmentTransaction};
 #[cfg(anchors)]
 use crate::chain::chaininterface::ConfirmationTarget;
@@ -36,7 +37,6 @@ use crate::chain::package::PackageSolvingData;
 use crate::chain::package::PackageTemplate;
 use crate::util::logger::Logger;
 use crate::util::ser::{Readable, ReadableArgs, MaybeReadable, Writer, Writeable, VecWriter};
-use crate::util::byte_utils;
 
 use crate::io;
 use crate::prelude::*;
@@ -174,6 +174,16 @@ impl Writeable for Option<Vec<Option<(usize, Signature)>>> {
        }
 }
 
+#[cfg(anchors)]
+/// The claim commonly referred to as the pre-signed second-stage HTLC transaction.
+pub(crate) struct ExternalHTLCClaim {
+       pub(crate) commitment_txid: Txid,
+       pub(crate) per_commitment_number: u64,
+       pub(crate) htlc: HTLCOutputInCommitment,
+       pub(crate) preimage: Option<PaymentPreimage>,
+       pub(crate) counterparty_sig: Signature,
+}
+
 // Represents the different types of claims for which events are yielded externally to satisfy said
 // claims.
 #[cfg(anchors)]
@@ -185,6 +195,12 @@ pub(crate) enum ClaimEvent {
                commitment_tx: Transaction,
                anchor_output_idx: u32,
        },
+       /// Event yielded to signal that the commitment transaction has confirmed and its HTLCs must be
+       /// resolved by broadcasting a transaction with sufficient fee to claim them.
+       BumpHTLC {
+               target_feerate_sat_per_1000_weight: u32,
+               htlcs: Vec<ExternalHTLCClaim>,
+       },
 }
 
 /// Represents the different ways an output can be claimed (i.e., spent to an address under our
@@ -198,6 +214,9 @@ pub(crate) enum OnchainClaim {
        Event(ClaimEvent),
 }
 
+/// An internal identifier to track pending package claims within the `OnchainTxHandler`.
+type PackageID = [u8; 32];
+
 /// OnchainTxHandler receives claiming requests, aggregates them if it's sound, broadcast and
 /// do RBF bumping if possible.
 pub struct OnchainTxHandler<ChannelSigner: Sign> {
@@ -225,11 +244,11 @@ pub struct OnchainTxHandler<ChannelSigner: Sign> {
        // us and is immutable until all outpoint of the claimable set are post-anti-reorg-delay solved.
        // Entry is cache of elements need to generate a bumped claiming transaction (see ClaimTxBumpMaterial)
        #[cfg(test)] // Used in functional_test to verify sanitization
-       pub(crate) pending_claim_requests: HashMap<Txid, PackageTemplate>,
+       pub(crate) pending_claim_requests: HashMap<PackageID, PackageTemplate>,
        #[cfg(not(test))]
-       pending_claim_requests: HashMap<Txid, PackageTemplate>,
+       pending_claim_requests: HashMap<PackageID, PackageTemplate>,
        #[cfg(anchors)]
-       pending_claim_events: HashMap<Txid, ClaimEvent>,
+       pending_claim_events: HashMap<PackageID, ClaimEvent>,
 
        // Used to link outpoints claimed in a connected block to a pending claim request.
        // Key is outpoint than monitor parsing has detected we have keys/scripts to claim
@@ -238,9 +257,9 @@ pub struct OnchainTxHandler<ChannelSigner: Sign> {
        // post-anti-reorg-delay solved, confirmaiton_block is used to erase entry if
        // block with output gets disconnected.
        #[cfg(test)] // Used in functional_test to verify sanitization
-       pub claimable_outpoints: HashMap<BitcoinOutPoint, (Txid, u32)>,
+       pub claimable_outpoints: HashMap<BitcoinOutPoint, (PackageID, u32)>,
        #[cfg(not(test))]
-       claimable_outpoints: HashMap<BitcoinOutPoint, (Txid, u32)>,
+       claimable_outpoints: HashMap<BitcoinOutPoint, (PackageID, u32)>,
 
        locktimed_packages: BTreeMap<u32, Vec<PackageTemplate>>,
 
@@ -271,29 +290,29 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                (key_data.0.len() as u32).write(writer)?;
                writer.write_all(&key_data.0[..])?;
 
-               writer.write_all(&byte_utils::be64_to_array(self.pending_claim_requests.len() as u64))?;
+               writer.write_all(&(self.pending_claim_requests.len() as u64).to_be_bytes())?;
                for (ref ancestor_claim_txid, request) in self.pending_claim_requests.iter() {
                        ancestor_claim_txid.write(writer)?;
                        request.write(writer)?;
                }
 
-               writer.write_all(&byte_utils::be64_to_array(self.claimable_outpoints.len() as u64))?;
+               writer.write_all(&(self.claimable_outpoints.len() as u64).to_be_bytes())?;
                for (ref outp, ref claim_and_height) in self.claimable_outpoints.iter() {
                        outp.write(writer)?;
                        claim_and_height.0.write(writer)?;
                        claim_and_height.1.write(writer)?;
                }
 
-               writer.write_all(&byte_utils::be64_to_array(self.locktimed_packages.len() as u64))?;
+               writer.write_all(&(self.locktimed_packages.len() as u64).to_be_bytes())?;
                for (ref locktime, ref packages) in self.locktimed_packages.iter() {
                        locktime.write(writer)?;
-                       writer.write_all(&byte_utils::be64_to_array(packages.len() as u64))?;
+                       writer.write_all(&(packages.len() as u64).to_be_bytes())?;
                        for ref package in packages.iter() {
                                package.write(writer)?;
                        }
                }
 
-               writer.write_all(&byte_utils::be64_to_array(self.onchain_events_awaiting_threshold_conf.len() as u64))?;
+               writer.write_all(&(self.onchain_events_awaiting_threshold_conf.len() as u64).to_be_bytes())?;
                for ref entry in self.onchain_events_awaiting_threshold_conf.iter() {
                        entry.write(writer)?;
                }
@@ -303,8 +322,12 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
        }
 }
 
-impl<'a, K: KeysInterface> ReadableArgs<&'a K> for OnchainTxHandler<K::Signer> {
-       fn read<R: io::Read>(reader: &mut R, keys_manager: &'a K) -> Result<Self, DecodeError> {
+impl<'a, K: KeysInterface> ReadableArgs<(&'a K, u64, [u8; 32])> for OnchainTxHandler<K::Signer> {
+       fn read<R: io::Read>(reader: &mut R, args: (&'a K, u64, [u8; 32])) -> Result<Self, DecodeError> {
+               let keys_manager = args.0;
+               let channel_value_satoshis = args.1;
+               let channel_keys_id = args.2;
+
                let _ver = read_ver_prefix!(reader, SERIALIZATION_VERSION);
 
                let destination_script = Readable::read(reader)?;
@@ -316,16 +339,21 @@ impl<'a, K: KeysInterface> ReadableArgs<&'a K> for OnchainTxHandler<K::Signer> {
 
                let channel_parameters = Readable::read(reader)?;
 
+               // Read the serialized signer bytes, but don't deserialize them, as we'll obtain our signer
+               // by re-deriving the private key material.
                let keys_len: u32 = Readable::read(reader)?;
-               let mut keys_data = Vec::with_capacity(cmp::min(keys_len as usize, MAX_ALLOC_SIZE));
-               while keys_data.len() != keys_len as usize {
+               let mut bytes_read = 0;
+               while bytes_read != keys_len as usize {
                        // Read 1KB at a time to avoid accidentally allocating 4GB on corrupted channel keys
                        let mut data = [0; 1024];
-                       let read_slice = &mut data[0..cmp::min(1024, keys_len as usize - keys_data.len())];
+                       let bytes_to_read = cmp::min(1024, keys_len as usize - bytes_read);
+                       let read_slice = &mut data[0..bytes_to_read];
                        reader.read_exact(read_slice)?;
-                       keys_data.extend_from_slice(read_slice);
+                       bytes_read += bytes_to_read;
                }
-               let signer = keys_manager.read_chan_signer(&keys_data)?;
+
+               let mut signer = keys_manager.derive_channel_signer(channel_value_satoshis, channel_keys_id);
+               signer.provide_channel_parameters(&channel_parameters);
 
                let pending_claim_requests_len: u64 = Readable::read(reader)?;
                let mut pending_claim_requests = HashMap::with_capacity(cmp::min(pending_claim_requests_len as usize, MAX_ALLOC_SIZE / 128));
@@ -453,7 +481,7 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                                // since requests can have outpoints split off.
                                if !self.onchain_events_awaiting_threshold_conf.iter()
                                        .any(|event_entry| if let OnchainEvent::Claim { claim_request } = event_entry.event {
-                                               first_claim_txid_height.0 == claim_request
+                                               first_claim_txid_height.0 == claim_request.into_inner()
                                        } else {
                                                // The onchain event is not a claim, keep seeking until we find one.
                                                false
@@ -476,15 +504,36 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                // didn't receive confirmation of it before, or not enough reorg-safe depth on top of it).
                let new_timer = Some(cached_request.get_height_timer(cur_height));
                if cached_request.is_malleable() {
+                       #[cfg(anchors)]
+                       { // Attributes are not allowed on if expressions on our current MSRV of 1.41.
+                               if cached_request.requires_external_funding() {
+                                       let target_feerate_sat_per_1000_weight = cached_request
+                                               .compute_package_feerate(fee_estimator, ConfirmationTarget::HighPriority);
+                                       if let Some(htlcs) = cached_request.construct_malleable_package_with_external_funding(self) {
+                                               return Some((
+                                                       new_timer,
+                                                       target_feerate_sat_per_1000_weight as u64,
+                                                       OnchainClaim::Event(ClaimEvent::BumpHTLC {
+                                                               target_feerate_sat_per_1000_weight,
+                                                               htlcs,
+                                                       }),
+                                               ));
+                                       } else {
+                                               return None;
+                                       }
+                               }
+                       }
+
                        let predicted_weight = cached_request.package_weight(&self.destination_script);
-                       if let Some((output_value, new_feerate)) =
-                                       cached_request.compute_package_output(predicted_weight, self.destination_script.dust_value().to_sat(), fee_estimator, logger) {
+                       if let Some((output_value, new_feerate)) = cached_request.compute_package_output(
+                               predicted_weight, self.destination_script.dust_value().to_sat(), fee_estimator, logger,
+                       ) {
                                assert!(new_feerate != 0);
 
                                let transaction = cached_request.finalize_malleable_package(self, output_value, self.destination_script.clone(), logger).unwrap();
                                log_trace!(logger, "...with timer {} and feerate {}", new_timer.unwrap(), new_feerate);
                                assert!(predicted_weight >= transaction.weight());
-                               return Some((new_timer, new_feerate, OnchainClaim::Tx(transaction)))
+                               return Some((new_timer, new_feerate, OnchainClaim::Tx(transaction)));
                        }
                } else {
                        // Untractable packages cannot have their fees bumped through Replace-By-Fee. Some
@@ -540,7 +589,7 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                                        debug_assert!(false, "Only HolderFundingOutput inputs should be untractable and require external funding");
                                        None
                                },
-                       });
+                       })
                }
                None
        }
@@ -619,27 +668,38 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                        if let Some((new_timer, new_feerate, claim)) = self.generate_claim(cur_height, &req, &*fee_estimator, &*logger) {
                                req.set_timer(new_timer);
                                req.set_feerate(new_feerate);
-                               let txid = match claim {
+                               let package_id = match claim {
                                        OnchainClaim::Tx(tx) => {
                                                log_info!(logger, "Broadcasting onchain {}", log_tx!(tx));
                                                broadcaster.broadcast_transaction(&tx);
-                                               tx.txid()
+                                               tx.txid().into_inner()
                                        },
                                        #[cfg(anchors)]
                                        OnchainClaim::Event(claim_event) => {
                                                log_info!(logger, "Yielding onchain event to spend inputs {:?}", req.outpoints());
-                                               let txid = match claim_event {
-                                                       ClaimEvent::BumpCommitment { ref commitment_tx, .. } => commitment_tx.txid(),
+                                               let package_id = match claim_event {
+                                                       ClaimEvent::BumpCommitment { ref commitment_tx, .. } => commitment_tx.txid().into_inner(),
+                                                       ClaimEvent::BumpHTLC { ref htlcs, .. } => {
+                                                               // Use the same construction as a lightning channel id to generate
+                                                               // the package id for this request based on the first HTLC. It
+                                                               // doesn't matter what we use as long as it's unique per request.
+                                                               let mut package_id = [0; 32];
+                                                               package_id[..].copy_from_slice(&htlcs[0].commitment_txid[..]);
+                                                               let htlc_output_index = htlcs[0].htlc.transaction_output_index.unwrap();
+                                                               package_id[30] ^= ((htlc_output_index >> 8) & 0xff) as u8;
+                                                               package_id[31] ^= ((htlc_output_index >> 0) & 0xff) as u8;
+                                                               package_id
+                                                       },
                                                };
-                                               self.pending_claim_events.insert(txid, claim_event);
-                                               txid
+                                               self.pending_claim_events.insert(package_id, claim_event);
+                                               package_id
                                        },
                                };
                                for k in req.outpoints() {
                                        log_info!(logger, "Registering claiming request for {}:{}", k.txid, k.vout);
-                                       self.claimable_outpoints.insert(k.clone(), (txid, conf_height));
+                                       self.claimable_outpoints.insert(k.clone(), (package_id, conf_height));
                                }
-                               self.pending_claim_requests.insert(txid, req);
+                               self.pending_claim_requests.insert(package_id, req);
                        }
                }
        }
@@ -672,15 +732,32 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                                                //... we need to verify equality between transaction outpoints and claim request
                                                // outpoints to know if transaction is the original claim or a bumped one issued
                                                // by us.
-                                               let mut set_equality = true;
-                                               if request.outpoints().len() != tx.input.len() {
-                                                       set_equality = false;
+                                               let mut are_sets_equal = true;
+                                               if !request.requires_external_funding() || !request.is_malleable() {
+                                                       // If the claim does not require external funds to be allocated through
+                                                       // additional inputs we can simply check the inputs in order as they
+                                                       // cannot change under us.
+                                                       if request.outpoints().len() != tx.input.len() {
+                                                               are_sets_equal = false;
+                                                       } else {
+                                                               for (claim_inp, tx_inp) in request.outpoints().iter().zip(tx.input.iter()) {
+                                                                       if **claim_inp != tx_inp.previous_output {
+                                                                               are_sets_equal = false;
+                                                                       }
+                                                               }
+                                                       }
                                                } else {
-                                                       for (claim_inp, tx_inp) in request.outpoints().iter().zip(tx.input.iter()) {
-                                                               if **claim_inp != tx_inp.previous_output {
-                                                                       set_equality = false;
+                                                       // Otherwise, we'll do a linear search for each input (we don't expect
+                                                       // large input sets to exist) to ensure the request's input set is fully
+                                                       // spent to be resilient against the external claim reordering inputs.
+                                                       let mut spends_all_inputs = true;
+                                                       for request_input in request.outpoints() {
+                                                               if tx.input.iter().find(|input| input.previous_output == *request_input).is_none() {
+                                                                       spends_all_inputs = false;
+                                                                       break;
                                                                }
                                                        }
+                                                       are_sets_equal = spends_all_inputs;
                                                }
 
                                                macro_rules! clean_claim_request_after_safety_delay {
@@ -689,7 +766,7 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                                                                        txid: tx.txid(),
                                                                        height: conf_height,
                                                                        block_hash: Some(conf_hash),
-                                                                       event: OnchainEvent::Claim { claim_request: first_claim_txid_height.0.clone() }
+                                                                       event: OnchainEvent::Claim { claim_request: Txid::from_inner(first_claim_txid_height.0) }
                                                                };
                                                                if !self.onchain_events_awaiting_threshold_conf.contains(&entry) {
                                                                        self.onchain_events_awaiting_threshold_conf.push(entry);
@@ -700,7 +777,7 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                                                // If this is our transaction (or our counterparty spent all the outputs
                                                // before we could anyway with same inputs order than us), wait for
                                                // ANTI_REORG_DELAY and clean the RBF tracking map.
-                                               if set_equality {
+                                               if are_sets_equal {
                                                        clean_claim_request_after_safety_delay!();
                                                } else { // If false, generate new claim request with update outpoint set
                                                        let mut at_least_one_drop = false;
@@ -745,14 +822,15 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                        if entry.has_reached_confirmation_threshold(cur_height) {
                                match entry.event {
                                        OnchainEvent::Claim { claim_request } => {
+                                               let package_id = claim_request.into_inner();
                                                // We may remove a whole set of claim outpoints here, as these one may have
                                                // been aggregated in a single tx and claimed so atomically
-                                               if let Some(request) = self.pending_claim_requests.remove(&claim_request) {
+                                               if let Some(request) = self.pending_claim_requests.remove(&package_id) {
                                                        for outpoint in request.outpoints() {
                                                                log_debug!(logger, "Removing claim tracking for {} due to maturation of claim tx {}.", outpoint, claim_request);
                                                                self.claimable_outpoints.remove(&outpoint);
                                                                #[cfg(anchors)]
-                                                               self.pending_claim_events.remove(&claim_request);
+                                                               self.pending_claim_events.remove(&package_id);
                                                        }
                                                }
                                        },
@@ -985,6 +1063,37 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                htlc_tx
        }
 
+       #[cfg(anchors)]
+       pub(crate) fn generate_external_htlc_claim(
+               &mut self, outp: &::bitcoin::OutPoint, preimage: &Option<PaymentPreimage>
+       ) -> Option<ExternalHTLCClaim> {
+               let find_htlc = |holder_commitment: &HolderCommitmentTransaction| -> Option<ExternalHTLCClaim> {
+                       let trusted_tx = holder_commitment.trust();
+                       if outp.txid != trusted_tx.txid() {
+                               return None;
+                       }
+                       trusted_tx.htlcs().iter().enumerate()
+                               .find(|(_, htlc)| if let Some(output_index) = htlc.transaction_output_index {
+                                       output_index == outp.vout
+                               } else {
+                                       false
+                               })
+                               .map(|(htlc_idx, htlc)| {
+                                       let counterparty_htlc_sig = holder_commitment.counterparty_htlc_sigs[htlc_idx];
+                                       ExternalHTLCClaim {
+                                               commitment_txid: trusted_tx.txid(),
+                                               per_commitment_number: trusted_tx.commitment_number(),
+                                               htlc: htlc.clone(),
+                                               preimage: *preimage,
+                                               counterparty_sig: counterparty_htlc_sig,
+                                       }
+                               })
+               };
+               // Check if the HTLC spends from the current holder commitment or the previous one otherwise.
+               find_htlc(&self.holder_commitment)
+                       .or_else(|| self.prev_holder_commitment.as_ref().map(|c| find_htlc(c)).flatten())
+       }
+
        pub(crate) fn opt_anchors(&self) -> bool {
                self.channel_transaction_parameters.opt_anchors.is_some()
        }
index 1307ad0eb3f1a23bc7eed7f613eb61900cbbcb64..227e5ccd119188204b6f374300b0e5dceb777f46 100644 (file)
@@ -26,8 +26,9 @@ use crate::ln::chan_utils;
 use crate::ln::msgs::DecodeError;
 use crate::chain::chaininterface::{FeeEstimator, ConfirmationTarget, MIN_RELAY_FEE_SAT_PER_1000_WEIGHT};
 use crate::chain::keysinterface::Sign;
+#[cfg(anchors)]
+use crate::chain::onchaintx::ExternalHTLCClaim;
 use crate::chain::onchaintx::OnchainTxHandler;
-use crate::util::byte_utils;
 use crate::util::logger::Logger;
 use crate::util::ser::{Readable, Writer, Writeable};
 
@@ -202,11 +203,11 @@ impl CounterpartyOfferedHTLCOutput {
 
 impl_writeable_tlv_based!(CounterpartyOfferedHTLCOutput, {
        (0, per_commitment_point, required),
-       (1, opt_anchors, option),
        (2, counterparty_delayed_payment_base_key, required),
        (4, counterparty_htlc_base_key, required),
        (6, preimage, required),
        (8, htlc, required),
+       (10, opt_anchors, option),
 });
 
 /// A struct to describe a HTLC output on a counterparty commitment transaction.
@@ -240,10 +241,10 @@ impl CounterpartyReceivedHTLCOutput {
 
 impl_writeable_tlv_based!(CounterpartyReceivedHTLCOutput, {
        (0, per_commitment_point, required),
-       (1, opt_anchors, option),
        (2, counterparty_delayed_payment_base_key, required),
        (4, counterparty_htlc_base_key, required),
        (6, htlc, required),
+       (8, opt_anchors, option),
 });
 
 /// A struct to describe a HTLC output on holder commitment transaction.
@@ -253,33 +254,41 @@ impl_writeable_tlv_based!(CounterpartyReceivedHTLCOutput, {
 #[derive(Clone, PartialEq, Eq)]
 pub(crate) struct HolderHTLCOutput {
        preimage: Option<PaymentPreimage>,
-       amount: u64,
+       amount_msat: u64,
        /// Defaults to 0 for HTLC-Success transactions, which have no expiry
        cltv_expiry: u32,
+       opt_anchors: Option<()>,
 }
 
 impl HolderHTLCOutput {
-       pub(crate) fn build_offered(amount: u64, cltv_expiry: u32) -> Self {
+       pub(crate) fn build_offered(amount_msat: u64, cltv_expiry: u32, opt_anchors: bool) -> Self {
                HolderHTLCOutput {
                        preimage: None,
-                       amount,
+                       amount_msat,
                        cltv_expiry,
+                       opt_anchors: if opt_anchors { Some(()) } else { None } ,
                }
        }
 
-       pub(crate) fn build_accepted(preimage: PaymentPreimage, amount: u64) -> Self {
+       pub(crate) fn build_accepted(preimage: PaymentPreimage, amount_msat: u64, opt_anchors: bool) -> Self {
                HolderHTLCOutput {
                        preimage: Some(preimage),
-                       amount,
+                       amount_msat,
                        cltv_expiry: 0,
+                       opt_anchors: if opt_anchors { Some(()) } else { None } ,
                }
        }
+
+       fn opt_anchors(&self) -> bool {
+               self.opt_anchors.is_some()
+       }
 }
 
 impl_writeable_tlv_based!(HolderHTLCOutput, {
-       (0, amount, required),
+       (0, amount_msat, required),
        (2, cltv_expiry, required),
-       (4, preimage, option)
+       (4, preimage, option),
+       (6, opt_anchors, option)
 });
 
 /// A struct to describe the channel output on the funding transaction.
@@ -309,7 +318,7 @@ impl HolderFundingOutput {
 
 impl_writeable_tlv_based!(HolderFundingOutput, {
        (0, funding_redeemscript, required),
-       (1, opt_anchors, option),
+       (2, opt_anchors, option),
        (3, funding_amount, option),
 });
 
@@ -334,10 +343,10 @@ impl PackageSolvingData {
                        PackageSolvingData::RevokedHTLCOutput(ref outp) => outp.amount,
                        PackageSolvingData::CounterpartyOfferedHTLCOutput(ref outp) => outp.htlc.amount_msat / 1000,
                        PackageSolvingData::CounterpartyReceivedHTLCOutput(ref outp) => outp.htlc.amount_msat / 1000,
-                       // Note: Currently, amounts of holder outputs spending witnesses aren't used
-                       // as we can't malleate spending package to increase their feerate. This
-                       // should change with the remaining anchor output patchset.
-                       PackageSolvingData::HolderHTLCOutput(..) => unreachable!(),
+                       PackageSolvingData::HolderHTLCOutput(ref outp) => {
+                               debug_assert!(outp.opt_anchors());
+                               outp.amount_msat / 1000
+                       },
                        PackageSolvingData::HolderFundingOutput(ref outp) => {
                                debug_assert!(outp.opt_anchors());
                                outp.funding_amount.unwrap()
@@ -346,18 +355,23 @@ impl PackageSolvingData {
                amt
        }
        fn weight(&self) -> usize {
-               let weight = match self {
-                       PackageSolvingData::RevokedOutput(ref outp) => { outp.weight as usize },
-                       PackageSolvingData::RevokedHTLCOutput(ref outp) => { outp.weight as usize },
-                       PackageSolvingData::CounterpartyOfferedHTLCOutput(ref outp) => { weight_offered_htlc(outp.opt_anchors()) as usize },
-                       PackageSolvingData::CounterpartyReceivedHTLCOutput(ref outp) => { weight_received_htlc(outp.opt_anchors()) as usize },
-                       // Note: Currently, weights of holder outputs spending witnesses aren't used
-                       // as we can't malleate spending package to increase their feerate. This
-                       // should change with the remaining anchor output patchset.
-                       PackageSolvingData::HolderHTLCOutput(..) => { unreachable!() },
-                       PackageSolvingData::HolderFundingOutput(..) => { unreachable!() },
-               };
-               weight
+               match self {
+                       PackageSolvingData::RevokedOutput(ref outp) => outp.weight as usize,
+                       PackageSolvingData::RevokedHTLCOutput(ref outp) => outp.weight as usize,
+                       PackageSolvingData::CounterpartyOfferedHTLCOutput(ref outp) => weight_offered_htlc(outp.opt_anchors()) as usize,
+                       PackageSolvingData::CounterpartyReceivedHTLCOutput(ref outp) => weight_received_htlc(outp.opt_anchors()) as usize,
+                       PackageSolvingData::HolderHTLCOutput(ref outp) => {
+                               debug_assert!(outp.opt_anchors());
+                               if outp.preimage.is_none() {
+                                       weight_offered_htlc(true) as usize
+                               } else {
+                                       weight_received_htlc(true) as usize
+                               }
+                       },
+                       // Since HolderFundingOutput maps to an untractable package that is already signed, its
+                       // weight can be determined from the transaction itself.
+                       PackageSolvingData::HolderFundingOutput(..) => unreachable!(),
+               }
        }
        fn is_compatible(&self, input: &PackageSolvingData) -> bool {
                match self {
@@ -381,57 +395,53 @@ impl PackageSolvingData {
        fn finalize_input<Signer: Sign>(&self, bumped_tx: &mut Transaction, i: usize, onchain_handler: &mut OnchainTxHandler<Signer>) -> bool {
                match self {
                        PackageSolvingData::RevokedOutput(ref outp) => {
-                               if let Ok(chan_keys) = TxCreationKeys::derive_new(&onchain_handler.secp_ctx, &outp.per_commitment_point, &outp.counterparty_delayed_payment_base_key, &outp.counterparty_htlc_base_key, &onchain_handler.signer.pubkeys().revocation_basepoint, &onchain_handler.signer.pubkeys().htlc_basepoint) {
-                                       let witness_script = chan_utils::get_revokeable_redeemscript(&chan_keys.revocation_key, outp.on_counterparty_tx_csv, &chan_keys.broadcaster_delayed_payment_key);
-                                       //TODO: should we panic on signer failure ?
-                                       if let Ok(sig) = onchain_handler.signer.sign_justice_revoked_output(&bumped_tx, i, outp.amount, &outp.per_commitment_key, &onchain_handler.secp_ctx) {
-                                               let mut ser_sig = sig.serialize_der().to_vec();
-                                               ser_sig.push(EcdsaSighashType::All as u8);
-                                               bumped_tx.input[i].witness.push(ser_sig);
-                                               bumped_tx.input[i].witness.push(vec!(1));
-                                               bumped_tx.input[i].witness.push(witness_script.clone().into_bytes());
-                                       } else { return false; }
-                               }
+                               let chan_keys = TxCreationKeys::derive_new(&onchain_handler.secp_ctx, &outp.per_commitment_point, &outp.counterparty_delayed_payment_base_key, &outp.counterparty_htlc_base_key, &onchain_handler.signer.pubkeys().revocation_basepoint, &onchain_handler.signer.pubkeys().htlc_basepoint);
+                               let witness_script = chan_utils::get_revokeable_redeemscript(&chan_keys.revocation_key, outp.on_counterparty_tx_csv, &chan_keys.broadcaster_delayed_payment_key);
+                               //TODO: should we panic on signer failure ?
+                               if let Ok(sig) = onchain_handler.signer.sign_justice_revoked_output(&bumped_tx, i, outp.amount, &outp.per_commitment_key, &onchain_handler.secp_ctx) {
+                                       let mut ser_sig = sig.serialize_der().to_vec();
+                                       ser_sig.push(EcdsaSighashType::All as u8);
+                                       bumped_tx.input[i].witness.push(ser_sig);
+                                       bumped_tx.input[i].witness.push(vec!(1));
+                                       bumped_tx.input[i].witness.push(witness_script.clone().into_bytes());
+                               } else { return false; }
                        },
                        PackageSolvingData::RevokedHTLCOutput(ref outp) => {
-                               if let Ok(chan_keys) = TxCreationKeys::derive_new(&onchain_handler.secp_ctx, &outp.per_commitment_point, &outp.counterparty_delayed_payment_base_key, &outp.counterparty_htlc_base_key, &onchain_handler.signer.pubkeys().revocation_basepoint, &onchain_handler.signer.pubkeys().htlc_basepoint) {
-                                       let witness_script = chan_utils::get_htlc_redeemscript_with_explicit_keys(&outp.htlc, onchain_handler.opt_anchors(), &chan_keys.broadcaster_htlc_key, &chan_keys.countersignatory_htlc_key, &chan_keys.revocation_key);
-                                       //TODO: should we panic on signer failure ?
-                                       if let Ok(sig) = onchain_handler.signer.sign_justice_revoked_htlc(&bumped_tx, i, outp.amount, &outp.per_commitment_key, &outp.htlc, &onchain_handler.secp_ctx) {
-                                               let mut ser_sig = sig.serialize_der().to_vec();
-                                               ser_sig.push(EcdsaSighashType::All as u8);
-                                               bumped_tx.input[i].witness.push(ser_sig);
-                                               bumped_tx.input[i].witness.push(chan_keys.revocation_key.clone().serialize().to_vec());
-                                               bumped_tx.input[i].witness.push(witness_script.clone().into_bytes());
-                                       } else { return false; }
-                               }
+                               let chan_keys = TxCreationKeys::derive_new(&onchain_handler.secp_ctx, &outp.per_commitment_point, &outp.counterparty_delayed_payment_base_key, &outp.counterparty_htlc_base_key, &onchain_handler.signer.pubkeys().revocation_basepoint, &onchain_handler.signer.pubkeys().htlc_basepoint);
+                               let witness_script = chan_utils::get_htlc_redeemscript_with_explicit_keys(&outp.htlc, onchain_handler.opt_anchors(), &chan_keys.broadcaster_htlc_key, &chan_keys.countersignatory_htlc_key, &chan_keys.revocation_key);
+                               //TODO: should we panic on signer failure ?
+                               if let Ok(sig) = onchain_handler.signer.sign_justice_revoked_htlc(&bumped_tx, i, outp.amount, &outp.per_commitment_key, &outp.htlc, &onchain_handler.secp_ctx) {
+                                       let mut ser_sig = sig.serialize_der().to_vec();
+                                       ser_sig.push(EcdsaSighashType::All as u8);
+                                       bumped_tx.input[i].witness.push(ser_sig);
+                                       bumped_tx.input[i].witness.push(chan_keys.revocation_key.clone().serialize().to_vec());
+                                       bumped_tx.input[i].witness.push(witness_script.clone().into_bytes());
+                               } else { return false; }
                        },
                        PackageSolvingData::CounterpartyOfferedHTLCOutput(ref outp) => {
-                               if let Ok(chan_keys) = TxCreationKeys::derive_new(&onchain_handler.secp_ctx, &outp.per_commitment_point, &outp.counterparty_delayed_payment_base_key, &outp.counterparty_htlc_base_key, &onchain_handler.signer.pubkeys().revocation_basepoint, &onchain_handler.signer.pubkeys().htlc_basepoint) {
-                                       let witness_script = chan_utils::get_htlc_redeemscript_with_explicit_keys(&outp.htlc, onchain_handler.opt_anchors(), &chan_keys.broadcaster_htlc_key, &chan_keys.countersignatory_htlc_key, &chan_keys.revocation_key);
-
-                                       if let Ok(sig) = onchain_handler.signer.sign_counterparty_htlc_transaction(&bumped_tx, i, &outp.htlc.amount_msat / 1000, &outp.per_commitment_point, &outp.htlc, &onchain_handler.secp_ctx) {
-                                               let mut ser_sig = sig.serialize_der().to_vec();
-                                               ser_sig.push(EcdsaSighashType::All as u8);
-                                               bumped_tx.input[i].witness.push(ser_sig);
-                                               bumped_tx.input[i].witness.push(outp.preimage.0.to_vec());
-                                               bumped_tx.input[i].witness.push(witness_script.clone().into_bytes());
-                                       }
+                               let chan_keys = TxCreationKeys::derive_new(&onchain_handler.secp_ctx, &outp.per_commitment_point, &outp.counterparty_delayed_payment_base_key, &outp.counterparty_htlc_base_key, &onchain_handler.signer.pubkeys().revocation_basepoint, &onchain_handler.signer.pubkeys().htlc_basepoint);
+                               let witness_script = chan_utils::get_htlc_redeemscript_with_explicit_keys(&outp.htlc, onchain_handler.opt_anchors(), &chan_keys.broadcaster_htlc_key, &chan_keys.countersignatory_htlc_key, &chan_keys.revocation_key);
+
+                               if let Ok(sig) = onchain_handler.signer.sign_counterparty_htlc_transaction(&bumped_tx, i, &outp.htlc.amount_msat / 1000, &outp.per_commitment_point, &outp.htlc, &onchain_handler.secp_ctx) {
+                                       let mut ser_sig = sig.serialize_der().to_vec();
+                                       ser_sig.push(EcdsaSighashType::All as u8);
+                                       bumped_tx.input[i].witness.push(ser_sig);
+                                       bumped_tx.input[i].witness.push(outp.preimage.0.to_vec());
+                                       bumped_tx.input[i].witness.push(witness_script.clone().into_bytes());
                                }
                        },
                        PackageSolvingData::CounterpartyReceivedHTLCOutput(ref outp) => {
-                               if let Ok(chan_keys) = TxCreationKeys::derive_new(&onchain_handler.secp_ctx, &outp.per_commitment_point, &outp.counterparty_delayed_payment_base_key, &outp.counterparty_htlc_base_key, &onchain_handler.signer.pubkeys().revocation_basepoint, &onchain_handler.signer.pubkeys().htlc_basepoint) {
-                                       let witness_script = chan_utils::get_htlc_redeemscript_with_explicit_keys(&outp.htlc, onchain_handler.opt_anchors(), &chan_keys.broadcaster_htlc_key, &chan_keys.countersignatory_htlc_key, &chan_keys.revocation_key);
-
-                                       bumped_tx.lock_time = PackedLockTime(outp.htlc.cltv_expiry); // Right now we don't aggregate time-locked transaction, if we do we should set lock_time before to avoid breaking hash computation
-                                       if let Ok(sig) = onchain_handler.signer.sign_counterparty_htlc_transaction(&bumped_tx, i, &outp.htlc.amount_msat / 1000, &outp.per_commitment_point, &outp.htlc, &onchain_handler.secp_ctx) {
-                                               let mut ser_sig = sig.serialize_der().to_vec();
-                                               ser_sig.push(EcdsaSighashType::All as u8);
-                                               bumped_tx.input[i].witness.push(ser_sig);
-                                               // Due to BIP146 (MINIMALIF) this must be a zero-length element to relay.
-                                               bumped_tx.input[i].witness.push(vec![]);
-                                               bumped_tx.input[i].witness.push(witness_script.clone().into_bytes());
-                                       }
+                               let chan_keys = TxCreationKeys::derive_new(&onchain_handler.secp_ctx, &outp.per_commitment_point, &outp.counterparty_delayed_payment_base_key, &outp.counterparty_htlc_base_key, &onchain_handler.signer.pubkeys().revocation_basepoint, &onchain_handler.signer.pubkeys().htlc_basepoint);
+                               let witness_script = chan_utils::get_htlc_redeemscript_with_explicit_keys(&outp.htlc, onchain_handler.opt_anchors(), &chan_keys.broadcaster_htlc_key, &chan_keys.countersignatory_htlc_key, &chan_keys.revocation_key);
+
+                               bumped_tx.lock_time = PackedLockTime(outp.htlc.cltv_expiry); // Right now we don't aggregate time-locked transaction, if we do we should set lock_time before to avoid breaking hash computation
+                               if let Ok(sig) = onchain_handler.signer.sign_counterparty_htlc_transaction(&bumped_tx, i, &outp.htlc.amount_msat / 1000, &outp.per_commitment_point, &outp.htlc, &onchain_handler.secp_ctx) {
+                                       let mut ser_sig = sig.serialize_der().to_vec();
+                                       ser_sig.push(EcdsaSighashType::All as u8);
+                                       bumped_tx.input[i].witness.push(ser_sig);
+                                       // Due to BIP146 (MINIMALIF) this must be a zero-length element to relay.
+                                       bumped_tx.input[i].witness.push(vec![]);
+                                       bumped_tx.input[i].witness.push(witness_script.clone().into_bytes());
                                }
                        },
                        _ => { panic!("API Error!"); }
@@ -440,8 +450,13 @@ impl PackageSolvingData {
        }
        fn get_finalized_tx<Signer: Sign>(&self, outpoint: &BitcoinOutPoint, onchain_handler: &mut OnchainTxHandler<Signer>) -> Option<Transaction> {
                match self {
-                       PackageSolvingData::HolderHTLCOutput(ref outp) => { return onchain_handler.get_fully_signed_htlc_tx(outpoint, &outp.preimage); }
-                       PackageSolvingData::HolderFundingOutput(ref outp) => { return Some(onchain_handler.get_fully_signed_holder_tx(&outp.funding_redeemscript)); }
+                       PackageSolvingData::HolderHTLCOutput(ref outp) => {
+                               debug_assert!(!outp.opt_anchors());
+                               return onchain_handler.get_fully_signed_htlc_tx(outpoint, &outp.preimage);
+                       }
+                       PackageSolvingData::HolderFundingOutput(ref outp) => {
+                               return Some(onchain_handler.get_fully_signed_holder_tx(&outp.funding_redeemscript));
+                       }
                        _ => { panic!("API Error!"); }
                }
        }
@@ -641,6 +656,25 @@ impl PackageTemplate {
                let output_weight = (8 + 1 + destination_script.len()) * WITNESS_SCALE_FACTOR;
                inputs_weight + witnesses_weight + transaction_weight + output_weight
        }
+       #[cfg(anchors)]
+       pub(crate) fn construct_malleable_package_with_external_funding<Signer: Sign>(
+               &self, onchain_handler: &mut OnchainTxHandler<Signer>,
+       ) -> Option<Vec<ExternalHTLCClaim>> {
+               debug_assert!(self.requires_external_funding());
+               let mut htlcs: Option<Vec<ExternalHTLCClaim>> = None;
+               for (previous_output, input) in &self.inputs {
+                       match input {
+                               PackageSolvingData::HolderHTLCOutput(ref outp) => {
+                                       debug_assert!(outp.opt_anchors());
+                                       onchain_handler.generate_external_htlc_claim(&previous_output, &outp.preimage).map(|htlc| {
+                                               htlcs.get_or_insert_with(|| Vec::with_capacity(self.inputs.len())).push(htlc);
+                                       });
+                               }
+                               _ => debug_assert!(false, "Expected HolderHTLCOutputs to not be aggregated with other input types"),
+                       }
+               }
+               htlcs
+       }
        pub(crate) fn finalize_malleable_package<L: Deref, Signer: Sign>(
                &self, onchain_handler: &mut OnchainTxHandler<Signer>, value: u64, destination_script: Script, logger: &L
        ) -> Option<Transaction> where L::Target: Logger {
@@ -745,6 +779,7 @@ impl PackageTemplate {
        pub(crate) fn requires_external_funding(&self) -> bool {
                self.inputs.iter().find(|input| match input.1 {
                        PackageSolvingData::HolderFundingOutput(ref outp) => outp.opt_anchors(),
+                       PackageSolvingData::HolderHTLCOutput(ref outp) => outp.opt_anchors(),
                        _ => false,
                }).is_some()
        }
@@ -755,7 +790,11 @@ impl PackageTemplate {
                        PackageSolvingData::RevokedHTLCOutput(..) => PackageMalleability::Malleable,
                        PackageSolvingData::CounterpartyOfferedHTLCOutput(..) => PackageMalleability::Malleable,
                        PackageSolvingData::CounterpartyReceivedHTLCOutput(..) => PackageMalleability::Malleable,
-                       PackageSolvingData::HolderHTLCOutput(..) => PackageMalleability::Untractable,
+                       PackageSolvingData::HolderHTLCOutput(ref outp) => if outp.opt_anchors() {
+                               PackageMalleability::Malleable
+                       } else {
+                               PackageMalleability::Untractable
+                       },
                        PackageSolvingData::HolderFundingOutput(..) => PackageMalleability::Untractable,
                };
                let mut inputs = Vec::with_capacity(1);
@@ -774,7 +813,7 @@ impl PackageTemplate {
 
 impl Writeable for PackageTemplate {
        fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
-               writer.write_all(&byte_utils::be64_to_array(self.inputs.len() as u64))?;
+               writer.write_all(&(self.inputs.len() as u64).to_be_bytes())?;
                for (ref outpoint, ref rev_outp) in self.inputs.iter() {
                        outpoint.write(writer)?;
                        rev_outp.write(writer)?;
@@ -804,7 +843,11 @@ impl Readable for PackageTemplate {
                                PackageSolvingData::RevokedHTLCOutput(..) => { (PackageMalleability::Malleable, true) },
                                PackageSolvingData::CounterpartyOfferedHTLCOutput(..) => { (PackageMalleability::Malleable, true) },
                                PackageSolvingData::CounterpartyReceivedHTLCOutput(..) => { (PackageMalleability::Malleable, false) },
-                               PackageSolvingData::HolderHTLCOutput(..) => { (PackageMalleability::Untractable, false) },
+                               PackageSolvingData::HolderHTLCOutput(ref outp) => if outp.opt_anchors() {
+                                       (PackageMalleability::Malleable, outp.preimage.is_some())
+                               } else {
+                                       (PackageMalleability::Untractable, false)
+                               },
                                PackageSolvingData::HolderFundingOutput(..) => { (PackageMalleability::Untractable, false) },
                        }
                } else { return Err(DecodeError::InvalidValue); };
@@ -964,7 +1007,7 @@ mod tests {
                () => {
                        {
                                let preimage = PaymentPreimage([2;32]);
-                               PackageSolvingData::HolderHTLCOutput(HolderHTLCOutput::build_accepted(preimage, 0))
+                               PackageSolvingData::HolderHTLCOutput(HolderHTLCOutput::build_accepted(preimage, 0, false))
                        }
                }
        }
index a8c9dcc826e540d91e4020c67789df23e30297a4..729425d70d5bd705bfe65e345567794c8381e64e 100644 (file)
@@ -14,6 +14,7 @@ use bitcoin::blockdata::script::{Script,Builder};
 use bitcoin::blockdata::opcodes;
 use bitcoin::blockdata::transaction::{TxIn,TxOut,OutPoint,Transaction, EcdsaSighashType};
 use bitcoin::util::sighash;
+use bitcoin::util::address::Payload;
 
 use bitcoin::hashes::{Hash, HashEngine};
 use bitcoin::hashes::sha256::Hash as Sha256;
@@ -23,13 +24,12 @@ use bitcoin::hash_types::{Txid, PubkeyHash};
 use crate::ln::{PaymentHash, PaymentPreimage};
 use crate::ln::msgs::DecodeError;
 use crate::util::ser::{Readable, Writeable, Writer};
-use crate::util::{byte_utils, transaction_utils};
+use crate::util::transaction_utils;
 
-use bitcoin::hash_types::WPubkeyHash;
 use bitcoin::secp256k1::{SecretKey, PublicKey, Scalar};
 use bitcoin::secp256k1::{Secp256k1, ecdsa::Signature, Message};
-use bitcoin::secp256k1::Error as SecpError;
 use bitcoin::{PackedLockTime, secp256k1, Sequence, Witness};
+use bitcoin::PublicKey as BitcoinPublicKey;
 
 use crate::io;
 use crate::prelude::*;
@@ -41,13 +41,20 @@ use core::ops::Deref;
 use crate::chain;
 use crate::util::crypto::sign;
 
-pub(crate) const MAX_HTLCS: u16 = 483;
-pub(crate) const OFFERED_HTLC_SCRIPT_WEIGHT: usize = 133;
-pub(crate) const OFFERED_HTLC_SCRIPT_WEIGHT_ANCHORS: usize = 136;
-// The weight of `accepted_htlc_script` can vary in function of its CLTV argument value. We define a
-// range that encompasses both its non-anchors and anchors variants.
+/// Maximum number of one-way in-flight HTLC (protocol-level value).
+pub const MAX_HTLCS: u16 = 483;
+/// The weight of a BIP141 witnessScript for a BOLT3's "offered HTLC output" on a commitment transaction, non-anchor variant.
+pub const OFFERED_HTLC_SCRIPT_WEIGHT: usize = 133;
+/// The weight of a BIP141 witnessScript for a BOLT3's "offered HTLC output" on a commitment transaction, anchor variant.
+pub const OFFERED_HTLC_SCRIPT_WEIGHT_ANCHORS: usize = 136;
+
+/// The weight of a BIP141 witnessScript for a BOLT3's "received HTLC output" can vary in function of its CLTV argument value.
+/// We define a range that encompasses both its non-anchors and anchors variants.
 pub(crate) const MIN_ACCEPTED_HTLC_SCRIPT_WEIGHT: usize = 136;
-pub(crate) const MAX_ACCEPTED_HTLC_SCRIPT_WEIGHT: usize = 143;
+/// The weight of a BIP141 witnessScript for a BOLT3's "received HTLC output" can vary in function of its CLTV argument value.
+/// We define a range that encompasses both its non-anchors and anchors variants.
+/// This is the maximum post-anchor value.
+pub const MAX_ACCEPTED_HTLC_SCRIPT_WEIGHT: usize = 143;
 
 /// Gets the weight for an HTLC-Success transaction.
 #[inline]
@@ -65,18 +72,24 @@ pub fn htlc_timeout_tx_weight(opt_anchors: bool) -> u64 {
        if opt_anchors { HTLC_TIMEOUT_ANCHOR_TX_WEIGHT } else { HTLC_TIMEOUT_TX_WEIGHT }
 }
 
+/// Describes the type of HTLC claim as determined by analyzing the witness.
 #[derive(PartialEq, Eq)]
-pub(crate) enum HTLCClaim {
+pub enum HTLCClaim {
+       /// Claims an offered output on a commitment transaction through the timeout path.
        OfferedTimeout,
+       /// Claims an offered output on a commitment transaction through the success path.
        OfferedPreimage,
+       /// Claims an accepted output on a commitment transaction through the timeout path.
        AcceptedTimeout,
+       /// Claims an accepted output on a commitment transaction through the success path.
        AcceptedPreimage,
+       /// Claims an offered/accepted output on a commitment transaction through the revocation path.
        Revocation,
 }
 
 impl HTLCClaim {
        /// Check if a given input witness attempts to claim a HTLC.
-       pub(crate) fn from_witness(witness: &Witness) -> Option<Self> {
+       pub fn from_witness(witness: &Witness) -> Option<Self> {
                debug_assert_eq!(OFFERED_HTLC_SCRIPT_WEIGHT_ANCHORS, MIN_ACCEPTED_HTLC_SCRIPT_WEIGHT);
                if witness.len() < 2 {
                        return None;
@@ -296,7 +309,7 @@ impl Writeable for CounterpartyCommitmentSecrets {
        fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
                for &(ref secret, ref idx) in self.old_secrets.iter() {
                        writer.write_all(secret)?;
-                       writer.write_all(&byte_utils::be64_to_array(*idx))?;
+                       writer.write_all(&idx.to_be_bytes())?;
                }
                write_tlv_fields!(writer, {});
                Ok(())
@@ -316,32 +329,29 @@ impl Readable for CounterpartyCommitmentSecrets {
 
 /// Derives a per-commitment-transaction private key (eg an htlc key or delayed_payment key)
 /// from the base secret and the per_commitment_point.
-///
-/// Note that this is infallible iff we trust that at least one of the two input keys are randomly
-/// generated (ie our own).
-pub fn derive_private_key<T: secp256k1::Signing>(secp_ctx: &Secp256k1<T>, per_commitment_point: &PublicKey, base_secret: &SecretKey) -> Result<SecretKey, SecpError> {
+pub fn derive_private_key<T: secp256k1::Signing>(secp_ctx: &Secp256k1<T>, per_commitment_point: &PublicKey, base_secret: &SecretKey) -> SecretKey {
        let mut sha = Sha256::engine();
        sha.input(&per_commitment_point.serialize());
        sha.input(&PublicKey::from_secret_key(&secp_ctx, &base_secret).serialize());
        let res = Sha256::from_engine(sha).into_inner();
 
        base_secret.clone().add_tweak(&Scalar::from_be_bytes(res).unwrap())
+               .expect("Addition only fails if the tweak is the inverse of the key. This is not possible when the tweak contains the hash of the key.")
 }
 
 /// Derives a per-commitment-transaction public key (eg an htlc key or a delayed_payment key)
 /// from the base point and the per_commitment_key. This is the public equivalent of
 /// derive_private_key - using only public keys to derive a public key instead of private keys.
-///
-/// Note that this is infallible iff we trust that at least one of the two input keys are randomly
-/// generated (ie our own).
-pub fn derive_public_key<T: secp256k1::Signing>(secp_ctx: &Secp256k1<T>, per_commitment_point: &PublicKey, base_point: &PublicKey) -> Result<PublicKey, SecpError> {
+pub fn derive_public_key<T: secp256k1::Signing>(secp_ctx: &Secp256k1<T>, per_commitment_point: &PublicKey, base_point: &PublicKey) -> PublicKey {
        let mut sha = Sha256::engine();
        sha.input(&per_commitment_point.serialize());
        sha.input(&base_point.serialize());
        let res = Sha256::from_engine(sha).into_inner();
 
-       let hashkey = PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&res)?);
+       let hashkey = PublicKey::from_secret_key(&secp_ctx,
+               &SecretKey::from_slice(&res).expect("Hashes should always be valid keys unless SHA-256 is broken"));
        base_point.combine(&hashkey)
+               .expect("Addition only fails if the tweak is the inverse of the key. This is not possible when the tweak contains the hash of the key.")
 }
 
 /// Derives a per-commitment-transaction revocation key from its constituent parts.
@@ -350,10 +360,9 @@ pub fn derive_public_key<T: secp256k1::Signing>(secp_ctx: &Secp256k1<T>, per_com
 /// commitment transaction, thus per_commitment_secret always come from cheater
 /// and revocation_base_secret always come from punisher, which is the broadcaster
 /// of the transaction spending with this key knowledge.
-///
-/// Note that this is infallible iff we trust that at least one of the two input keys are randomly
-/// generated (ie our own).
-pub fn derive_private_revocation_key<T: secp256k1::Signing>(secp_ctx: &Secp256k1<T>, per_commitment_secret: &SecretKey, countersignatory_revocation_base_secret: &SecretKey) -> Result<SecretKey, SecpError> {
+pub fn derive_private_revocation_key<T: secp256k1::Signing>(secp_ctx: &Secp256k1<T>,
+       per_commitment_secret: &SecretKey, countersignatory_revocation_base_secret: &SecretKey)
+-> SecretKey {
        let countersignatory_revocation_base_point = PublicKey::from_secret_key(&secp_ctx, &countersignatory_revocation_base_secret);
        let per_commitment_point = PublicKey::from_secret_key(&secp_ctx, &per_commitment_secret);
 
@@ -372,9 +381,12 @@ pub fn derive_private_revocation_key<T: secp256k1::Signing>(secp_ctx: &Secp256k1
                Sha256::from_engine(sha).into_inner()
        };
 
-       let countersignatory_contrib = countersignatory_revocation_base_secret.clone().mul_tweak(&Scalar::from_be_bytes(rev_append_commit_hash_key).unwrap())?;
-       let broadcaster_contrib = per_commitment_secret.clone().mul_tweak(&Scalar::from_be_bytes(commit_append_rev_hash_key).unwrap())?;
+       let countersignatory_contrib = countersignatory_revocation_base_secret.clone().mul_tweak(&Scalar::from_be_bytes(rev_append_commit_hash_key).unwrap())
+               .expect("Multiplying a secret key by a hash is expected to never fail per secp256k1 docs");
+       let broadcaster_contrib = per_commitment_secret.clone().mul_tweak(&Scalar::from_be_bytes(commit_append_rev_hash_key).unwrap())
+               .expect("Multiplying a secret key by a hash is expected to never fail per secp256k1 docs");
        countersignatory_contrib.add_tweak(&Scalar::from_be_bytes(broadcaster_contrib.secret_bytes()).unwrap())
+               .expect("Addition only fails if the tweak is the inverse of the key. This is not possible when the tweak commits to the key.")
 }
 
 /// Derives a per-commitment-transaction revocation public key from its constituent parts. This is
@@ -388,7 +400,9 @@ pub fn derive_private_revocation_key<T: secp256k1::Signing>(secp_ctx: &Secp256k1
 ///
 /// Note that this is infallible iff we trust that at least one of the two input keys are randomly
 /// generated (ie our own).
-pub fn derive_public_revocation_key<T: secp256k1::Verification>(secp_ctx: &Secp256k1<T>, per_commitment_point: &PublicKey, countersignatory_revocation_base_point: &PublicKey) -> Result<PublicKey, SecpError> {
+pub fn derive_public_revocation_key<T: secp256k1::Verification>(secp_ctx: &Secp256k1<T>,
+       per_commitment_point: &PublicKey, countersignatory_revocation_base_point: &PublicKey)
+-> PublicKey {
        let rev_append_commit_hash_key = {
                let mut sha = Sha256::engine();
                sha.input(&countersignatory_revocation_base_point.serialize());
@@ -404,9 +418,12 @@ pub fn derive_public_revocation_key<T: secp256k1::Verification>(secp_ctx: &Secp2
                Sha256::from_engine(sha).into_inner()
        };
 
-       let countersignatory_contrib = countersignatory_revocation_base_point.clone().mul_tweak(&secp_ctx, &Scalar::from_be_bytes(rev_append_commit_hash_key).unwrap())?;
-       let broadcaster_contrib = per_commitment_point.clone().mul_tweak(&secp_ctx, &Scalar::from_be_bytes(commit_append_rev_hash_key).unwrap())?;
+       let countersignatory_contrib = countersignatory_revocation_base_point.clone().mul_tweak(&secp_ctx, &Scalar::from_be_bytes(rev_append_commit_hash_key).unwrap())
+               .expect("Multiplying a valid public key by a hash is expected to never fail per secp256k1 docs");
+       let broadcaster_contrib = per_commitment_point.clone().mul_tweak(&secp_ctx, &Scalar::from_be_bytes(commit_append_rev_hash_key).unwrap())
+               .expect("Multiplying a valid public key by a hash is expected to never fail per secp256k1 docs");
        countersignatory_contrib.combine(&broadcaster_contrib)
+               .expect("Addition only fails if the tweak is the inverse of the key. This is not possible when the tweak commits to the key.")
 }
 
 /// The set of public keys which are used in the creation of one commitment transaction.
@@ -445,7 +462,7 @@ impl_writeable_tlv_based!(TxCreationKeys, {
 });
 
 /// One counterparty's public keys which do not change over the life of a channel.
-#[derive(Clone, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct ChannelPublicKeys {
        /// The public key which is used to sign all commitment transactions, as it appears in the
        /// on-chain channel lock-in 2-of-2 multisig output.
@@ -479,19 +496,19 @@ impl_writeable_tlv_based!(ChannelPublicKeys, {
 impl TxCreationKeys {
        /// Create per-state keys from channel base points and the per-commitment point.
        /// Key set is asymmetric and can't be used as part of counter-signatory set of transactions.
-       pub fn derive_new<T: secp256k1::Signing + secp256k1::Verification>(secp_ctx: &Secp256k1<T>, per_commitment_point: &PublicKey, broadcaster_delayed_payment_base: &PublicKey, broadcaster_htlc_base: &PublicKey, countersignatory_revocation_base: &PublicKey, countersignatory_htlc_base: &PublicKey) -> Result<TxCreationKeys, SecpError> {
-               Ok(TxCreationKeys {
+       pub fn derive_new<T: secp256k1::Signing + secp256k1::Verification>(secp_ctx: &Secp256k1<T>, per_commitment_point: &PublicKey, broadcaster_delayed_payment_base: &PublicKey, broadcaster_htlc_base: &PublicKey, countersignatory_revocation_base: &PublicKey, countersignatory_htlc_base: &PublicKey) -> TxCreationKeys {
+               TxCreationKeys {
                        per_commitment_point: per_commitment_point.clone(),
-                       revocation_key: derive_public_revocation_key(&secp_ctx, &per_commitment_point, &countersignatory_revocation_base)?,
-                       broadcaster_htlc_key: derive_public_key(&secp_ctx, &per_commitment_point, &broadcaster_htlc_base)?,
-                       countersignatory_htlc_key: derive_public_key(&secp_ctx, &per_commitment_point, &countersignatory_htlc_base)?,
-                       broadcaster_delayed_payment_key: derive_public_key(&secp_ctx, &per_commitment_point, &broadcaster_delayed_payment_base)?,
-               })
+                       revocation_key: derive_public_revocation_key(&secp_ctx, &per_commitment_point, &countersignatory_revocation_base),
+                       broadcaster_htlc_key: derive_public_key(&secp_ctx, &per_commitment_point, &broadcaster_htlc_base),
+                       countersignatory_htlc_key: derive_public_key(&secp_ctx, &per_commitment_point, &countersignatory_htlc_base),
+                       broadcaster_delayed_payment_key: derive_public_key(&secp_ctx, &per_commitment_point, &broadcaster_delayed_payment_base),
+               }
        }
 
        /// Generate per-state keys from channel static keys.
        /// Key set is asymmetric and can't be used as part of counter-signatory set of transactions.
-       pub fn from_channel_static_keys<T: secp256k1::Signing + secp256k1::Verification>(per_commitment_point: &PublicKey, broadcaster_keys: &ChannelPublicKeys, countersignatory_keys: &ChannelPublicKeys, secp_ctx: &Secp256k1<T>) -> Result<TxCreationKeys, SecpError> {
+       pub fn from_channel_static_keys<T: secp256k1::Signing + secp256k1::Verification>(per_commitment_point: &PublicKey, broadcaster_keys: &ChannelPublicKeys, countersignatory_keys: &ChannelPublicKeys, secp_ctx: &Secp256k1<T>) -> TxCreationKeys {
                TxCreationKeys::derive_new(
                        &secp_ctx,
                        &per_commitment_point,
@@ -660,9 +677,26 @@ pub fn make_funding_redeemscript(broadcaster: &PublicKey, countersignatory: &Pub
 ///
 /// Panics if htlc.transaction_output_index.is_none() (as such HTLCs do not appear in the
 /// commitment transaction).
-pub fn build_htlc_transaction(commitment_txid: &Txid, feerate_per_kw: u32, contest_delay: u16, htlc: &HTLCOutputInCommitment, opt_anchors: bool, broadcaster_delayed_payment_key: &PublicKey, revocation_key: &PublicKey) -> Transaction {
+pub fn build_htlc_transaction(commitment_txid: &Txid, feerate_per_kw: u32, contest_delay: u16, htlc: &HTLCOutputInCommitment, opt_anchors: bool, use_non_zero_fee_anchors: bool, broadcaster_delayed_payment_key: &PublicKey, revocation_key: &PublicKey) -> Transaction {
        let mut txins: Vec<TxIn> = Vec::new();
-       txins.push(TxIn {
+       txins.push(build_htlc_input(commitment_txid, htlc, opt_anchors));
+
+       let mut txouts: Vec<TxOut> = Vec::new();
+       txouts.push(build_htlc_output(
+               feerate_per_kw, contest_delay, htlc, opt_anchors, use_non_zero_fee_anchors,
+               broadcaster_delayed_payment_key, revocation_key
+       ));
+
+       Transaction {
+               version: 2,
+               lock_time: PackedLockTime(if htlc.offered { htlc.cltv_expiry } else { 0 }),
+               input: txins,
+               output: txouts,
+       }
+}
+
+pub(crate) fn build_htlc_input(commitment_txid: &Txid, htlc: &HTLCOutputInCommitment, opt_anchors: bool) -> TxIn {
+       TxIn {
                previous_output: OutPoint {
                        txid: commitment_txid.clone(),
                        vout: htlc.transaction_output_index.expect("Can't build an HTLC transaction for a dust output"),
@@ -670,37 +704,65 @@ pub fn build_htlc_transaction(commitment_txid: &Txid, feerate_per_kw: u32, conte
                script_sig: Script::new(),
                sequence: Sequence(if opt_anchors { 1 } else { 0 }),
                witness: Witness::new(),
-       });
+       }
+}
 
+pub(crate) fn build_htlc_output(
+       feerate_per_kw: u32, contest_delay: u16, htlc: &HTLCOutputInCommitment, opt_anchors: bool,
+       use_non_zero_fee_anchors: bool, broadcaster_delayed_payment_key: &PublicKey, revocation_key: &PublicKey
+) -> TxOut {
        let weight = if htlc.offered {
                htlc_timeout_tx_weight(opt_anchors)
        } else {
                htlc_success_tx_weight(opt_anchors)
        };
-       let output_value = if opt_anchors {
+       let output_value = if opt_anchors && !use_non_zero_fee_anchors {
                htlc.amount_msat / 1000
        } else {
                let total_fee = feerate_per_kw as u64 * weight / 1000;
                htlc.amount_msat / 1000 - total_fee
        };
 
-       let mut txouts: Vec<TxOut> = Vec::new();
-       txouts.push(TxOut {
+       TxOut {
                script_pubkey: get_revokeable_redeemscript(revocation_key, contest_delay, broadcaster_delayed_payment_key).to_v0_p2wsh(),
                value: output_value,
-       });
+       }
+}
 
-       Transaction {
-               version: 2,
-               lock_time: PackedLockTime(if htlc.offered { htlc.cltv_expiry } else { 0 }),
-               input: txins,
-               output: txouts,
+/// Returns the witness required to satisfy and spend a HTLC input.
+pub fn build_htlc_input_witness(
+       local_sig: &Signature, remote_sig: &Signature, preimage: &Option<PaymentPreimage>,
+       redeem_script: &Script, opt_anchors: bool,
+) -> Witness {
+       let remote_sighash_type = if opt_anchors {
+               EcdsaSighashType::SinglePlusAnyoneCanPay
+       } else {
+               EcdsaSighashType::All
+       };
+       let mut remote_sig = remote_sig.serialize_der().to_vec();
+       remote_sig.push(remote_sighash_type as u8);
+
+       let mut local_sig = local_sig.serialize_der().to_vec();
+       local_sig.push(EcdsaSighashType::All as u8);
+
+       let mut witness = Witness::new();
+       // First push the multisig dummy, note that due to BIP147 (NULLDUMMY) it must be a zero-length element.
+       witness.push(vec![]);
+       witness.push(remote_sig);
+       witness.push(local_sig);
+       if let Some(preimage) = preimage {
+               witness.push(preimage.0.to_vec());
+       } else {
+               // Due to BIP146 (MINIMALIF) this must be a zero-length element to relay.
+               witness.push(vec![]);
        }
+       witness.push(redeem_script.to_bytes());
+       witness
 }
 
 /// Gets the witnessScript for the to_remote output when anchors are enabled.
 #[inline]
-pub(crate) fn get_to_countersignatory_with_anchors_redeemscript(payment_point: &PublicKey) -> Script {
+pub fn get_to_countersignatory_with_anchors_redeemscript(payment_point: &PublicKey) -> Script {
        Builder::new()
                .push_slice(&payment_point.serialize()[..])
                .push_opcode(opcodes::all::OP_CHECKSIGVERIFY)
@@ -749,7 +811,7 @@ pub fn build_anchor_input_witness(funding_key: &PublicKey, funding_sig: &Signatu
 ///
 /// Normally, this is converted to the broadcaster/countersignatory-organized DirectedChannelTransactionParameters
 /// before use, via the as_holder_broadcastable and as_counterparty_broadcastable functions.
-#[derive(Clone)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct ChannelTransactionParameters {
        /// Holder public keys
        pub holder_pubkeys: ChannelPublicKeys,
@@ -765,11 +827,15 @@ pub struct ChannelTransactionParameters {
        pub funding_outpoint: Option<chain::transaction::OutPoint>,
        /// Are anchors (zero fee HTLC transaction variant) used for this channel. Boolean is
        /// serialization backwards-compatible.
-       pub opt_anchors: Option<()>
+       pub opt_anchors: Option<()>,
+       /// Are non-zero-fee anchors are enabled (used in conjuction with opt_anchors)
+       /// It is intended merely for backwards compatibility with signers that need it.
+       /// There is no support for this feature in LDK channel negotiation.
+       pub opt_non_zero_fee_anchors: Option<()>,
 }
 
 /// Late-bound per-channel counterparty data used to build transactions.
-#[derive(Clone)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct CounterpartyChannelTransactionParameters {
        /// Counter-party public keys
        pub pubkeys: ChannelPublicKeys,
@@ -820,6 +886,7 @@ impl_writeable_tlv_based!(ChannelTransactionParameters, {
        (6, counterparty_parameters, option),
        (8, funding_outpoint, option),
        (10, opt_anchors, option),
+       (12, opt_non_zero_fee_anchors, option),
 });
 
 /// Static channel fields used to build transactions given per-commitment fields, organized by
@@ -942,7 +1009,8 @@ impl HolderCommitmentTransaction {
                        is_outbound_from_holder: false,
                        counterparty_parameters: Some(CounterpartyChannelTransactionParameters { pubkeys: channel_pubkeys.clone(), selected_contest_delay: 0 }),
                        funding_outpoint: Some(chain::transaction::OutPoint { txid: Txid::all_zeros(), index: 0 }),
-                       opt_anchors: None
+                       opt_anchors: None,
+                       opt_non_zero_fee_anchors: None,
                };
                let mut htlcs_with_aux: Vec<(_, ())> = Vec::new();
                let inner = CommitmentTransaction::new_with_auxiliary_htlc_data(0, 0, 0, false, dummy_key.clone(), dummy_key.clone(), keys, 0, &mut htlcs_with_aux, &channel_parameters.as_counterparty_broadcastable());
@@ -1160,6 +1228,8 @@ pub struct CommitmentTransaction {
        htlcs: Vec<HTLCOutputInCommitment>,
        // A boolean that is serialization backwards-compatible
        opt_anchors: Option<()>,
+       // Whether non-zero-fee anchors should be used
+       opt_non_zero_fee_anchors: Option<()>,
        // A cache of the parties' pubkeys required to construct the transaction, see doc for trust()
        keys: TxCreationKeys,
        // For access to the pre-built transaction, see doc for trust()
@@ -1193,6 +1263,7 @@ impl_writeable_tlv_based!(CommitmentTransaction, {
        (10, built, required),
        (12, htlcs, vec_type),
        (14, opt_anchors, option),
+       (16, opt_non_zero_fee_anchors, option),
 });
 
 impl CommitmentTransaction {
@@ -1225,9 +1296,18 @@ impl CommitmentTransaction {
                                transaction,
                                txid
                        },
+                       opt_non_zero_fee_anchors: None,
                }
        }
 
+       /// Use non-zero fee anchors
+       ///
+       /// (C-not exported) due to move, and also not likely to be useful for binding users
+       pub fn with_non_zero_fee_anchors(mut self) -> Self {
+               self.opt_non_zero_fee_anchors = Some(());
+               self
+       }
+
        fn internal_rebuild_transaction(&self, keys: &TxCreationKeys, channel_parameters: &DirectedChannelTransactionParameters, broadcaster_funding_key: &PublicKey, countersignatory_funding_key: &PublicKey) -> Result<BuiltCommitmentTransaction, ()> {
                let (obscured_commitment_transaction_number, txins) = Self::internal_build_inputs(self.commitment_number, channel_parameters);
 
@@ -1266,7 +1346,7 @@ impl CommitmentTransaction {
                        let script = if opt_anchors {
                            get_to_countersignatory_with_anchors_redeemscript(&countersignatory_pubkeys.payment_point).to_v0_p2wsh()
                        } else {
-                           get_p2wpkh_redeemscript(&countersignatory_pubkeys.payment_point)
+                           Payload::p2wpkh(&BitcoinPublicKey::new(countersignatory_pubkeys.payment_point)).unwrap().script_pubkey()
                        };
                        txouts.push((
                                TxOut {
@@ -1428,7 +1508,7 @@ impl CommitmentTransaction {
        pub fn verify<T: secp256k1::Signing + secp256k1::Verification>(&self, channel_parameters: &DirectedChannelTransactionParameters, broadcaster_keys: &ChannelPublicKeys, countersignatory_keys: &ChannelPublicKeys, secp_ctx: &Secp256k1<T>) -> Result<TrustedCommitmentTransaction, ()> {
                // This is the only field of the key cache that we trust
                let per_commitment_point = self.keys.per_commitment_point;
-               let keys = TxCreationKeys::from_channel_static_keys(&per_commitment_point, broadcaster_keys, countersignatory_keys, secp_ctx).unwrap();
+               let keys = TxCreationKeys::from_channel_static_keys(&per_commitment_point, broadcaster_keys, countersignatory_keys, secp_ctx);
                if keys != self.keys {
                        return Err(());
                }
@@ -1488,11 +1568,11 @@ impl<'a> TrustedCommitmentTransaction<'a> {
                let keys = &inner.keys;
                let txid = inner.built.txid;
                let mut ret = Vec::with_capacity(inner.htlcs.len());
-               let holder_htlc_key = derive_private_key(secp_ctx, &inner.keys.per_commitment_point, htlc_base_key).map_err(|_| ())?;
+               let holder_htlc_key = derive_private_key(secp_ctx, &inner.keys.per_commitment_point, htlc_base_key);
 
                for this_htlc in inner.htlcs.iter() {
                        assert!(this_htlc.transaction_output_index.is_some());
-                       let htlc_tx = build_htlc_transaction(&txid, inner.feerate_per_kw, channel_parameters.contest_delay(), &this_htlc, self.opt_anchors(), &keys.broadcaster_delayed_payment_key, &keys.revocation_key);
+                       let htlc_tx = build_htlc_transaction(&txid, inner.feerate_per_kw, channel_parameters.contest_delay(), &this_htlc, self.opt_anchors(), self.opt_non_zero_fee_anchors.is_some(), &keys.broadcaster_delayed_payment_key, &keys.revocation_key);
 
                        let htlc_redeemscript = get_htlc_redeemscript_with_explicit_keys(&this_htlc, self.opt_anchors(), &keys.broadcaster_htlc_key, &keys.countersignatory_htlc_key, &keys.revocation_key);
 
@@ -1514,30 +1594,13 @@ impl<'a> TrustedCommitmentTransaction<'a> {
                // Further, we should never be provided the preimage for an HTLC-Timeout transaction.
                if  this_htlc.offered && preimage.is_some() { unreachable!(); }
 
-               let mut htlc_tx = build_htlc_transaction(&txid, inner.feerate_per_kw, channel_parameters.contest_delay(), &this_htlc, self.opt_anchors(), &keys.broadcaster_delayed_payment_key, &keys.revocation_key);
+               let mut htlc_tx = build_htlc_transaction(&txid, inner.feerate_per_kw, channel_parameters.contest_delay(), &this_htlc, self.opt_anchors(), self.opt_non_zero_fee_anchors.is_some(), &keys.broadcaster_delayed_payment_key, &keys.revocation_key);
 
                let htlc_redeemscript = get_htlc_redeemscript_with_explicit_keys(&this_htlc, self.opt_anchors(), &keys.broadcaster_htlc_key, &keys.countersignatory_htlc_key, &keys.revocation_key);
 
-               let sighashtype = if self.opt_anchors() { EcdsaSighashType::SinglePlusAnyoneCanPay } else { EcdsaSighashType::All };
-
-               // First push the multisig dummy, note that due to BIP147 (NULLDUMMY) it must be a zero-length element.
-               htlc_tx.input[0].witness.push(Vec::new());
-
-               let mut cp_sig_ser = counterparty_signature.serialize_der().to_vec();
-               cp_sig_ser.push(sighashtype as u8);
-               htlc_tx.input[0].witness.push(cp_sig_ser);
-               let mut holder_sig_ser = signature.serialize_der().to_vec();
-               holder_sig_ser.push(EcdsaSighashType::All as u8);
-               htlc_tx.input[0].witness.push(holder_sig_ser);
-
-               if this_htlc.offered {
-                       // Due to BIP146 (MINIMALIF) this must be a zero-length element to relay.
-                       htlc_tx.input[0].witness.push(Vec::new());
-               } else {
-                       htlc_tx.input[0].witness.push(preimage.unwrap().0.to_vec());
-               }
-
-               htlc_tx.input[0].witness.push(htlc_redeemscript.as_bytes().to_vec());
+               htlc_tx.input[0].witness = chan_utils::build_htlc_input_witness(
+                       signature, counterparty_signature, preimage, &htlc_redeemscript, self.opt_anchors(),
+               );
                htlc_tx
        }
 }
@@ -1572,18 +1635,12 @@ pub fn get_commitment_transaction_number_obscure_factor(
                | ((res[31] as u64) << 0 * 8)
 }
 
-fn get_p2wpkh_redeemscript(key: &PublicKey) -> Script {
-       Builder::new().push_opcode(opcodes::all::OP_PUSHBYTES_0)
-               .push_slice(&WPubkeyHash::hash(&key.serialize())[..])
-               .into_script()
-}
-
 #[cfg(test)]
 mod tests {
        use super::CounterpartyCommitmentSecrets;
        use crate::{hex, chain};
        use crate::prelude::*;
-       use crate::ln::chan_utils::{get_htlc_redeemscript, get_to_countersignatory_with_anchors_redeemscript, get_p2wpkh_redeemscript, CommitmentTransaction, TxCreationKeys, ChannelTransactionParameters, CounterpartyChannelTransactionParameters, HTLCOutputInCommitment};
+       use crate::ln::chan_utils::{get_htlc_redeemscript, get_to_countersignatory_with_anchors_redeemscript, CommitmentTransaction, TxCreationKeys, ChannelTransactionParameters, CounterpartyChannelTransactionParameters, HTLCOutputInCommitment};
        use bitcoin::secp256k1::{PublicKey, SecretKey, Secp256k1};
        use crate::util::test_utils;
        use crate::chain::keysinterface::{KeysInterface, BaseSign};
@@ -1591,6 +1648,8 @@ mod tests {
        use bitcoin::hashes::Hash;
        use crate::ln::PaymentHash;
        use bitcoin::hashes::hex::ToHex;
+       use bitcoin::util::address::Payload;
+       use bitcoin::PublicKey as BitcoinPublicKey;
 
        #[test]
        fn test_anchors() {
@@ -1599,22 +1658,23 @@ mod tests {
                let seed = [42; 32];
                let network = Network::Testnet;
                let keys_provider = test_utils::TestKeysInterface::new(&seed, network);
-               let signer = keys_provider.get_channel_signer(false, 3000);
-               let counterparty_signer = keys_provider.get_channel_signer(false, 3000);
+               let signer = keys_provider.derive_channel_signer(3000, keys_provider.generate_channel_keys_id(false, 1_000_000, 0));
+               let counterparty_signer = keys_provider.derive_channel_signer(3000, keys_provider.generate_channel_keys_id(true, 1_000_000, 1));
                let delayed_payment_base = &signer.pubkeys().delayed_payment_basepoint;
                let per_commitment_secret = SecretKey::from_slice(&hex::decode("1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100").unwrap()[..]).unwrap();
                let per_commitment_point = PublicKey::from_secret_key(&secp_ctx, &per_commitment_secret);
                let htlc_basepoint = &signer.pubkeys().htlc_basepoint;
                let holder_pubkeys = signer.pubkeys();
                let counterparty_pubkeys = counterparty_signer.pubkeys();
-               let keys = TxCreationKeys::derive_new(&secp_ctx, &per_commitment_point, delayed_payment_base, htlc_basepoint, &counterparty_pubkeys.revocation_basepoint, &counterparty_pubkeys.htlc_basepoint).unwrap();
+               let keys = TxCreationKeys::derive_new(&secp_ctx, &per_commitment_point, delayed_payment_base, htlc_basepoint, &counterparty_pubkeys.revocation_basepoint, &counterparty_pubkeys.htlc_basepoint);
                let mut channel_parameters = ChannelTransactionParameters {
                        holder_pubkeys: holder_pubkeys.clone(),
                        holder_selected_contest_delay: 0,
                        is_outbound_from_holder: false,
                        counterparty_parameters: Some(CounterpartyChannelTransactionParameters { pubkeys: counterparty_pubkeys.clone(), selected_contest_delay: 0 }),
                        funding_outpoint: Some(chain::transaction::OutPoint { txid: Txid::all_zeros(), index: 0 }),
-                       opt_anchors: None
+                       opt_anchors: None,
+                       opt_non_zero_fee_anchors: None,
                };
 
                let mut htlcs_with_aux: Vec<(_, ())> = Vec::new();
@@ -1629,7 +1689,7 @@ mod tests {
                        &mut htlcs_with_aux, &channel_parameters.as_holder_broadcastable()
                );
                assert_eq!(tx.built.transaction.output.len(), 2);
-               assert_eq!(tx.built.transaction.output[1].script_pubkey, get_p2wpkh_redeemscript(&counterparty_pubkeys.payment_point));
+               assert_eq!(tx.built.transaction.output[1].script_pubkey, Payload::p2wpkh(&BitcoinPublicKey::new(counterparty_pubkeys.payment_point)).unwrap().script_pubkey());
 
                // Generate broadcaster and counterparty outputs as well as two anchors
                let tx = CommitmentTransaction::new_with_auxiliary_htlc_data(
@@ -1695,9 +1755,9 @@ mod tests {
                assert_eq!(tx.built.transaction.output[0].script_pubkey, get_htlc_redeemscript(&received_htlc, false, &keys).to_v0_p2wsh());
                assert_eq!(tx.built.transaction.output[1].script_pubkey, get_htlc_redeemscript(&offered_htlc, false, &keys).to_v0_p2wsh());
                assert_eq!(get_htlc_redeemscript(&received_htlc, false, &keys).to_v0_p2wsh().to_hex(),
-                                  "002085cf52e41ba7c099a39df504e7b61f6de122971ceb53b06731876eaeb85e8dc5");
+                                  "0020e43a7c068553003fe68fcae424fb7b28ec5ce48cd8b6744b3945631389bad2fb");
                assert_eq!(get_htlc_redeemscript(&offered_htlc, false, &keys).to_v0_p2wsh().to_hex(),
-                                  "002049f0736bb335c61a04d2623a24df878a7592a3c51fa7258d41b2c85318265e73");
+                                  "0020215d61bba56b19e9eadb6107f5a85d7f99c40f65992443f69229c290165bc00d");
 
                // Generate broadcaster output and received and offered HTLC outputs,  with anchors
                channel_parameters.opt_anchors = Some(());
@@ -1714,9 +1774,9 @@ mod tests {
                assert_eq!(tx.built.transaction.output[2].script_pubkey, get_htlc_redeemscript(&received_htlc, true, &keys).to_v0_p2wsh());
                assert_eq!(tx.built.transaction.output[3].script_pubkey, get_htlc_redeemscript(&offered_htlc, true, &keys).to_v0_p2wsh());
                assert_eq!(get_htlc_redeemscript(&received_htlc, true, &keys).to_v0_p2wsh().to_hex(),
-                                  "002067114123af3f95405bae4fd930fc95de03e3c86baaee8b2dd29b43dd26cf613c");
+                                  "0020b70d0649c72b38756885c7a30908d912a7898dd5d79457a7280b8e9a20f3f2bc");
                assert_eq!(get_htlc_redeemscript(&offered_htlc, true, &keys).to_v0_p2wsh().to_hex(),
-                                  "0020a06e3b0d4fcf704f2b9c41e16a70099e39989466c3142b8573a1154542f28f57");
+                                  "002087a3faeb1950a469c0e2db4a79b093a41b9526e5a6fc6ef5cb949bde3be379c7");
        }
 
        #[test]
index 9b0e303676254fc89daaabe3a90322901fa9a5f2..2721cc81e509b5c93f5d99095b8fb90b51b6c6cf 100644 (file)
@@ -112,7 +112,7 @@ fn test_monitor_and_persister_update_fail() {
                let mut w = test_utils::TestVecWriter(Vec::new());
                monitor.write(&mut w).unwrap();
                let new_monitor = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(
-                       &mut io::Cursor::new(&w.0), &test_utils::OnlyReadsKeysInterface {}).unwrap().1;
+                       &mut io::Cursor::new(&w.0), nodes[0].keys_manager).unwrap().1;
                assert!(new_monitor == *monitor);
                let chain_mon = test_utils::TestChainMonitor::new(Some(&chain_source), &tx_broadcaster, &logger, &chanmon_cfgs[0].fee_estimator, &persister, &node_cfgs[0].keys_manager);
                assert_eq!(chain_mon.watch_channel(outpoint, new_monitor), ChannelMonitorUpdateStatus::Completed);
@@ -200,9 +200,11 @@ fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
        let events_3 = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events_3.len(), 1);
        match events_3[0] {
-               Event::PaymentReceived { ref payment_hash, ref purpose, amount_msat } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
                        assert_eq!(payment_hash_1, *payment_hash);
                        assert_eq!(amount_msat, 1_000_000);
+                       assert_eq!(receiver_node_id.unwrap(), nodes[1].node.get_our_node_id());
+                       assert_eq!(via_channel_id, Some(channel_id));
                        match &purpose {
                                PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                        assert!(payment_preimage.is_none());
@@ -324,7 +326,6 @@ fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
                                nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
                                check_added_monitors!(nodes[0], 1);
                                assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
-                               nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
                        }
 
                        (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
@@ -568,9 +569,11 @@ fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
        let events_5 = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events_5.len(), 1);
        match events_5[0] {
-               Event::PaymentReceived { ref payment_hash, ref purpose, amount_msat } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
                        assert_eq!(payment_hash_2, *payment_hash);
                        assert_eq!(amount_msat, 1_000_000);
+                       assert_eq!(receiver_node_id.unwrap(), nodes[1].node.get_our_node_id());
+                       assert_eq!(via_channel_id, Some(channel_id));
                        match &purpose {
                                PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                        assert!(payment_preimage.is_none());
@@ -633,7 +636,6 @@ fn test_monitor_update_fail_cs() {
        chanmon_cfgs[1].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
        nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
        check_added_monitors!(nodes[1], 1);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 
@@ -664,7 +666,6 @@ fn test_monitor_update_fail_cs() {
                        chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
                        nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
                        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
-                       nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
                        check_added_monitors!(nodes[0], 1);
                        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
                },
@@ -685,9 +686,11 @@ fn test_monitor_update_fail_cs() {
        let events = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
        match events[0] {
-               Event::PaymentReceived { payment_hash, ref purpose, amount_msat } => {
+               Event::PaymentClaimable { payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
                        assert_eq!(payment_hash, our_payment_hash);
                        assert_eq!(amount_msat, 1_000_000);
+                       assert_eq!(receiver_node_id.unwrap(), nodes[1].node.get_our_node_id());
+                       assert_eq!(via_channel_id, Some(channel_id));
                        match &purpose {
                                PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                        assert!(payment_preimage.is_none());
@@ -726,7 +729,6 @@ fn test_monitor_update_fail_no_rebroadcast() {
        chanmon_cfgs[1].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
        nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
        assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
        check_added_monitors!(nodes[1], 1);
@@ -741,7 +743,7 @@ fn test_monitor_update_fail_no_rebroadcast() {
        let events = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
        match events[0] {
-               Event::PaymentReceived { payment_hash, .. } => {
+               Event::PaymentClaimable { payment_hash, .. } => {
                        assert_eq!(payment_hash, our_payment_hash);
                },
                _ => panic!("Unexpected event"),
@@ -784,12 +786,11 @@ fn test_monitor_update_raa_while_paused() {
        nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
        nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
        check_added_monitors!(nodes[0], 1);
+       assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 
        nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Existing pending monitor update prevented responses to RAA".to_string(), 1);
        check_added_monitors!(nodes[0], 1);
 
        chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
@@ -813,12 +814,12 @@ fn test_monitor_update_raa_while_paused() {
        nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
        check_added_monitors!(nodes[0], 1);
        expect_pending_htlcs_forwardable!(nodes[0]);
-       expect_payment_received!(nodes[0], our_payment_hash_2, our_payment_secret_2, 1000000);
+       expect_payment_claimable!(nodes[0], our_payment_hash_2, our_payment_secret_2, 1000000);
 
        nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
        check_added_monitors!(nodes[1], 1);
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], our_payment_hash_1, our_payment_secret_1, 1000000);
+       expect_payment_claimable!(nodes[1], our_payment_hash_1, our_payment_secret_1, 1000000);
 
        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
        claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2);
@@ -875,7 +876,6 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
        chanmon_cfgs[1].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
        nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
        assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
        check_added_monitors!(nodes[1], 1);
@@ -911,8 +911,6 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
                nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
                check_added_monitors!(nodes[1], 1);
                assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
-               nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
-               assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
                (Some(payment_preimage_4), Some(payment_hash_4))
        } else { (None, None) };
 
@@ -1051,11 +1049,11 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
        let events_6 = nodes[2].node.get_and_clear_pending_events();
        assert_eq!(events_6.len(), 2);
        match events_6[0] {
-               Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
+               Event::PaymentClaimable { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
                _ => panic!("Unexpected event"),
        };
        match events_6[1] {
-               Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_3); },
+               Event::PaymentClaimable { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_3); },
                _ => panic!("Unexpected event"),
        };
 
@@ -1074,7 +1072,7 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
                let events_9 = nodes[0].node.get_and_clear_pending_events();
                assert_eq!(events_9.len(), 1);
                match events_9[0] {
-                       Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
+                       Event::PaymentClaimable { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
                        _ => panic!("Unexpected event"),
                };
                claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap());
@@ -1136,7 +1134,7 @@ fn test_monitor_update_fail_reestablish() {
                get_event_msg!(nodes[0], MessageSendEvent::SendChannelUpdate, nodes[1].node.get_our_node_id())
                        .contents.flags & 2, 0); // The "disabled" bit should be unset as we just reconnected
 
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
+       nodes[1].node.get_and_clear_pending_msg_events(); // Free the holding cell
        check_added_monitors!(nodes[1], 1);
 
        nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
@@ -1228,12 +1226,11 @@ fn raa_no_response_awaiting_raa_state() {
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
        nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
        check_added_monitors!(nodes[1], 1);
+       assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 
        nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Existing pending monitor update prevented responses to RAA".to_string(), 1);
        check_added_monitors!(nodes[1], 1);
 
        chanmon_cfgs[1].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
@@ -1243,7 +1240,7 @@ fn raa_no_response_awaiting_raa_state() {
        check_added_monitors!(nodes[1], 0);
        let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash_1, payment_secret_1, 1000000);
+       expect_payment_claimable!(nodes[1], payment_hash_1, payment_secret_1, 1000000);
 
        // We send a third payment here, which is somewhat of a redundant test, but the
        // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
@@ -1272,7 +1269,7 @@ fn raa_no_response_awaiting_raa_state() {
        nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
        check_added_monitors!(nodes[1], 1);
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash_2, payment_secret_2, 1000000);
+       expect_payment_claimable!(nodes[1], payment_hash_2, payment_secret_2, 1000000);
        let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 
        nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
@@ -1285,7 +1282,7 @@ fn raa_no_response_awaiting_raa_state() {
        nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
        check_added_monitors!(nodes[1], 1);
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash_3, payment_secret_3, 1000000);
+       expect_payment_claimable!(nodes[1], payment_hash_3, payment_secret_3, 1000000);
 
        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2);
@@ -1331,7 +1328,6 @@ fn claim_while_disconnected_monitor_update_fail() {
 
        nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
        let _bs_channel_update = get_event_msg!(nodes[1], MessageSendEvent::SendChannelUpdate, nodes[0].node.get_our_node_id());
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
        check_added_monitors!(nodes[1], 1);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 
@@ -1348,7 +1344,6 @@ fn claim_while_disconnected_monitor_update_fail() {
        nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
        check_added_monitors!(nodes[1], 1);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
        // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
        // until we've channel_monitor_update'd and updated for the new commitment transaction.
 
@@ -1399,7 +1394,7 @@ fn claim_while_disconnected_monitor_update_fail() {
        check_added_monitors!(nodes[1], 1);
 
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash_2, payment_secret_2, 1000000);
+       expect_payment_claimable!(nodes[1], payment_hash_2, payment_secret_2, 1000000);
 
        nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
        check_added_monitors!(nodes[0], 1);
@@ -1440,7 +1435,6 @@ fn monitor_failed_no_reestablish_response() {
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
        nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
        check_added_monitors!(nodes[1], 1);
 
        // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
@@ -1475,7 +1469,7 @@ fn monitor_failed_no_reestablish_response() {
        check_added_monitors!(nodes[1], 1);
 
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash_1, payment_secret_1, 1000000);
+       expect_payment_claimable!(nodes[1], payment_hash_1, payment_secret_1, 1000000);
 
        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
 }
@@ -1540,7 +1534,6 @@ fn first_message_on_recv_ordering() {
        // to the next message also tests resetting the delivery order.
        nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
        check_added_monitors!(nodes[1], 1);
 
        // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
@@ -1550,7 +1543,6 @@ fn first_message_on_recv_ordering() {
        nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
        check_added_monitors!(nodes[1], 1);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 
        chanmon_cfgs[1].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
        let (outpoint, latest_update, _) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
@@ -1558,7 +1550,7 @@ fn first_message_on_recv_ordering() {
        check_added_monitors!(nodes[1], 0);
 
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash_1, payment_secret_1, 1000000);
+       expect_payment_claimable!(nodes[1], payment_hash_1, payment_secret_1, 1000000);
 
        let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
        nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
@@ -1571,7 +1563,7 @@ fn first_message_on_recv_ordering() {
        check_added_monitors!(nodes[1], 1);
 
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash_2, payment_secret_2, 1000000);
+       expect_payment_claimable!(nodes[1], payment_hash_2, payment_secret_2, 1000000);
 
        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2);
@@ -1599,7 +1591,7 @@ fn test_monitor_update_fail_claim() {
        chanmon_cfgs[1].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
        nodes[1].node.claim_funds(payment_preimage_1);
        expect_payment_claimed!(nodes[1], payment_hash_1, 1_000_000);
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Temporary failure claiming HTLC, treating as success: Failed to update ChannelMonitor".to_string(), 1);
+       assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
        check_added_monitors!(nodes[1], 1);
 
        // Note that at this point there is a pending commitment transaction update for A being held by
@@ -1638,7 +1630,8 @@ fn test_monitor_update_fail_claim() {
        commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
 
        // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
-       let (outpoint, latest_update, _) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
+       let channel_id = chan_1.2;
+       let (outpoint, latest_update, _) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
        nodes[1].chain_monitor.chain_monitor.force_channel_monitor_updated(outpoint, latest_update);
        check_added_monitors!(nodes[1], 0);
 
@@ -1659,9 +1652,12 @@ fn test_monitor_update_fail_claim() {
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 2);
        match events[0] {
-               Event::PaymentReceived { ref payment_hash, ref purpose, amount_msat } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id } => {
                        assert_eq!(payment_hash_2, *payment_hash);
                        assert_eq!(1_000_000, amount_msat);
+                       assert_eq!(receiver_node_id.unwrap(), nodes[0].node.get_our_node_id());
+                       assert_eq!(via_channel_id, Some(channel_id));
+                       assert_eq!(via_user_channel_id, Some(42));
                        match &purpose {
                                PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                        assert!(payment_preimage.is_none());
@@ -1673,9 +1669,11 @@ fn test_monitor_update_fail_claim() {
                _ => panic!("Unexpected event"),
        }
        match events[1] {
-               Event::PaymentReceived { ref payment_hash, ref purpose, amount_msat } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
                        assert_eq!(payment_hash_3, *payment_hash);
                        assert_eq!(1_000_000, amount_msat);
+                       assert_eq!(receiver_node_id.unwrap(), nodes[0].node.get_our_node_id());
+                       assert_eq!(via_channel_id, Some(channel_id));
                        match &purpose {
                                PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                        assert!(payment_preimage.is_none());
@@ -1730,7 +1728,6 @@ fn test_monitor_update_on_pending_forwards() {
        expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_2.2 }]);
        check_added_monitors!(nodes[1], 1);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 
        chanmon_cfgs[1].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
        let (outpoint, latest_update, _) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
@@ -1753,7 +1750,7 @@ fn test_monitor_update_on_pending_forwards() {
                _ => panic!("Unexpected event"),
        };
        nodes[0].node.process_pending_htlc_forwards();
-       expect_payment_received!(nodes[0], payment_hash_2, payment_secret_2, 1000000);
+       expect_payment_claimable!(nodes[0], payment_hash_2, payment_secret_2, 1000000);
 
        claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2);
 }
@@ -1791,9 +1788,7 @@ fn monitor_update_claim_fail_no_response() {
        expect_payment_claimed!(nodes[1], payment_hash_1, 1_000_000);
        check_added_monitors!(nodes[1], 1);
 
-       let events = nodes[1].node.get_and_clear_pending_msg_events();
-       assert_eq!(events.len(), 0);
-       nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Temporary failure claiming HTLC, treating as success: Failed to update ChannelMonitor".to_string(), 1);
+       assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 
        chanmon_cfgs[1].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
        let (outpoint, latest_update, _) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
@@ -1804,7 +1799,7 @@ fn monitor_update_claim_fail_no_response() {
        nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
        check_added_monitors!(nodes[1], 1);
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash_2, payment_secret_2, 1000000);
+       expect_payment_claimable!(nodes[1], payment_hash_2, payment_secret_2, 1000000);
 
        let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
        nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
@@ -1841,9 +1836,8 @@ fn do_during_funding_monitor_fail(confirm_a_first: bool, restore_b_before_conf:
 
        chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
        nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
-       assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
-       nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
        check_added_monitors!(nodes[0], 1);
+       assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
        assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
        chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
        let (outpoint, latest_update, _) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
@@ -2079,7 +2073,7 @@ fn test_pending_update_fee_ack_on_reconnect() {
        check_added_monitors!(nodes[1], 1);
 
        expect_pending_htlcs_forwardable!(nodes[0]);
-       expect_payment_received!(nodes[0], payment_hash, payment_secret, 1_000_000);
+       expect_payment_claimable!(nodes[0], payment_hash, payment_secret, 1_000_000);
 
        claim_payment(&nodes[1], &[&nodes[0]], payment_preimage);
 }
@@ -2374,7 +2368,7 @@ fn do_channel_holding_cell_serialize(disconnect: bool, reload_a: bool) {
        let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
        nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash_1, payment_secret_1, 100000);
+       expect_payment_claimable!(nodes[1], payment_hash_1, payment_secret_1, 100000);
        check_added_monitors!(nodes[1], 1);
 
        commitment_signed_dance!(nodes[1], nodes[0], (), false, true, false);
@@ -2391,7 +2385,7 @@ fn do_channel_holding_cell_serialize(disconnect: bool, reload_a: bool) {
        };
 
        nodes[1].node.process_pending_htlc_forwards();
-       expect_payment_received!(nodes[1], payment_hash_2, payment_secret_2, 100000);
+       expect_payment_claimable!(nodes[1], payment_hash_2, payment_secret_2, 100000);
 
        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2);
index 2980b0f7cd56cf1d45452a6beba9199cb6c66899..67e894a9d3526886ecfe5d2f1fd5c1b3375a5159 100644 (file)
@@ -27,14 +27,15 @@ use crate::ln::features::{ChannelTypeFeatures, InitFeatures};
 use crate::ln::msgs;
 use crate::ln::msgs::{DecodeError, OptionalField, DataLossProtect};
 use crate::ln::script::{self, ShutdownScript};
-use crate::ln::channelmanager::{self, CounterpartyForwardingInfo, PendingHTLCStatus, HTLCSource, HTLCFailReason, HTLCFailureMsg, PendingHTLCInfo, RAACommitmentOrder, BREAKDOWN_TIMEOUT, MIN_CLTV_EXPIRY_DELTA, MAX_LOCAL_BREAKDOWN_TIMEOUT};
+use crate::ln::channelmanager::{self, CounterpartyForwardingInfo, PendingHTLCStatus, HTLCSource, HTLCFailureMsg, PendingHTLCInfo, RAACommitmentOrder, BREAKDOWN_TIMEOUT, MIN_CLTV_EXPIRY_DELTA, MAX_LOCAL_BREAKDOWN_TIMEOUT};
 use crate::ln::chan_utils::{CounterpartyCommitmentSecrets, TxCreationKeys, HTLCOutputInCommitment, htlc_success_tx_weight, htlc_timeout_tx_weight, make_funding_redeemscript, ChannelPublicKeys, CommitmentTransaction, HolderCommitmentTransaction, ChannelTransactionParameters, CounterpartyChannelTransactionParameters, MAX_HTLCS, get_commitment_transaction_number_obscure_factor, ClosingTransaction};
 use crate::ln::chan_utils;
+use crate::ln::onion_utils::HTLCFailReason;
 use crate::chain::BestBlock;
 use crate::chain::chaininterface::{FeeEstimator, ConfirmationTarget, LowerBoundedFeeEstimator};
 use crate::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate, ChannelMonitorUpdateStep, LATENCY_GRACE_PERIOD_BLOCKS};
 use crate::chain::transaction::{OutPoint, TransactionData};
-use crate::chain::keysinterface::{Sign, KeysInterface};
+use crate::chain::keysinterface::{Sign, KeysInterface, BaseSign};
 use crate::util::events::ClosureReason;
 use crate::util::ser::{Readable, ReadableArgs, Writeable, Writer, VecWriter};
 use crate::util::logger::Logger;
@@ -439,8 +440,6 @@ pub(super) struct ReestablishResponses {
        pub raa: Option<msgs::RevokeAndACK>,
        pub commitment_update: Option<msgs::CommitmentUpdate>,
        pub order: RAACommitmentOrder,
-       pub mon_update: Option<ChannelMonitorUpdate>,
-       pub holding_cell_failed_htlcs: Vec<(HTLCSource, PaymentHash)>,
        pub announcement_sigs: Option<msgs::AnnouncementSignatures>,
        pub shutdown_msg: Option<msgs::Shutdown>,
 }
@@ -739,6 +738,10 @@ pub(super) struct Channel<Signer: Sign> {
 
        // We track whether we already emitted a `ChannelReady` event.
        channel_ready_event_emitted: bool,
+
+       /// The unique identifier used to re-derive the private key material for the channel through
+       /// [`KeysInterface::derive_channel_signer`].
+       channel_keys_id: [u8; 32],
 }
 
 #[cfg(any(test, fuzzing))]
@@ -911,7 +914,8 @@ impl<Signer: Sign> Channel<Signer> {
                let opt_anchors = false; // TODO - should be based on features
 
                let holder_selected_contest_delay = config.channel_handshake_config.our_to_self_delay;
-               let holder_signer = keys_provider.get_channel_signer(false, channel_value_satoshis);
+               let channel_keys_id = keys_provider.generate_channel_keys_id(false, channel_value_satoshis, user_id);
+               let holder_signer = keys_provider.derive_channel_signer(channel_value_satoshis, channel_keys_id);
                let pubkeys = holder_signer.pubkeys().clone();
 
                if !their_features.supports_wumbo() && channel_value_satoshis > MAX_FUNDING_SATOSHIS_NO_WUMBO {
@@ -1040,6 +1044,7 @@ impl<Signer: Sign> Channel<Signer> {
                                counterparty_parameters: None,
                                funding_outpoint: None,
                                opt_anchors: if opt_anchors { Some(()) } else { None },
+                               opt_non_zero_fee_anchors: None
                        },
                        funding_transaction: None,
 
@@ -1072,6 +1077,7 @@ impl<Signer: Sign> Channel<Signer> {
                        historical_inbound_htlc_fulfills: HashSet::new(),
 
                        channel_type: Self::get_initial_channel_type(&config),
+                       channel_keys_id,
                })
        }
 
@@ -1154,7 +1160,8 @@ impl<Signer: Sign> Channel<Signer> {
                        return Err(ChannelError::Close("Channel Type was not understood - we require static remote key".to_owned()));
                }
 
-               let holder_signer = keys_provider.get_channel_signer(true, msg.funding_satoshis);
+               let channel_keys_id = keys_provider.generate_channel_keys_id(true, msg.funding_satoshis, user_id);
+               let holder_signer = keys_provider.derive_channel_signer(msg.funding_satoshis, channel_keys_id);
                let pubkeys = holder_signer.pubkeys().clone();
                let counterparty_pubkeys = ChannelPublicKeys {
                        funding_pubkey: msg.funding_pubkey,
@@ -1385,6 +1392,7 @@ impl<Signer: Sign> Channel<Signer> {
                                }),
                                funding_outpoint: None,
                                opt_anchors: if opt_anchors { Some(()) } else { None },
+                               opt_non_zero_fee_anchors: None
                        },
                        funding_transaction: None,
 
@@ -1417,6 +1425,7 @@ impl<Signer: Sign> Channel<Signer> {
                        historical_inbound_htlc_fulfills: HashSet::new(),
 
                        channel_type,
+                       channel_keys_id,
                };
 
                Ok(chan)
@@ -1749,27 +1758,27 @@ impl<Signer: Sign> Channel<Signer> {
        /// our counterparty!)
        /// The result is a transaction which we can revoke broadcastership of (ie a "local" transaction)
        /// TODO Some magic rust shit to compile-time check this?
-       fn build_holder_transaction_keys(&self, commitment_number: u64) -> Result<TxCreationKeys, ChannelError> {
+       fn build_holder_transaction_keys(&self, commitment_number: u64) -> TxCreationKeys {
                let per_commitment_point = self.holder_signer.get_per_commitment_point(commitment_number, &self.secp_ctx);
                let delayed_payment_base = &self.get_holder_pubkeys().delayed_payment_basepoint;
                let htlc_basepoint = &self.get_holder_pubkeys().htlc_basepoint;
                let counterparty_pubkeys = self.get_counterparty_pubkeys();
 
-               Ok(secp_check!(TxCreationKeys::derive_new(&self.secp_ctx, &per_commitment_point, delayed_payment_base, htlc_basepoint, &counterparty_pubkeys.revocation_basepoint, &counterparty_pubkeys.htlc_basepoint), "Local tx keys generation got bogus keys".to_owned()))
+               TxCreationKeys::derive_new(&self.secp_ctx, &per_commitment_point, delayed_payment_base, htlc_basepoint, &counterparty_pubkeys.revocation_basepoint, &counterparty_pubkeys.htlc_basepoint)
        }
 
        #[inline]
        /// Creates a set of keys for build_commitment_transaction to generate a transaction which we
        /// will sign and send to our counterparty.
        /// If an Err is returned, it is a ChannelError::Close (for get_outbound_funding_created)
-       fn build_remote_transaction_keys(&self) -> Result<TxCreationKeys, ChannelError> {
+       fn build_remote_transaction_keys(&self) -> TxCreationKeys {
                //TODO: Ensure that the payment_key derived here ends up in the library users' wallet as we
                //may see payments to it!
                let revocation_basepoint = &self.get_holder_pubkeys().revocation_basepoint;
                let htlc_basepoint = &self.get_holder_pubkeys().htlc_basepoint;
                let counterparty_pubkeys = self.get_counterparty_pubkeys();
 
-               Ok(secp_check!(TxCreationKeys::derive_new(&self.secp_ctx, &self.counterparty_cur_commitment_point.unwrap(), &counterparty_pubkeys.delayed_payment_basepoint, &counterparty_pubkeys.htlc_basepoint, revocation_basepoint, htlc_basepoint), "Remote tx keys generation got bogus keys".to_owned()))
+               TxCreationKeys::derive_new(&self.secp_ctx, &self.counterparty_cur_commitment_point.unwrap(), &counterparty_pubkeys.delayed_payment_basepoint, &counterparty_pubkeys.htlc_basepoint, revocation_basepoint, htlc_basepoint)
        }
 
        /// Gets the redeemscript for the funding transaction output (ie the funding transaction output
@@ -1946,9 +1955,26 @@ impl<Signer: Sign> Channel<Signer> {
        /// an HTLC more than once or fulfill once and then attempt to fail after reconnect. We cannot,
        /// however, fail more than once as we wait for an upstream failure to be irrevocably committed
        /// before we fail backwards.
-       /// If we do fail twice, we debug_assert!(false) and return Ok(None). Thus, will always return
-       /// Ok(_) if debug assertions are turned on or preconditions are met.
-       pub fn get_update_fail_htlc<L: Deref>(&mut self, htlc_id_arg: u64, err_packet: msgs::OnionErrorPacket, logger: &L) -> Result<Option<msgs::UpdateFailHTLC>, ChannelError> where L::Target: Logger {
+       ///
+       /// If we do fail twice, we `debug_assert!(false)` and return `Ok(None)`. Thus, this will always
+       /// return `Ok(_)` if preconditions are met. In any case, `Err`s will only be
+       /// [`ChannelError::Ignore`].
+       pub fn queue_fail_htlc<L: Deref>(&mut self, htlc_id_arg: u64, err_packet: msgs::OnionErrorPacket, logger: &L)
+       -> Result<(), ChannelError> where L::Target: Logger {
+               self.fail_htlc(htlc_id_arg, err_packet, true, logger)
+                       .map(|msg_opt| assert!(msg_opt.is_none(), "We forced holding cell?"))
+       }
+
+       /// We can only have one resolution per HTLC. In some cases around reconnect, we may fulfill
+       /// an HTLC more than once or fulfill once and then attempt to fail after reconnect. We cannot,
+       /// however, fail more than once as we wait for an upstream failure to be irrevocably committed
+       /// before we fail backwards.
+       ///
+       /// If we do fail twice, we `debug_assert!(false)` and return `Ok(None)`. Thus, this will always
+       /// return `Ok(_)` if preconditions are met. In any case, `Err`s will only be
+       /// [`ChannelError::Ignore`].
+       fn fail_htlc<L: Deref>(&mut self, htlc_id_arg: u64, err_packet: msgs::OnionErrorPacket, mut force_holding_cell: bool, logger: &L)
+       -> Result<Option<msgs::UpdateFailHTLC>, ChannelError> where L::Target: Logger {
                if (self.channel_state & (ChannelState::ChannelReady as u32)) != (ChannelState::ChannelReady as u32) {
                        panic!("Was asked to fail an HTLC when channel was not in an operational state");
                }
@@ -1986,8 +2012,13 @@ impl<Signer: Sign> Channel<Signer> {
                        return Ok(None);
                }
 
-               // Now update local state:
                if (self.channel_state & (ChannelState::AwaitingRemoteRevoke as u32 | ChannelState::PeerDisconnected as u32 | ChannelState::MonitorUpdateInProgress as u32)) != 0 {
+                       debug_assert!(force_holding_cell, "!force_holding_cell is only called when emptying the holding cell, so we shouldn't end up back in it!");
+                       force_holding_cell = true;
+               }
+
+               // Now update local state:
+               if force_holding_cell {
                        for pending_update in self.holding_cell_htlc_updates.iter() {
                                match pending_update {
                                        &HTLCUpdateAwaitingACK::ClaimHTLC { htlc_id, .. } => {
@@ -2157,7 +2188,7 @@ impl<Signer: Sign> Channel<Signer> {
        fn funding_created_signature<L: Deref>(&mut self, sig: &Signature, logger: &L) -> Result<(Txid, CommitmentTransaction, Signature), ChannelError> where L::Target: Logger {
                let funding_script = self.get_funding_redeemscript();
 
-               let keys = self.build_holder_transaction_keys(self.cur_holder_commitment_transaction_number)?;
+               let keys = self.build_holder_transaction_keys(self.cur_holder_commitment_transaction_number);
                let initial_commitment_tx = self.build_commitment_transaction(self.cur_holder_commitment_transaction_number, &keys, true, false, logger).tx;
                {
                        let trusted_tx = initial_commitment_tx.trust();
@@ -2171,7 +2202,7 @@ impl<Signer: Sign> Channel<Signer> {
                        secp_check!(self.secp_ctx.verify_ecdsa(&sighash, &sig, self.counterparty_funding_pubkey()), "Invalid funding_created signature from peer".to_owned());
                }
 
-               let counterparty_keys = self.build_remote_transaction_keys()?;
+               let counterparty_keys = self.build_remote_transaction_keys();
                let counterparty_initial_commitment_tx = self.build_commitment_transaction(self.cur_counterparty_commitment_transaction_number, &counterparty_keys, false, false, logger).tx;
 
                let counterparty_trusted_tx = counterparty_initial_commitment_tx.trust();
@@ -2190,7 +2221,13 @@ impl<Signer: Sign> Channel<Signer> {
                &self.get_counterparty_pubkeys().funding_pubkey
        }
 
-       pub fn funding_created<L: Deref>(&mut self, msg: &msgs::FundingCreated, best_block: BestBlock, logger: &L) -> Result<(msgs::FundingSigned, ChannelMonitor<Signer>, Option<msgs::ChannelReady>), ChannelError> where L::Target: Logger {
+       pub fn funding_created<K: Deref, L: Deref>(
+               &mut self, msg: &msgs::FundingCreated, best_block: BestBlock, keys_source: &K, logger: &L
+       ) -> Result<(msgs::FundingSigned, ChannelMonitor<<K::Target as KeysInterface>::Signer>, Option<msgs::ChannelReady>), ChannelError>
+       where
+               K::Target: KeysInterface,
+               L::Target: Logger
+       {
                if self.is_outbound() {
                        return Err(ChannelError::Close("Received funding_created for an outbound channel?".to_owned()));
                }
@@ -2213,7 +2250,7 @@ impl<Signer: Sign> Channel<Signer> {
                self.channel_transaction_parameters.funding_outpoint = Some(funding_txo);
                // This is an externally observable change before we finish all our checks.  In particular
                // funding_created_signature may fail.
-               self.holder_signer.ready_channel(&self.channel_transaction_parameters);
+               self.holder_signer.provide_channel_parameters(&self.channel_transaction_parameters);
 
                let (counterparty_initial_commitment_txid, initial_commitment_tx, signature) = match self.funding_created_signature(&msg.signature, logger) {
                        Ok(res) => res,
@@ -2245,7 +2282,9 @@ impl<Signer: Sign> Channel<Signer> {
                let funding_txo_script = funding_redeemscript.to_v0_p2wsh();
                let obscure_factor = get_commitment_transaction_number_obscure_factor(&self.get_holder_pubkeys().payment_point, &self.get_counterparty_pubkeys().payment_point, self.is_outbound());
                let shutdown_script = self.shutdown_scriptpubkey.clone().map(|script| script.into_inner());
-               let channel_monitor = ChannelMonitor::new(self.secp_ctx.clone(), self.holder_signer.clone(),
+               let mut monitor_signer = keys_source.derive_channel_signer(self.channel_value_satoshis, self.channel_keys_id);
+               monitor_signer.provide_channel_parameters(&self.channel_transaction_parameters);
+               let channel_monitor = ChannelMonitor::new(self.secp_ctx.clone(), monitor_signer,
                                                          shutdown_script, self.get_holder_selected_contest_delay(),
                                                          &self.destination_script, (funding_txo, funding_txo_script.clone()),
                                                          &self.channel_transaction_parameters,
@@ -2270,7 +2309,13 @@ impl<Signer: Sign> Channel<Signer> {
 
        /// Handles a funding_signed message from the remote end.
        /// If this call is successful, broadcast the funding transaction (and not before!)
-       pub fn funding_signed<L: Deref>(&mut self, msg: &msgs::FundingSigned, best_block: BestBlock, logger: &L) -> Result<(ChannelMonitor<Signer>, Transaction, Option<msgs::ChannelReady>), ChannelError> where L::Target: Logger {
+       pub fn funding_signed<K: Deref, L: Deref>(
+               &mut self, msg: &msgs::FundingSigned, best_block: BestBlock, keys_source: &K, logger: &L
+       ) -> Result<(ChannelMonitor<<K::Target as KeysInterface>::Signer>, Transaction, Option<msgs::ChannelReady>), ChannelError>
+       where
+               K::Target: KeysInterface,
+               L::Target: Logger
+       {
                if !self.is_outbound() {
                        return Err(ChannelError::Close("Received funding_signed for an inbound channel?".to_owned()));
                }
@@ -2285,7 +2330,7 @@ impl<Signer: Sign> Channel<Signer> {
 
                let funding_script = self.get_funding_redeemscript();
 
-               let counterparty_keys = self.build_remote_transaction_keys()?;
+               let counterparty_keys = self.build_remote_transaction_keys();
                let counterparty_initial_commitment_tx = self.build_commitment_transaction(self.cur_counterparty_commitment_transaction_number, &counterparty_keys, false, false, logger).tx;
                let counterparty_trusted_tx = counterparty_initial_commitment_tx.trust();
                let counterparty_initial_bitcoin_tx = counterparty_trusted_tx.built_transaction();
@@ -2293,7 +2338,7 @@ impl<Signer: Sign> Channel<Signer> {
                log_trace!(logger, "Initial counterparty tx for channel {} is: txid {} tx {}",
                        log_bytes!(self.channel_id()), counterparty_initial_bitcoin_tx.txid, encode::serialize_hex(&counterparty_initial_bitcoin_tx.transaction));
 
-               let holder_signer = self.build_holder_transaction_keys(self.cur_holder_commitment_transaction_number)?;
+               let holder_signer = self.build_holder_transaction_keys(self.cur_holder_commitment_transaction_number);
                let initial_commitment_tx = self.build_commitment_transaction(self.cur_holder_commitment_transaction_number, &holder_signer, true, false, logger).tx;
                {
                        let trusted_tx = initial_commitment_tx.trust();
@@ -2322,7 +2367,9 @@ impl<Signer: Sign> Channel<Signer> {
                let funding_txo_script = funding_redeemscript.to_v0_p2wsh();
                let obscure_factor = get_commitment_transaction_number_obscure_factor(&self.get_holder_pubkeys().payment_point, &self.get_counterparty_pubkeys().payment_point, self.is_outbound());
                let shutdown_script = self.shutdown_scriptpubkey.clone().map(|script| script.into_inner());
-               let channel_monitor = ChannelMonitor::new(self.secp_ctx.clone(), self.holder_signer.clone(),
+               let mut monitor_signer = keys_source.derive_channel_signer(self.channel_value_satoshis, self.channel_keys_id);
+               monitor_signer.provide_channel_parameters(&self.channel_transaction_parameters);
+               let channel_monitor = ChannelMonitor::new(self.secp_ctx.clone(), monitor_signer,
                                                          shutdown_script, self.get_holder_selected_contest_delay(),
                                                          &self.destination_script, (funding_txo, funding_txo_script),
                                                          &self.channel_transaction_parameters,
@@ -2959,7 +3006,7 @@ impl<Signer: Sign> Channel<Signer> {
 
                let funding_script = self.get_funding_redeemscript();
 
-               let keys = self.build_holder_transaction_keys(self.cur_holder_commitment_transaction_number).map_err(|e| (None, e))?;
+               let keys = self.build_holder_transaction_keys(self.cur_holder_commitment_transaction_number);
 
                let commitment_stats = self.build_commitment_transaction(self.cur_holder_commitment_transaction_number, &keys, true, false, logger);
                let commitment_txid = {
@@ -3018,7 +3065,7 @@ impl<Signer: Sign> Channel<Signer> {
                        if let Some(_) = htlc.transaction_output_index {
                                let htlc_tx = chan_utils::build_htlc_transaction(&commitment_txid, commitment_stats.feerate_per_kw,
                                        self.get_counterparty_selected_contest_delay().unwrap(), &htlc, self.opt_anchors(),
-                                       &keys.broadcaster_delayed_payment_key, &keys.revocation_key);
+                                       false, &keys.broadcaster_delayed_payment_key, &keys.revocation_key);
 
                                let htlc_redeemscript = chan_utils::get_htlc_redeemscript(&htlc, self.opt_anchors(), &keys);
                                let htlc_sighashtype = if self.opt_anchors() { EcdsaSighashType::SinglePlusAnyoneCanPay } else { EcdsaSighashType::All };
@@ -3146,8 +3193,8 @@ impl<Signer: Sign> Channel<Signer> {
                } else { Ok((None, Vec::new())) }
        }
 
-       /// Used to fulfill holding_cell_htlcs when we get a remote ack (or implicitly get it by them
-       /// fulfilling or failing the last pending HTLC)
+       /// Frees any pending commitment updates in the holding cell, generating the relevant messages
+       /// for our counterparty.
        fn free_holding_cell_htlcs<L: Deref>(&mut self, logger: &L) -> Result<(Option<(msgs::CommitmentUpdate, ChannelMonitorUpdate)>, Vec<(HTLCSource, PaymentHash)>), ChannelError> where L::Target: Logger {
                assert_eq!(self.channel_state & ChannelState::MonitorUpdateInProgress as u32, 0);
                if self.holding_cell_htlc_updates.len() != 0 || self.holding_cell_update_fee.is_some() {
@@ -3173,7 +3220,7 @@ impl<Signer: Sign> Channel<Signer> {
                                // to rebalance channels.
                                match &htlc_update {
                                        &HTLCUpdateAwaitingACK::AddHTLC {amount_msat, cltv_expiry, ref payment_hash, ref source, ref onion_routing_packet, ..} => {
-                                               match self.send_htlc(amount_msat, *payment_hash, cltv_expiry, source.clone(), onion_routing_packet.clone(), logger) {
+                                               match self.send_htlc(amount_msat, *payment_hash, cltv_expiry, source.clone(), onion_routing_packet.clone(), false, logger) {
                                                        Ok(update_add_msg_option) => update_add_htlcs.push(update_add_msg_option.unwrap()),
                                                        Err(e) => {
                                                                match e {
@@ -3209,13 +3256,13 @@ impl<Signer: Sign> Channel<Signer> {
                                                monitor_update.updates.append(&mut additional_monitor_update.updates);
                                        },
                                        &HTLCUpdateAwaitingACK::FailHTLC { htlc_id, ref err_packet } => {
-                                               match self.get_update_fail_htlc(htlc_id, err_packet.clone(), logger) {
+                                               match self.fail_htlc(htlc_id, err_packet.clone(), false, logger) {
                                                        Ok(update_fail_msg_option) => {
                                                                // If an HTLC failure was previously added to the holding cell (via
-                                                               // `get_update_fail_htlc`) then generating the fail message itself
-                                                               // must not fail - we should never end up in a state where we
-                                                               // double-fail an HTLC or fail-then-claim an HTLC as it indicates
-                                                               // we didn't wait for a full revocation before failing.
+                                                               // `queue_fail_htlc`) then generating the fail message itself must
+                                                               // not fail - we should never end up in a state where we double-fail
+                                                               // an HTLC or fail-then-claim an HTLC as it indicates we didn't wait
+                                                               // for a full revocation before failing.
                                                                update_fail_htlcs.push(update_fail_msg_option.unwrap())
                                                        },
                                                        Err(e) => {
@@ -3232,7 +3279,7 @@ impl<Signer: Sign> Channel<Signer> {
                                return Ok((None, htlcs_to_fail));
                        }
                        let update_fee = if let Some(feerate) = self.holding_cell_update_fee.take() {
-                               self.send_update_fee(feerate, logger)
+                               self.send_update_fee(feerate, false, logger)
                        } else {
                                None
                        };
@@ -3532,12 +3579,22 @@ impl<Signer: Sign> Channel<Signer> {
                }
        }
 
+       /// Queues up an outbound update fee by placing it in the holding cell. You should call
+       /// [`Self::maybe_free_holding_cell_htlcs`] in order to actually generate and send the
+       /// commitment update.
+       pub fn queue_update_fee<L: Deref>(&mut self, feerate_per_kw: u32, logger: &L) where L::Target: Logger {
+               let msg_opt = self.send_update_fee(feerate_per_kw, true, logger);
+               assert!(msg_opt.is_none(), "We forced holding cell?");
+       }
+
        /// Adds a pending update to this channel. See the doc for send_htlc for
        /// further details on the optionness of the return value.
        /// If our balance is too low to cover the cost of the next commitment transaction at the
        /// new feerate, the update is cancelled.
-       /// You MUST call send_commitment prior to any other calls on this Channel
-       fn send_update_fee<L: Deref>(&mut self, feerate_per_kw: u32, logger: &L) -> Option<msgs::UpdateFee> where L::Target: Logger {
+       ///
+       /// You MUST call [`Self::send_commitment_no_state_update`] prior to any other calls on this
+       /// [`Channel`] if `force_holding_cell` is false.
+       fn send_update_fee<L: Deref>(&mut self, feerate_per_kw: u32, mut force_holding_cell: bool, logger: &L) -> Option<msgs::UpdateFee> where L::Target: Logger {
                if !self.is_outbound() {
                        panic!("Cannot send fee from inbound channel");
                }
@@ -3551,7 +3608,7 @@ impl<Signer: Sign> Channel<Signer> {
                // Before proposing a feerate update, check that we can actually afford the new fee.
                let inbound_stats = self.get_inbound_pending_htlc_stats(Some(feerate_per_kw));
                let outbound_stats = self.get_outbound_pending_htlc_stats(Some(feerate_per_kw));
-               let keys = if let Ok(keys) = self.build_holder_transaction_keys(self.cur_holder_commitment_transaction_number) { keys } else { return None; };
+               let keys = self.build_holder_transaction_keys(self.cur_holder_commitment_transaction_number);
                let commitment_stats = self.build_commitment_transaction(self.cur_holder_commitment_transaction_number, &keys, true, true, logger);
                let buffer_fee_msat = Channel::<Signer>::commit_tx_fee_sat(feerate_per_kw, commitment_stats.num_nondust_htlcs + outbound_stats.on_holder_tx_holding_cell_htlcs_count as usize + CONCURRENT_INBOUND_HTLC_FEE_BUFFER as usize, self.opt_anchors()) * 1000;
                let holder_balance_msat = commitment_stats.local_balance_msat - outbound_stats.holding_cell_msat;
@@ -3574,6 +3631,10 @@ impl<Signer: Sign> Channel<Signer> {
                }
 
                if (self.channel_state & (ChannelState::AwaitingRemoteRevoke as u32 | ChannelState::MonitorUpdateInProgress as u32)) != 0 {
+                       force_holding_cell = true;
+               }
+
+               if force_holding_cell {
                        self.holding_cell_update_fee = Some(feerate_per_kw);
                        return None;
                }
@@ -3587,16 +3648,6 @@ impl<Signer: Sign> Channel<Signer> {
                })
        }
 
-       pub fn send_update_fee_and_commit<L: Deref>(&mut self, feerate_per_kw: u32, logger: &L) -> Result<Option<(msgs::UpdateFee, msgs::CommitmentSigned, ChannelMonitorUpdate)>, ChannelError> where L::Target: Logger {
-               match self.send_update_fee(feerate_per_kw, logger) {
-                       Some(update_fee) => {
-                               let (commitment_signed, monitor_update) = self.send_commitment_no_status_check(logger)?;
-                               Ok(Some((update_fee, commitment_signed, monitor_update)))
-                       },
-                       None => Ok(None)
-               }
-       }
-
        /// Removes any uncommitted inbound HTLCs and resets the state of uncommitted outbound HTLC
        /// updates, to be used on peer disconnection. After this, update_*_htlc messages need to be
        /// resent.
@@ -3964,9 +4015,8 @@ impl<Signer: Sign> Channel<Signer> {
                                // Short circuit the whole handler as there is nothing we can resend them
                                return Ok(ReestablishResponses {
                                        channel_ready: None,
-                                       raa: None, commitment_update: None, mon_update: None,
+                                       raa: None, commitment_update: None,
                                        order: RAACommitmentOrder::CommitmentFirst,
-                                       holding_cell_failed_htlcs: Vec::new(),
                                        shutdown_msg, announcement_sigs,
                                });
                        }
@@ -3979,9 +4029,8 @@ impl<Signer: Sign> Channel<Signer> {
                                        next_per_commitment_point,
                                        short_channel_id_alias: Some(self.outbound_scid_alias),
                                }),
-                               raa: None, commitment_update: None, mon_update: None,
+                               raa: None, commitment_update: None,
                                order: RAACommitmentOrder::CommitmentFirst,
-                               holding_cell_failed_htlcs: Vec::new(),
                                shutdown_msg, announcement_sigs,
                        });
                }
@@ -4024,46 +4073,12 @@ impl<Signer: Sign> Channel<Signer> {
                                log_debug!(logger, "Reconnected channel {} with no loss", log_bytes!(self.channel_id()));
                        }
 
-                       if (self.channel_state & (ChannelState::AwaitingRemoteRevoke as u32 | ChannelState::MonitorUpdateInProgress as u32)) == 0 {
-                               // We're up-to-date and not waiting on a remote revoke (if we are our
-                               // channel_reestablish should result in them sending a revoke_and_ack), but we may
-                               // have received some updates while we were disconnected. Free the holding cell
-                               // now!
-                               match self.free_holding_cell_htlcs(logger) {
-                                       Err(ChannelError::Close(msg)) => Err(ChannelError::Close(msg)),
-                                       Err(ChannelError::Warn(_)) | Err(ChannelError::Ignore(_)) =>
-                                               panic!("Got non-channel-failing result from free_holding_cell_htlcs"),
-                                       Ok((Some((commitment_update, monitor_update)), holding_cell_failed_htlcs)) => {
-                                               Ok(ReestablishResponses {
-                                                       channel_ready, shutdown_msg, announcement_sigs,
-                                                       raa: required_revoke,
-                                                       commitment_update: Some(commitment_update),
-                                                       order: self.resend_order.clone(),
-                                                       mon_update: Some(monitor_update),
-                                                       holding_cell_failed_htlcs,
-                                               })
-                                       },
-                                       Ok((None, holding_cell_failed_htlcs)) => {
-                                               Ok(ReestablishResponses {
-                                                       channel_ready, shutdown_msg, announcement_sigs,
-                                                       raa: required_revoke,
-                                                       commitment_update: None,
-                                                       order: self.resend_order.clone(),
-                                                       mon_update: None,
-                                                       holding_cell_failed_htlcs,
-                                               })
-                                       },
-                               }
-                       } else {
-                               Ok(ReestablishResponses {
-                                       channel_ready, shutdown_msg, announcement_sigs,
-                                       raa: required_revoke,
-                                       commitment_update: None,
-                                       order: self.resend_order.clone(),
-                                       mon_update: None,
-                                       holding_cell_failed_htlcs: Vec::new(),
-                               })
-                       }
+                       Ok(ReestablishResponses {
+                               channel_ready, shutdown_msg, announcement_sigs,
+                               raa: required_revoke,
+                               commitment_update: None,
+                               order: self.resend_order.clone(),
+                       })
                } else if msg.next_local_commitment_number == next_counterparty_commitment_number - 1 {
                        if required_revoke.is_some() {
                                log_debug!(logger, "Reconnected channel {} with lost outbound RAA and lost remote commitment tx", log_bytes!(self.channel_id()));
@@ -4075,9 +4090,8 @@ impl<Signer: Sign> Channel<Signer> {
                                self.monitor_pending_commitment_signed = true;
                                Ok(ReestablishResponses {
                                        channel_ready, shutdown_msg, announcement_sigs,
-                                       commitment_update: None, raa: None, mon_update: None,
+                                       commitment_update: None, raa: None,
                                        order: self.resend_order.clone(),
-                                       holding_cell_failed_htlcs: Vec::new(),
                                })
                        } else {
                                Ok(ReestablishResponses {
@@ -4085,8 +4099,6 @@ impl<Signer: Sign> Channel<Signer> {
                                        raa: required_revoke,
                                        commitment_update: Some(self.get_last_commitment_update(logger)),
                                        order: self.resend_order.clone(),
-                                       mon_update: None,
-                                       holding_cell_failed_htlcs: Vec::new(),
                                })
                        }
                } else {
@@ -4534,6 +4546,16 @@ impl<Signer: Sign> Channel<Signer> {
                self.funding_tx_confirmed_in
        }
 
+       /// Returns the current number of confirmations on the funding transaction.
+       pub fn get_funding_tx_confirmations(&self, height: u32) -> u32 {
+               if self.funding_tx_confirmation_height == 0 {
+                       // We either haven't seen any confirmation yet, or observed a reorg.
+                       return 0;
+               }
+
+               height.checked_sub(self.funding_tx_confirmation_height).map_or(0, |c| c + 1)
+       }
+
        fn get_holder_selected_contest_delay(&self) -> u16 {
                self.channel_transaction_parameters.holder_selected_contest_delay
        }
@@ -5250,7 +5272,7 @@ impl<Signer: Sign> Channel<Signer> {
 
        /// If an Err is returned, it is a ChannelError::Close (for get_outbound_funding_created)
        fn get_outbound_funding_created_signature<L: Deref>(&mut self, logger: &L) -> Result<Signature, ChannelError> where L::Target: Logger {
-               let counterparty_keys = self.build_remote_transaction_keys()?;
+               let counterparty_keys = self.build_remote_transaction_keys();
                let counterparty_initial_commitment_tx = self.build_commitment_transaction(self.cur_counterparty_commitment_transaction_number, &counterparty_keys, false, false, logger).tx;
                Ok(self.holder_signer.sign_counterparty_commitment(&counterparty_initial_commitment_tx, Vec::new(), &self.secp_ctx)
                                .map_err(|_| ChannelError::Close("Failed to get signatures for new commitment_signed".to_owned()))?.0)
@@ -5277,7 +5299,7 @@ impl<Signer: Sign> Channel<Signer> {
                }
 
                self.channel_transaction_parameters.funding_outpoint = Some(funding_txo);
-               self.holder_signer.ready_channel(&self.channel_transaction_parameters);
+               self.holder_signer.provide_channel_parameters(&self.channel_transaction_parameters);
 
                let signature = match self.get_outbound_funding_created_signature(logger) {
                        Ok(res) => res,
@@ -5499,8 +5521,26 @@ impl<Signer: Sign> Channel<Signer> {
 
        // Send stuff to our remote peers:
 
+       /// Queues up an outbound HTLC to send by placing it in the holding cell. You should call
+       /// [`Self::maybe_free_holding_cell_htlcs`] in order to actually generate and send the
+       /// commitment update.
+       ///
+       /// `Err`s will only be [`ChannelError::Ignore`].
+       pub fn queue_add_htlc<L: Deref>(&mut self, amount_msat: u64, payment_hash: PaymentHash, cltv_expiry: u32, source: HTLCSource,
+               onion_routing_packet: msgs::OnionPacket, logger: &L)
+       -> Result<(), ChannelError> where L::Target: Logger {
+               self
+                       .send_htlc(amount_msat, payment_hash, cltv_expiry, source, onion_routing_packet, true, logger)
+                       .map(|msg_opt| assert!(msg_opt.is_none(), "We forced holding cell?"))
+                       .map_err(|err| {
+                               if let ChannelError::Ignore(_) = err { /* fine */ }
+                               else { debug_assert!(false, "Queueing cannot trigger channel failure"); }
+                               err
+                       })
+       }
+
        /// Adds a pending outbound HTLC to this channel, note that you probably want
-       /// send_htlc_and_commit instead cause you'll want both messages at once.
+       /// [`Self::send_htlc_and_commit`] instead cause you'll want both messages at once.
        ///
        /// This returns an optional UpdateAddHTLC as we may be in a state where we cannot add HTLCs on
        /// the wire:
@@ -5511,10 +5551,13 @@ impl<Signer: Sign> Channel<Signer> {
        ///   we may not yet have sent the previous commitment update messages and will need to
        ///   regenerate them.
        ///
-       /// You MUST call send_commitment prior to calling any other methods on this Channel!
+       /// You MUST call [`Self::send_commitment_no_state_update`] prior to calling any other methods
+       /// on this [`Channel`] if `force_holding_cell` is false.
        ///
-       /// If an Err is returned, it's a ChannelError::Ignore!
-       pub fn send_htlc<L: Deref>(&mut self, amount_msat: u64, payment_hash: PaymentHash, cltv_expiry: u32, source: HTLCSource, onion_routing_packet: msgs::OnionPacket, logger: &L) -> Result<Option<msgs::UpdateAddHTLC>, ChannelError> where L::Target: Logger {
+       /// `Err`s will only be [`ChannelError::Ignore`].
+       fn send_htlc<L: Deref>(&mut self, amount_msat: u64, payment_hash: PaymentHash, cltv_expiry: u32, source: HTLCSource,
+               onion_routing_packet: msgs::OnionPacket, mut force_holding_cell: bool, logger: &L)
+       -> Result<Option<msgs::UpdateAddHTLC>, ChannelError> where L::Target: Logger {
                if (self.channel_state & (ChannelState::ChannelReady as u32 | BOTH_SIDES_SHUTDOWN_MASK)) != (ChannelState::ChannelReady as u32) {
                        return Err(ChannelError::Ignore("Cannot send HTLC until channel is fully established and we haven't started shutting down".to_owned()));
                }
@@ -5551,7 +5594,7 @@ impl<Signer: Sign> Channel<Signer> {
                        return Err(ChannelError::Ignore(format!("Cannot send value that would put us over the max HTLC value in flight our peer will accept ({})", self.counterparty_max_htlc_value_in_flight_msat)));
                }
 
-               let keys = self.build_holder_transaction_keys(self.cur_holder_commitment_transaction_number)?;
+               let keys = self.build_holder_transaction_keys(self.cur_holder_commitment_transaction_number);
                let commitment_stats = self.build_commitment_transaction(self.cur_holder_commitment_transaction_number, &keys, true, true, logger);
                if !self.is_outbound() {
                        // Check that we won't violate the remote channel reserve by adding this HTLC.
@@ -5609,8 +5652,12 @@ impl<Signer: Sign> Channel<Signer> {
                        return Err(ChannelError::Ignore(format!("Cannot send value that would put our balance under counterparty-announced channel reserve value ({})", chan_reserve_msat)));
                }
 
-               // Now update local state:
                if (self.channel_state & (ChannelState::AwaitingRemoteRevoke as u32 | ChannelState::MonitorUpdateInProgress as u32)) != 0 {
+                       force_holding_cell = true;
+               }
+
+               // Now update local state:
+               if force_holding_cell {
                        self.holding_cell_htlc_updates.push(HTLCUpdateAwaitingACK::AddHTLC {
                                amount_msat,
                                payment_hash,
@@ -5643,41 +5690,6 @@ impl<Signer: Sign> Channel<Signer> {
                Ok(Some(res))
        }
 
-       /// Creates a signed commitment transaction to send to the remote peer.
-       /// Always returns a ChannelError::Close if an immediately-preceding (read: the
-       /// last call to this Channel) send_htlc returned Ok(Some(_)) and there is an Err.
-       /// May panic if called except immediately after a successful, Ok(Some(_))-returning send_htlc.
-       pub fn send_commitment<L: Deref>(&mut self, logger: &L) -> Result<(msgs::CommitmentSigned, ChannelMonitorUpdate), ChannelError> where L::Target: Logger {
-               if (self.channel_state & (ChannelState::ChannelReady as u32)) != (ChannelState::ChannelReady as u32) {
-                       panic!("Cannot create commitment tx until channel is fully established");
-               }
-               if (self.channel_state & (ChannelState::AwaitingRemoteRevoke as u32)) == (ChannelState::AwaitingRemoteRevoke as u32) {
-                       panic!("Cannot create commitment tx until remote revokes their previous commitment");
-               }
-               if (self.channel_state & (ChannelState::PeerDisconnected as u32)) == (ChannelState::PeerDisconnected as u32) {
-                       panic!("Cannot create commitment tx while disconnected, as send_htlc will have returned an Err so a send_commitment precondition has been violated");
-               }
-               if (self.channel_state & (ChannelState::MonitorUpdateInProgress as u32)) == (ChannelState::MonitorUpdateInProgress as u32) {
-                       panic!("Cannot create commitment tx while awaiting monitor update unfreeze, as send_htlc will have returned an Err so a send_commitment precondition has been violated");
-               }
-               let mut have_updates = self.is_outbound() && self.pending_update_fee.is_some();
-               for htlc in self.pending_outbound_htlcs.iter() {
-                       if let OutboundHTLCState::LocalAnnounced(_) = htlc.state {
-                               have_updates = true;
-                       }
-                       if have_updates { break; }
-               }
-               for htlc in self.pending_inbound_htlcs.iter() {
-                       if let InboundHTLCState::LocalRemoved(_) = htlc.state {
-                               have_updates = true;
-                       }
-                       if have_updates { break; }
-               }
-               if !have_updates {
-                       panic!("Cannot create commitment tx until we have some updates to send");
-               }
-               self.send_commitment_no_status_check(logger)
-       }
        /// Only fails in case of bad keys
        fn send_commitment_no_status_check<L: Deref>(&mut self, logger: &L) -> Result<(msgs::CommitmentSigned, ChannelMonitorUpdate), ChannelError> where L::Target: Logger {
                log_trace!(logger, "Updating HTLC state for a newly-sent commitment_signed...");
@@ -5743,7 +5755,7 @@ impl<Signer: Sign> Channel<Signer> {
        /// Only fails in case of bad keys. Used for channel_reestablish commitment_signed generation
        /// when we shouldn't change HTLC/channel state.
        fn send_commitment_no_state_update<L: Deref>(&self, logger: &L) -> Result<(msgs::CommitmentSigned, (Txid, Vec<(HTLCOutputInCommitment, Option<&HTLCSource>)>)), ChannelError> where L::Target: Logger {
-               let counterparty_keys = self.build_remote_transaction_keys()?;
+               let counterparty_keys = self.build_remote_transaction_keys();
                let commitment_stats = self.build_commitment_transaction(self.cur_counterparty_commitment_transaction_number, &counterparty_keys, false, true, logger);
                let counterparty_commitment_txid = commitment_stats.tx.trust().txid();
                let (signature, htlc_signatures);
@@ -5784,7 +5796,7 @@ impl<Signer: Sign> Channel<Signer> {
 
                        for (ref htlc_sig, ref htlc) in htlc_signatures.iter().zip(htlcs) {
                                log_trace!(logger, "Signed remote HTLC tx {} with redeemscript {} with pubkey {} -> {} in channel {}",
-                                       encode::serialize_hex(&chan_utils::build_htlc_transaction(&counterparty_commitment_txid, commitment_stats.feerate_per_kw, self.get_holder_selected_contest_delay(), htlc, self.opt_anchors(), &counterparty_keys.broadcaster_delayed_payment_key, &counterparty_keys.revocation_key)),
+                                       encode::serialize_hex(&chan_utils::build_htlc_transaction(&counterparty_commitment_txid, commitment_stats.feerate_per_kw, self.get_holder_selected_contest_delay(), htlc, self.opt_anchors(), false, &counterparty_keys.broadcaster_delayed_payment_key, &counterparty_keys.revocation_key)),
                                        encode::serialize_hex(&chan_utils::get_htlc_redeemscript(&htlc, self.opt_anchors(), &counterparty_keys)),
                                        log_bytes!(counterparty_keys.broadcaster_htlc_key.serialize()),
                                        log_bytes!(htlc_sig.serialize_compact()[..]), log_bytes!(self.channel_id()));
@@ -5800,10 +5812,11 @@ impl<Signer: Sign> Channel<Signer> {
 
        /// Adds a pending outbound HTLC to this channel, and creates a signed commitment transaction
        /// to send to the remote peer in one go.
-       /// Shorthand for calling send_htlc() followed by send_commitment(), see docs on those for
-       /// more info.
+       ///
+       /// Shorthand for calling [`Self::send_htlc`] followed by a commitment update, see docs on
+       /// [`Self::send_htlc`] and [`Self::send_commitment_no_state_update`] for more info.
        pub fn send_htlc_and_commit<L: Deref>(&mut self, amount_msat: u64, payment_hash: PaymentHash, cltv_expiry: u32, source: HTLCSource, onion_routing_packet: msgs::OnionPacket, logger: &L) -> Result<Option<(msgs::UpdateAddHTLC, msgs::CommitmentSigned, ChannelMonitorUpdate)>, ChannelError> where L::Target: Logger {
-               match self.send_htlc(amount_msat, payment_hash, cltv_expiry, source, onion_routing_packet, logger)? {
+               match self.send_htlc(amount_msat, payment_hash, cltv_expiry, source, onion_routing_packet, false, logger)? {
                        Some(update_add_htlc) => {
                                let (commitment_signed, monitor_update) = self.send_commitment_no_status_check(logger)?;
                                Ok(Some((update_add_htlc, commitment_signed, monitor_update)))
@@ -5950,9 +5963,21 @@ impl<Signer: Sign> Channel<Signer> {
                self.update_time_counter += 1;
                (monitor_update, dropped_outbound_htlcs)
        }
+
+       pub fn inflight_htlc_sources(&self) -> impl Iterator<Item=(&HTLCSource, &PaymentHash)> {
+               self.holding_cell_htlc_updates.iter()
+                       .flat_map(|htlc_update| {
+                               match htlc_update {
+                                       HTLCUpdateAwaitingACK::AddHTLC { source, payment_hash, .. }
+                                               => Some((source, payment_hash)),
+                                       _ => None,
+                               }
+                       })
+                       .chain(self.pending_outbound_htlcs.iter().map(|htlc| (&htlc.source, &htlc.payment_hash)))
+       }
 }
 
-const SERIALIZATION_VERSION: u8 = 2;
+const SERIALIZATION_VERSION: u8 = 3;
 const MIN_SERIALIZATION_VERSION: u8 = 2;
 
 impl_writeable_tlv_based_enum!(InboundHTLCRemovalReason,;
@@ -6014,7 +6039,7 @@ impl<Signer: Sign> Writeable for Channel<Signer> {
                // Note that we write out as if remove_uncommitted_htlcs_and_mark_paused had just been
                // called.
 
-               write_ver_prefix!(writer, SERIALIZATION_VERSION, MIN_SERIALIZATION_VERSION);
+               write_ver_prefix!(writer, MIN_SERIALIZATION_VERSION, MIN_SERIALIZATION_VERSION);
 
                // `user_id` used to be a single u64 value. In order to remain backwards compatible with
                // versions prior to 0.0.113, the u128 is serialized as two separate u64 values. We write
@@ -6296,6 +6321,7 @@ impl<Signer: Sign> Writeable for Channel<Signer> {
                        (21, self.outbound_scid_alias, required),
                        (23, channel_ready_event_emitted, option),
                        (25, user_id_high_opt, option),
+                       (27, self.channel_keys_id, required),
                });
 
                Ok(())
@@ -6332,16 +6358,20 @@ impl<'a, K: Deref> ReadableArgs<(&'a K, u32)> for Channel<<K::Target as KeysInte
 
                let latest_monitor_update_id = Readable::read(reader)?;
 
-               let keys_len: u32 = Readable::read(reader)?;
-               let mut keys_data = Vec::with_capacity(cmp::min(keys_len as usize, MAX_ALLOC_SIZE));
-               while keys_data.len() != keys_len as usize {
-                       // Read 1KB at a time to avoid accidentally allocating 4GB on corrupted channel keys
-                       let mut data = [0; 1024];
-                       let read_slice = &mut data[0..cmp::min(1024, keys_len as usize - keys_data.len())];
-                       reader.read_exact(read_slice)?;
-                       keys_data.extend_from_slice(read_slice);
+               let mut keys_data = None;
+               if ver <= 2 {
+                       // Read the serialize signer bytes. We'll choose to deserialize them or not based on whether
+                       // the `channel_keys_id` TLV is present below.
+                       let keys_len: u32 = Readable::read(reader)?;
+                       keys_data = Some(Vec::with_capacity(cmp::min(keys_len as usize, MAX_ALLOC_SIZE)));
+                       while keys_data.as_ref().unwrap().len() != keys_len as usize {
+                               // Read 1KB at a time to avoid accidentally allocating 4GB on corrupted channel keys
+                               let mut data = [0; 1024];
+                               let read_slice = &mut data[0..cmp::min(1024, keys_len as usize - keys_data.as_ref().unwrap().len())];
+                               reader.read_exact(read_slice)?;
+                               keys_data.as_mut().unwrap().extend_from_slice(read_slice);
+                       }
                }
-               let holder_signer = keys_source.read_chan_signer(&keys_data)?;
 
                // Read the old serialization for shutdown_pubkey, preferring the TLV field later if set.
                let mut shutdown_scriptpubkey = match <PublicKey as Readable>::read(reader) {
@@ -6559,6 +6589,7 @@ impl<'a, K: Deref> ReadableArgs<(&'a K, u32)> for Channel<<K::Target as KeysInte
                let mut channel_ready_event_emitted = None;
 
                let mut user_id_high_opt: Option<u64> = None;
+               let mut channel_keys_id: Option<[u8; 32]> = None;
 
                read_tlv_fields!(reader, {
                        (0, announcement_sigs, option),
@@ -6578,8 +6609,25 @@ impl<'a, K: Deref> ReadableArgs<(&'a K, u32)> for Channel<<K::Target as KeysInte
                        (21, outbound_scid_alias, option),
                        (23, channel_ready_event_emitted, option),
                        (25, user_id_high_opt, option),
+                       (27, channel_keys_id, option),
                });
 
+               let (channel_keys_id, holder_signer) = if let Some(channel_keys_id) = channel_keys_id {
+                       let mut holder_signer = keys_source.derive_channel_signer(channel_value_satoshis, channel_keys_id);
+                       // If we've gotten to the funding stage of the channel, populate the signer with its
+                       // required channel parameters.
+                       let non_shutdown_state = channel_state & (!MULTI_STATE_FLAGS);
+                       if non_shutdown_state >= (ChannelState::FundingCreated as u32) {
+                               holder_signer.provide_channel_parameters(&channel_parameters);
+                       }
+                       (channel_keys_id, holder_signer)
+               } else {
+                       // `keys_data` can be `None` if we had corrupted data.
+                       let keys_data = keys_data.ok_or(DecodeError::InvalidValue)?;
+                       let holder_signer = keys_source.read_chan_signer(&keys_data)?;
+                       (holder_signer.channel_keys_id(), holder_signer)
+               };
+
                if let Some(preimages) = preimages_opt {
                        let mut iter = preimages.into_iter();
                        for htlc in pending_outbound_htlcs.iter_mut() {
@@ -6729,6 +6777,7 @@ impl<'a, K: Deref> ReadableArgs<(&'a K, u32)> for Channel<<K::Target as KeysInte
                        historical_inbound_htlc_fulfills,
 
                        channel_type: channel_type.unwrap(),
+                       channel_keys_id,
                })
        }
 }
@@ -6753,7 +6802,7 @@ mod tests {
        use crate::ln::chan_utils::{htlc_success_tx_weight, htlc_timeout_tx_weight};
        use crate::chain::BestBlock;
        use crate::chain::chaininterface::{FeeEstimator, LowerBoundedFeeEstimator, ConfirmationTarget};
-       use crate::chain::keysinterface::{InMemorySigner, Recipient, KeyMaterial, KeysInterface};
+       use crate::chain::keysinterface::{BaseSign, InMemorySigner, Recipient, KeyMaterial, KeysInterface};
        use crate::chain::transaction::OutPoint;
        use crate::util::config::UserConfig;
        use crate::util::enforcing_trait_impls::EnforcingSigner;
@@ -6821,7 +6870,10 @@ mod tests {
                        ShutdownScript::new_p2wpkh_from_pubkey(PublicKey::from_secret_key(&secp_ctx, &channel_close_key))
                }
 
-               fn get_channel_signer(&self, _inbound: bool, _channel_value_satoshis: u64) -> InMemorySigner {
+               fn generate_channel_keys_id(&self, _inbound: bool, _channel_value_satoshis: u64, _user_channel_id: u128) -> [u8; 32] {
+                       self.signer.channel_keys_id()
+               }
+               fn derive_channel_signer(&self, _channel_value_satoshis: u64, _channel_keys_id: [u8; 32]) -> Self::Signer {
                        self.signer.clone()
                }
                fn get_secure_random_bytes(&self) -> [u8; 32] { [0; 32] }
@@ -7036,10 +7088,10 @@ mod tests {
                }]};
                let funding_outpoint = OutPoint{ txid: tx.txid(), index: 0 };
                let funding_created_msg = node_a_chan.get_outbound_funding_created(tx.clone(), funding_outpoint, &&logger).unwrap();
-               let (funding_signed_msg, _, _) = node_b_chan.funding_created(&funding_created_msg, best_block, &&logger).unwrap();
+               let (funding_signed_msg, _, _) = node_b_chan.funding_created(&funding_created_msg, best_block, &&keys_provider, &&logger).unwrap();
 
                // Node B --> Node A: funding signed
-               let _ = node_a_chan.funding_signed(&funding_signed_msg, best_block, &&logger);
+               let _ = node_a_chan.funding_signed(&funding_signed_msg, best_block, &&keys_provider, &&logger);
 
                // Now disconnect the two nodes and check that the commitment point in
                // Node B's channel_reestablish message is sane.
@@ -7280,7 +7332,7 @@ mod tests {
                        // These aren't set in the test vectors:
                        [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff],
                        10_000_000,
-                       [0; 32]
+                       [0; 32],
                );
 
                assert_eq!(signer.pubkeys().funding_pubkey.serialize()[..],
@@ -7309,7 +7361,7 @@ mod tests {
                                selected_contest_delay: 144
                        });
                chan.channel_transaction_parameters.funding_outpoint = Some(funding_info);
-               signer.ready_channel(&chan.channel_transaction_parameters);
+               signer.provide_channel_parameters(&chan.channel_transaction_parameters);
 
                assert_eq!(counterparty_pubkeys.payment_point.serialize()[..],
                           hex::decode("032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991").unwrap()[..]);
@@ -7327,7 +7379,7 @@ mod tests {
                let per_commitment_secret = SecretKey::from_slice(&hex::decode("1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100").unwrap()[..]).unwrap();
                let per_commitment_point = PublicKey::from_secret_key(&secp_ctx, &per_commitment_secret);
                let htlc_basepoint = &chan.holder_signer.pubkeys().htlc_basepoint;
-               let keys = TxCreationKeys::derive_new(&secp_ctx, &per_commitment_point, delayed_payment_base, htlc_basepoint, &counterparty_pubkeys.revocation_basepoint, &counterparty_pubkeys.htlc_basepoint).unwrap();
+               let keys = TxCreationKeys::derive_new(&secp_ctx, &per_commitment_point, delayed_payment_base, htlc_basepoint, &counterparty_pubkeys.revocation_basepoint, &counterparty_pubkeys.htlc_basepoint);
 
                macro_rules! test_commitment {
                        ( $counterparty_sig_hex: expr, $sig_hex: expr, $tx_hex: expr, $($remain:tt)* ) => {
@@ -7398,7 +7450,7 @@ mod tests {
                                        let ref htlc = htlcs[$htlc_idx];
                                        let htlc_tx = chan_utils::build_htlc_transaction(&unsigned_tx.txid, chan.feerate_per_kw,
                                                chan.get_counterparty_selected_contest_delay().unwrap(),
-                                               &htlc, $opt_anchors, &keys.broadcaster_delayed_payment_key, &keys.revocation_key);
+                                               &htlc, $opt_anchors, false, &keys.broadcaster_delayed_payment_key, &keys.revocation_key);
                                        let htlc_redeemscript = chan_utils::get_htlc_redeemscript(&htlc, $opt_anchors, &keys);
                                        let htlc_sighashtype = if $opt_anchors { EcdsaSighashType::SinglePlusAnyoneCanPay } else { EcdsaSighashType::All };
                                        let htlc_sighash = Message::from_slice(&sighash::SighashCache::new(&htlc_tx).segwit_signature_hash(0, &htlc_redeemscript, htlc.amount_msat / 1000, htlc_sighashtype).unwrap()[..]).unwrap();
@@ -7980,16 +8032,16 @@ mod tests {
                let per_commitment_point = PublicKey::from_secret_key(&secp_ctx, &per_commitment_secret);
                assert_eq!(per_commitment_point.serialize()[..], hex::decode("025f7117a78150fe2ef97db7cfc83bd57b2e2c0d0dd25eaf467a4a1c2a45ce1486").unwrap()[..]);
 
-               assert_eq!(chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &base_point).unwrap().serialize()[..],
+               assert_eq!(chan_utils::derive_public_key(&secp_ctx, &per_commitment_point, &base_point).serialize()[..],
                                hex::decode("0235f2dbfaa89b57ec7b055afe29849ef7ddfeb1cefdb9ebdc43f5494984db29e5").unwrap()[..]);
 
-               assert_eq!(chan_utils::derive_private_key(&secp_ctx, &per_commitment_point, &base_secret).unwrap(),
+               assert_eq!(chan_utils::derive_private_key(&secp_ctx, &per_commitment_point, &base_secret),
                                SecretKey::from_slice(&hex::decode("cbced912d3b21bf196a766651e436aff192362621ce317704ea2f75d87e7be0f").unwrap()[..]).unwrap());
 
-               assert_eq!(chan_utils::derive_public_revocation_key(&secp_ctx, &per_commitment_point, &base_point).unwrap().serialize()[..],
+               assert_eq!(chan_utils::derive_public_revocation_key(&secp_ctx, &per_commitment_point, &base_point).serialize()[..],
                                hex::decode("02916e326636d19c33f13e8c0c3a03dd157f332f3e99c317c141dd865eb01f8ff0").unwrap()[..]);
 
-               assert_eq!(chan_utils::derive_private_revocation_key(&secp_ctx, &per_commitment_secret, &base_secret).unwrap(),
+               assert_eq!(chan_utils::derive_private_revocation_key(&secp_ctx, &per_commitment_secret, &base_secret),
                                SecretKey::from_slice(&hex::decode("d09ffff62ddb2297ab000cc85bcb4283fdeb6aa052affbc9dddcf33b61078110").unwrap()[..]).unwrap());
        }
 
index 940d4dbbce56a471438799f2ae7daef863f68c24..83334c77bf39d895252a222cab80ff00143fad65 100644 (file)
@@ -46,15 +46,16 @@ use crate::ln::channel::{Channel, ChannelError, ChannelUpdateStatus, UpdateFulfi
 use crate::ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, NodeFeatures};
 #[cfg(any(feature = "_test_utils", test))]
 use crate::ln::features::InvoiceFeatures;
-use crate::routing::router::{PaymentParameters, Route, RouteHop, RoutePath, RouteParameters};
+use crate::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, RoutePath, RouteParameters};
 use crate::ln::msgs;
 use crate::ln::onion_utils;
+use crate::ln::onion_utils::HTLCFailReason;
 use crate::ln::msgs::{ChannelMessageHandler, DecodeError, LightningError, MAX_VALUE_MSAT};
 use crate::ln::wire::Encode;
 use crate::chain::keysinterface::{Sign, KeysInterface, KeysManager, Recipient};
 use crate::util::config::{UserConfig, ChannelConfig};
 use crate::util::events::{Event, EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination};
-use crate::util::{byte_utils, events};
+use crate::util::events;
 use crate::util::wakers::{Future, Notifier};
 use crate::util::scid_utils::fake_scid;
 use crate::util::ser::{BigSize, FixedLengthReader, Readable, ReadableArgs, MaybeReadable, Writeable, Writer, VecWriter};
@@ -92,8 +93,8 @@ use core::ops::Deref;
 pub(super) enum PendingHTLCRouting {
        Forward {
                onion_packet: msgs::OnionPacket,
-               /// The SCID from the onion that we should forward to. This could be a "real" SCID, an
-               /// outbound SCID alias, or a phantom node SCID.
+               /// The SCID from the onion that we should forward to. This could be a real SCID or a fake one
+               /// generated using `get_fake_scid` from the scid_utils::fake_scid module.
                short_channel_id: u64, // This should be NonZero<u64> eventually when we bump MSRV
        },
        Receive {
@@ -142,6 +143,7 @@ pub(super) struct PendingAddHTLCInfo {
        prev_short_channel_id: u64,
        prev_htlc_id: u64,
        prev_funding_outpoint: OutPoint,
+       prev_user_channel_id: u128,
 }
 
 pub(super) enum HTLCForwardInfo {
@@ -206,6 +208,24 @@ impl Readable for PaymentId {
                Ok(PaymentId(buf))
        }
 }
+
+/// An identifier used to uniquely identify an intercepted HTLC to LDK.
+/// (C-not exported) as we just use [u8; 32] directly
+#[derive(Hash, Copy, Clone, PartialEq, Eq, Debug)]
+pub struct InterceptId(pub [u8; 32]);
+
+impl Writeable for InterceptId {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               self.0.write(w)
+       }
+}
+
+impl Readable for InterceptId {
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let buf: [u8; 32] = Readable::read(r)?;
+               Ok(InterceptId(buf))
+       }
+}
 /// Tracks the inbound corresponding to an outbound HTLC
 #[allow(clippy::derive_hash_xor_eq)] // Our Hash is faithful to the data, we just don't have SecretKey::hash
 #[derive(Clone, PartialEq, Eq)]
@@ -257,31 +277,12 @@ impl HTLCSource {
        }
 }
 
-#[derive(Clone)] // See Channel::revoke_and_ack for why, tl;dr: Rust bug
-pub(super) enum HTLCFailReason {
-       LightningError {
-               err: msgs::OnionErrorPacket,
-       },
-       Reason {
-               failure_code: u16,
-               data: Vec<u8>,
-       }
-}
-
 struct ReceiveError {
        err_code: u16,
        err_data: Vec<u8>,
        msg: &'static str,
 }
 
-/// Return value for claim_funds_from_hop
-enum ClaimFundsFromHop {
-       PrevHopForceClosed,
-       MonitorUpdateFail(PublicKey, MsgHandleErrInternal, Option<u64>),
-       Success(u64),
-       DuplicateClaim,
-}
-
 type ShutdownResult = (Option<(OutPoint, ChannelMonitorUpdate)>, Vec<(HTLCSource, PaymentHash, PublicKey, [u8; 32])>);
 
 /// Error type returned across the channel_state mutex boundary. When an Err is generated for a
@@ -395,16 +396,39 @@ pub(super) enum RAACommitmentOrder {
        RevokeAndACKFirst,
 }
 
-// Note this is only exposed in cfg(test):
-pub(super) struct ChannelHolder<Signer: Sign> {
-       pub(super) by_id: HashMap<[u8; 32], Channel<Signer>>,
+/// Information about a payment which is currently being claimed.
+struct ClaimingPayment {
+       amount_msat: u64,
+       payment_purpose: events::PaymentPurpose,
+       receiver_node_id: PublicKey,
+}
+impl_writeable_tlv_based!(ClaimingPayment, {
+       (0, amount_msat, required),
+       (2, payment_purpose, required),
+       (4, receiver_node_id, required),
+});
+
+/// Information about claimable or being-claimed payments
+struct ClaimablePayments {
        /// Map from payment hash to the payment data and any HTLCs which are to us and can be
        /// failed/claimed by the user.
        ///
-       /// Note that while this is held in the same mutex as the channels themselves, no consistency
-       /// guarantees are made about the channels given here actually existing anymore by the time you
-       /// go to read them!
+       /// Note that, no consistency guarantees are made about the channels given here actually
+       /// existing anymore by the time you go to read them!
+       ///
+       /// When adding to the map, [`Self::pending_claiming_payments`] must also be checked to ensure
+       /// we don't get a duplicate payment.
        claimable_htlcs: HashMap<PaymentHash, (events::PaymentPurpose, Vec<ClaimableHTLC>)>,
+
+       /// Map from payment hash to the payment data for HTLCs which we have begun claiming, but which
+       /// are waiting on a [`ChannelMonitorUpdate`] to complete in order to be surfaced to the user
+       /// as an [`events::Event::PaymentClaimed`].
+       pending_claiming_payments: HashMap<PaymentHash, ClaimingPayment>,
+}
+
+// Note this is only exposed in cfg(test):
+pub(super) struct ChannelHolder<Signer: Sign> {
+       pub(super) by_id: HashMap<[u8; 32], Channel<Signer>>,
        /// Messages to send to peers - pushed to in the same lock that they are generated in (except
        /// for broadcast messages, where ordering isn't as strict).
        pub(super) pending_msg_events: Vec<MessageSendEvent>,
@@ -419,6 +443,16 @@ enum BackgroundEvent {
        ClosingMonitorUpdate((OutPoint, ChannelMonitorUpdate)),
 }
 
+pub(crate) enum MonitorUpdateCompletionAction {
+       /// Indicates that a payment ultimately destined for us was claimed and we should emit an
+       /// [`events::Event::PaymentClaimed`] to the user if we haven't yet generated such an event for
+       /// this payment. Note that this is only best-effort. On restart it's possible such a duplicate
+       /// event can be generated.
+       PaymentClaimed { payment_hash: PaymentHash },
+       /// Indicates an [`events::Event`] should be surfaced to the user.
+       EmitEvent { event: events::Event },
+}
+
 /// State we hold per-peer. In the future we should put channels in here, but for now we only hold
 /// the latest Init features we heard from the peer.
 struct PeerState {
@@ -672,20 +706,24 @@ pub type SimpleRefChannelManager<'a, 'b, 'c, 'd, 'e, M, T, F, L> = ChannelManage
 // `total_consistency_lock`
 //  |
 //  |__`forward_htlcs`
-//  |
-//  |__`channel_state`
 //  |   |
-//  |   |__`id_to_peer`
+//  |   |__`pending_intercepted_htlcs`
+//  |
+//  |__`pending_inbound_payments`
 //  |   |
-//  |   |__`short_to_chan_info`
+//  |   |__`claimable_payments`
 //  |   |
-//  |   |__`per_peer_state`
+//  |   |__`pending_outbound_payments`
 //  |       |
-//  |       |__`outbound_scid_aliases`
-//  |       |
-//  |       |__`pending_inbound_payments`
+//  |       |__`channel_state`
+//  |           |
+//  |           |__`id_to_peer`
 //  |           |
-//  |           |__`pending_outbound_payments`
+//  |           |__`short_to_chan_info`
+//  |           |
+//  |           |__`per_peer_state`
+//  |               |
+//  |               |__`outbound_scid_aliases`
 //  |               |
 //  |               |__`best_block`
 //  |               |
@@ -720,9 +758,9 @@ pub struct ChannelManager<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
        channel_state: Mutex<ChannelHolder<<K::Target as KeysInterface>::Signer>>,
 
        /// Storage for PaymentSecrets and any requirements on future inbound payments before we will
-       /// expose them to users via a PaymentReceived event. HTLCs which do not meet the requirements
+       /// expose them to users via a PaymentClaimable event. HTLCs which do not meet the requirements
        /// here are failed when we process them as pending-forwardable-HTLCs, and entries are removed
-       /// after we generate a PaymentReceived upon receipt of all MPP parts or when they time out.
+       /// after we generate a PaymentClaimable upon receipt of all MPP parts or when they time out.
        ///
        /// See `ChannelManager` struct-level documentation for lock order requirements.
        pending_inbound_payments: Mutex<HashMap<PaymentHash, PendingInboundPayment>>,
@@ -755,6 +793,17 @@ pub struct ChannelManager<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
        pub(super) forward_htlcs: Mutex<HashMap<u64, Vec<HTLCForwardInfo>>>,
        #[cfg(not(test))]
        forward_htlcs: Mutex<HashMap<u64, Vec<HTLCForwardInfo>>>,
+       /// Storage for HTLCs that have been intercepted and bubbled up to the user. We hold them here
+       /// until the user tells us what we should do with them.
+       ///
+       /// See `ChannelManager` struct-level documentation for lock order requirements.
+       pending_intercepted_htlcs: Mutex<HashMap<InterceptId, PendingAddHTLCInfo>>,
+
+       /// The sets of payments which are claimable or currently being claimed. See
+       /// [`ClaimablePayments`]' individual field docs for more info.
+       ///
+       /// See `ChannelManager` struct-level documentation for lock order requirements.
+       claimable_payments: Mutex<ClaimablePayments>,
 
        /// The set of outbound SCID aliases across all our channels, including unconfirmed channels
        /// and some closed channels which reached a usable state prior to being closed. This is used
@@ -1141,6 +1190,10 @@ pub struct ChannelDetails {
        /// [`ChannelHandshakeConfig::minimum_depth`]: crate::util::config::ChannelHandshakeConfig::minimum_depth
        /// [`ChannelHandshakeLimits::max_minimum_depth`]: crate::util::config::ChannelHandshakeLimits::max_minimum_depth
        pub confirmations_required: Option<u32>,
+       /// The current number of confirmations on the funding transaction.
+       ///
+       /// This value will be `None` for objects serialized with LDK versions prior to 0.0.113.
+       pub confirmations: Option<u32>,
        /// The number of blocks (after our commitment transaction confirms) that we will need to wait
        /// until we can claim our funds after we force-close the channel. During this time our
        /// counterparty is allowed to punish us if we broadcasted a stale state. If our counterparty
@@ -1517,134 +1570,6 @@ macro_rules! emit_channel_ready_event {
        }
 }
 
-macro_rules! handle_chan_restoration_locked {
-       ($self: ident, $channel_lock: expr, $channel_state: expr, $channel_entry: expr,
-        $raa: expr, $commitment_update: expr, $order: expr, $chanmon_update: expr,
-        $pending_forwards: expr, $funding_broadcastable: expr, $channel_ready: expr, $announcement_sigs: expr) => { {
-               let mut htlc_forwards = None;
-
-               let chanmon_update: Option<ChannelMonitorUpdate> = $chanmon_update; // Force type-checking to resolve
-               let chanmon_update_is_none = chanmon_update.is_none();
-               let counterparty_node_id = $channel_entry.get().get_counterparty_node_id();
-               let res = loop {
-                       let forwards: Vec<(PendingHTLCInfo, u64)> = $pending_forwards; // Force type-checking to resolve
-                       if !forwards.is_empty() {
-                               htlc_forwards = Some(($channel_entry.get().get_short_channel_id().unwrap_or($channel_entry.get().outbound_scid_alias()),
-                                       $channel_entry.get().get_funding_txo().unwrap(), forwards));
-                       }
-
-                       if chanmon_update.is_some() {
-                               // On reconnect, we, by definition, only resend a channel_ready if there have been
-                               // no commitment updates, so the only channel monitor update which could also be
-                               // associated with a channel_ready would be the funding_created/funding_signed
-                               // monitor update. That monitor update failing implies that we won't send
-                               // channel_ready until it's been updated, so we can't have a channel_ready and a
-                               // monitor update here (so we don't bother to handle it correctly below).
-                               assert!($channel_ready.is_none());
-                               // A channel monitor update makes no sense without either a channel_ready or a
-                               // commitment update to process after it. Since we can't have a channel_ready, we
-                               // only bother to handle the monitor-update + commitment_update case below.
-                               assert!($commitment_update.is_some());
-                       }
-
-                       if let Some(msg) = $channel_ready {
-                               // Similar to the above, this implies that we're letting the channel_ready fly
-                               // before it should be allowed to.
-                               assert!(chanmon_update.is_none());
-                               send_channel_ready!($self, $channel_state.pending_msg_events, $channel_entry.get(), msg);
-                       }
-                       if let Some(msg) = $announcement_sigs {
-                               $channel_state.pending_msg_events.push(events::MessageSendEvent::SendAnnouncementSignatures {
-                                       node_id: counterparty_node_id,
-                                       msg,
-                               });
-                       }
-
-                       emit_channel_ready_event!($self, $channel_entry.get_mut());
-
-                       let funding_broadcastable: Option<Transaction> = $funding_broadcastable; // Force type-checking to resolve
-                       if let Some(monitor_update) = chanmon_update {
-                               // We only ever broadcast a funding transaction in response to a funding_signed
-                               // message and the resulting monitor update. Thus, on channel_reestablish
-                               // message handling we can't have a funding transaction to broadcast. When
-                               // processing a monitor update finishing resulting in a funding broadcast, we
-                               // cannot have a second monitor update, thus this case would indicate a bug.
-                               assert!(funding_broadcastable.is_none());
-                               // Given we were just reconnected or finished updating a channel monitor, the
-                               // only case where we can get a new ChannelMonitorUpdate would be if we also
-                               // have some commitment updates to send as well.
-                               assert!($commitment_update.is_some());
-                               match $self.chain_monitor.update_channel($channel_entry.get().get_funding_txo().unwrap(), monitor_update) {
-                                       ChannelMonitorUpdateStatus::Completed => {},
-                                       e => {
-                                               // channel_reestablish doesn't guarantee the order it returns is sensical
-                                               // for the messages it returns, but if we're setting what messages to
-                                               // re-transmit on monitor update success, we need to make sure it is sane.
-                                               let mut order = $order;
-                                               if $raa.is_none() {
-                                                       order = RAACommitmentOrder::CommitmentFirst;
-                                               }
-                                               break handle_monitor_update_res!($self, e, $channel_entry, order, $raa.is_some(), true);
-                                       }
-                               }
-                       }
-
-                       macro_rules! handle_cs { () => {
-                               if let Some(update) = $commitment_update {
-                                       $channel_state.pending_msg_events.push(events::MessageSendEvent::UpdateHTLCs {
-                                               node_id: counterparty_node_id,
-                                               updates: update,
-                                       });
-                               }
-                       } }
-                       macro_rules! handle_raa { () => {
-                               if let Some(revoke_and_ack) = $raa {
-                                       $channel_state.pending_msg_events.push(events::MessageSendEvent::SendRevokeAndACK {
-                                               node_id: counterparty_node_id,
-                                               msg: revoke_and_ack,
-                                       });
-                               }
-                       } }
-                       match $order {
-                               RAACommitmentOrder::CommitmentFirst => {
-                                       handle_cs!();
-                                       handle_raa!();
-                               },
-                               RAACommitmentOrder::RevokeAndACKFirst => {
-                                       handle_raa!();
-                                       handle_cs!();
-                               },
-                       }
-                       if let Some(tx) = funding_broadcastable {
-                               log_info!($self.logger, "Broadcasting funding transaction with txid {}", tx.txid());
-                               $self.tx_broadcaster.broadcast_transaction(&tx);
-                       }
-                       break Ok(());
-               };
-
-               if chanmon_update_is_none {
-                       // If there was no ChannelMonitorUpdate, we should never generate an Err in the res loop
-                       // above. Doing so would imply calling handle_err!() from channel_monitor_updated() which
-                       // should *never* end up calling back to `chain_monitor.update_channel()`.
-                       assert!(res.is_ok());
-               }
-
-               (htlc_forwards, res, counterparty_node_id)
-       } }
-}
-
-macro_rules! post_handle_chan_restoration {
-       ($self: ident, $locked_res: expr) => { {
-               let (htlc_forwards, res, counterparty_node_id) = $locked_res;
-
-               let _ = handle_error!($self, res, counterparty_node_id);
-
-               if let Some(forwards) = htlc_forwards {
-                       $self.forward_htlcs(&mut [forwards][..]);
-               }
-       } }
-}
-
 impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F, L>
        where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
         T::Target: BroadcasterInterface,
@@ -1678,13 +1603,14 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
                        channel_state: Mutex::new(ChannelHolder{
                                by_id: HashMap::new(),
-                               claimable_htlcs: HashMap::new(),
                                pending_msg_events: Vec::new(),
                        }),
                        outbound_scid_aliases: Mutex::new(HashSet::new()),
                        pending_inbound_payments: Mutex::new(HashMap::new()),
                        pending_outbound_payments: Mutex::new(HashMap::new()),
                        forward_htlcs: Mutex::new(HashMap::new()),
+                       claimable_payments: Mutex::new(ClaimablePayments { claimable_htlcs: HashMap::new(), pending_claiming_payments: HashMap::new() }),
+                       pending_intercepted_htlcs: Mutex::new(HashMap::new()),
                        id_to_peer: Mutex::new(HashMap::new()),
                        short_to_chan_info: FairRwLock::new(HashMap::new()),
 
@@ -1817,6 +1743,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                let mut res = Vec::new();
                {
                        let channel_state = self.channel_state.lock().unwrap();
+                       let best_block_height = self.best_block.read().unwrap().height();
                        res.reserve(channel_state.by_id.len());
                        for (channel_id, channel) in channel_state.by_id.iter().filter(f) {
                                let balance = channel.get_available_balances();
@@ -1853,6 +1780,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        next_outbound_htlc_limit_msat: balance.next_outbound_htlc_limit_msat,
                                        user_channel_id: channel.get_user_id(),
                                        confirmations_required: channel.minimum_depth(),
+                                       confirmations: Some(channel.get_funding_tx_confirmations(best_block_height)),
                                        force_close_spend_delay: channel.get_counterparty_selected_contest_delay(),
                                        is_outbound: channel.is_outbound(),
                                        is_channel_ready: channel.is_usable(),
@@ -1922,14 +1850,16 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        if *counterparty_node_id != chan_entry.get().get_counterparty_node_id(){
                                                return Err(APIError::APIMisuseError { err: "The passed counterparty_node_id doesn't match the channel's counterparty node_id".to_owned() });
                                        }
-                                       let per_peer_state = self.per_peer_state.read().unwrap();
-                                       let (shutdown_msg, monitor_update, htlcs) = match per_peer_state.get(&counterparty_node_id) {
-                                               Some(peer_state) => {
-                                                       let peer_state = peer_state.lock().unwrap();
-                                                       let their_features = &peer_state.latest_features;
-                                                       chan_entry.get_mut().get_shutdown(&self.keys_manager, their_features, target_feerate_sats_per_1000_weight)?
-                                               },
-                                               None => return Err(APIError::ChannelUnavailable { err: format!("Not connected to node: {}", counterparty_node_id) }),
+                                       let (shutdown_msg, monitor_update, htlcs) = {
+                                               let per_peer_state = self.per_peer_state.read().unwrap();
+                                               match per_peer_state.get(&counterparty_node_id) {
+                                                       Some(peer_state) => {
+                                                               let peer_state = peer_state.lock().unwrap();
+                                                               let their_features = &peer_state.latest_features;
+                                                               chan_entry.get_mut().get_shutdown(&self.keys_manager, their_features, target_feerate_sats_per_1000_weight)?
+                                                       },
+                                                       None => return Err(APIError::ChannelUnavailable { err: format!("Not connected to node: {}", counterparty_node_id) }),
+                                               }
                                        };
                                        failed_htlcs = htlcs;
 
@@ -1965,8 +1895,9 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                };
 
                for htlc_source in failed_htlcs.drain(..) {
+                       let reason = HTLCFailReason::from_failure_code(0x4000 | 8);
                        let receiver = HTLCDestination::NextHopChannel { node_id: Some(*counterparty_node_id), channel_id: *channel_id };
-                       self.fail_htlc_backwards_internal(htlc_source.0, &htlc_source.1, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }, receiver);
+                       self.fail_htlc_backwards_internal(&htlc_source.0, &htlc_source.1, &reason, receiver);
                }
 
                let _ = handle_error!(self, result, *counterparty_node_id);
@@ -2023,8 +1954,9 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                log_debug!(self.logger, "Finishing force-closure of channel with {} HTLCs to fail", failed_htlcs.len());
                for htlc_source in failed_htlcs.drain(..) {
                        let (source, payment_hash, counterparty_node_id, channel_id) = htlc_source;
+                       let reason = HTLCFailReason::from_failure_code(0x4000 | 8);
                        let receiver = HTLCDestination::NextHopChannel { node_id: Some(counterparty_node_id), channel_id };
-                       self.fail_htlc_backwards_internal(source, &payment_hash, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }, receiver);
+                       self.fail_htlc_backwards_internal(&source, &payment_hash, &reason, receiver);
                }
                if let Some((funding_txo, monitor_update)) = monitor_update_option {
                        // There isn't anything we can do if we get an update failure - we're already
@@ -2130,7 +2062,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                        return Err(ReceiveError {
                                msg: "Upstream node set CLTV to the wrong value",
                                err_code: 18,
-                               err_data: byte_utils::be32_to_array(cltv_expiry).to_vec()
+                               err_data: cltv_expiry.to_be_bytes().to_vec()
                        })
                }
                // final_expiry_too_soon
@@ -2139,17 +2071,20 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                // Also, ensure that, in the case of an unknown preimage for the received payment hash, our
                // payment logic has enough time to fail the HTLC backward before our onchain logic triggers a
                // channel closure (see HTLC_FAIL_BACK_BUFFER rationale).
-               if (hop_data.outgoing_cltv_value as u64) <= self.best_block.read().unwrap().height() as u64 + HTLC_FAIL_BACK_BUFFER as u64 + 1  {
+               let current_height: u32 = self.best_block.read().unwrap().height();
+               if (hop_data.outgoing_cltv_value as u64) <= current_height as u64 + HTLC_FAIL_BACK_BUFFER as u64 + 1 {
+                       let mut err_data = Vec::with_capacity(12);
+                       err_data.extend_from_slice(&amt_msat.to_be_bytes());
+                       err_data.extend_from_slice(&current_height.to_be_bytes());
                        return Err(ReceiveError {
-                               err_code: 17,
-                               err_data: Vec::new(),
+                               err_code: 0x4000 | 15, err_data,
                                msg: "The final CLTV expiry is too soon to handle",
                        });
                }
                if hop_data.amt_to_forward > amt_msat {
                        return Err(ReceiveError {
                                err_code: 19,
-                               err_data: byte_utils::be64_to_array(amt_msat).to_vec(),
+                               err_data: amt_msat.to_be_bytes().to_vec(),
                                msg: "Upstream node sent less than we were supposed to receive in payment",
                        });
                }
@@ -2250,7 +2185,8 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        return PendingHTLCStatus::Fail(HTLCFailureMsg::Relay(msgs::UpdateFailHTLC {
                                                channel_id: msg.channel_id,
                                                htlc_id: msg.htlc_id,
-                                               reason: onion_utils::build_first_hop_failure_packet(&shared_secret, $err_code, $data),
+                                               reason: HTLCFailReason::reason($err_code, $data.to_vec())
+                                                       .get_encrypted_failure_packet(&shared_secret, &None),
                                        }));
                                }
                        }
@@ -2315,14 +2251,17 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                        // with a short_channel_id of 0. This is important as various things later assume
                        // short_channel_id is non-0 in any ::Forward.
                        if let &PendingHTLCRouting::Forward { ref short_channel_id, .. } = routing {
-                               if let Some((err, code, chan_update)) = loop {
+                               if let Some((err, mut code, chan_update)) = loop {
                                        let id_option = self.short_to_chan_info.read().unwrap().get(&short_channel_id).cloned();
                                        let mut channel_state = self.channel_state.lock().unwrap();
                                        let forwarding_id_opt = match id_option {
                                                None => { // unknown_next_peer
                                                        // Note that this is likely a timing oracle for detecting whether an scid is a
-                                                       // phantom.
-                                                       if fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, *short_channel_id, &self.genesis_hash) {
+                                                       // phantom or an intercept.
+                                                       if (self.default_configuration.accept_intercept_htlcs &&
+                                                          fake_scid::is_valid_intercept(&self.fake_scid_rand_bytes, *short_channel_id, &self.genesis_hash)) ||
+                                                          fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, *short_channel_id, &self.genesis_hash)
+                                                       {
                                                                None
                                                        } else {
                                                                break Some(("Don't have available channel for forwarding as requested.", 0x4000 | 10, None));
@@ -2369,10 +2308,13 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                }
                                                chan_update_opt
                                        } else {
-                                               if (msg.cltv_expiry as u64) < (*outgoing_cltv_value) as u64 + MIN_CLTV_EXPIRY_DELTA as u64 { // incorrect_cltv_expiry
+                                               if (msg.cltv_expiry as u64) < (*outgoing_cltv_value) as u64 + MIN_CLTV_EXPIRY_DELTA as u64 {
+                                                       // We really should set `incorrect_cltv_expiry` here but as we're not
+                                                       // forwarding over a real channel we can't generate a channel_update
+                                                       // for it. Instead we just return a generic temporary_node_failure.
                                                        break Some((
                                                                "Forwarding node has tampered with the intended HTLC values or origin node has an obsolete cltv_expiry_delta",
-                                                               0x1000 | 13, None,
+                                                               0x2000 | 2, None,
                                                        ));
                                                }
                                                None
@@ -2418,6 +2360,12 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                (chan_update.serialized_length() as u16 + 2).write(&mut res).expect("Writes cannot fail");
                                                msgs::ChannelUpdate::TYPE.write(&mut res).expect("Writes cannot fail");
                                                chan_update.write(&mut res).expect("Writes cannot fail");
+                                       } else if code & 0x1000 == 0x1000 {
+                                               // If we're trying to return an error that requires a `channel_update` but
+                                               // we're forwarding to a phantom or intercept "channel" (i.e. cannot
+                                               // generate an update), just use the generic "temporary_node_failure"
+                                               // instead.
+                                               code = 0x2000 | 2;
                                        }
                                        return_err!(err, code, &res.0[..]);
                                }
@@ -2493,10 +2441,10 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                let session_priv = SecretKey::from_slice(&session_priv_bytes[..]).expect("RNG is busted");
 
                let onion_keys = onion_utils::construct_onion_keys(&self.secp_ctx, &path, &session_priv)
-                       .map_err(|_| APIError::RouteError{err: "Pubkey along hop was maliciously selected"})?;
+                       .map_err(|_| APIError::InvalidRoute{err: "Pubkey along hop was maliciously selected"})?;
                let (onion_payloads, htlc_msat, htlc_cltv) = onion_utils::build_onion_payloads(path, total_value, payment_secret, cur_height, keysend_preimage)?;
                if onion_utils::route_size_insane(&onion_payloads) {
-                       return Err(APIError::RouteError{err: "Route size too large considering onion data"});
+                       return Err(APIError::InvalidRoute{err: "Route size too large considering onion data"});
                }
                let onion_packet = onion_utils::construct_onion_packet(onion_payloads, onion_keys, prng_seed, payment_hash);
 
@@ -2513,7 +2461,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                        if let hash_map::Entry::Occupied(mut chan) = channel_state.by_id.entry(id) {
                                match {
                                        if chan.get().get_counterparty_node_id() != path.first().unwrap().pubkey {
-                                               return Err(APIError::RouteError{err: "Node ID mismatch on first hop!"});
+                                               return Err(APIError::InvalidRoute{err: "Node ID mismatch on first hop!"});
                                        }
                                        if !chan.get().is_live() {
                                                return Err(APIError::ChannelUnavailable{err: "Peer for first hop currently disconnected/pending monitor update!".to_owned()});
@@ -2588,7 +2536,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        /// fields for more info.
        ///
        /// If a pending payment is currently in-flight with the same [`PaymentId`] provided, this
-       /// method will error with an [`APIError::RouteError`]. Note, however, that once a payment
+       /// method will error with an [`APIError::InvalidRoute`]. Note, however, that once a payment
        /// is no longer pending (either via [`ChannelManager::abandon_payment`], or handling of an
        /// [`Event::PaymentSent`]) LDK will not stop you from sending a second payment with the same
        /// [`PaymentId`].
@@ -2607,7 +2555,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        /// PaymentSendFailure for more info.
        ///
        /// In general, a path may raise:
-       ///  * [`APIError::RouteError`] when an invalid route or forwarding parameter (cltv_delta, fee,
+       ///  * [`APIError::InvalidRoute`] when an invalid route or forwarding parameter (cltv_delta, fee,
        ///    node public key) is specified.
        ///  * [`APIError::ChannelUnavailable`] if the next-hop channel is not available for updates
        ///    (including due to previous monitor update failure or new permanent monitor update
@@ -2672,7 +2620,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
        fn send_payment_internal(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, keysend_preimage: Option<PaymentPreimage>, payment_id: PaymentId, recv_value_msat: Option<u64>, onion_session_privs: Vec<[u8; 32]>) -> Result<(), PaymentSendFailure> {
                if route.paths.len() < 1 {
-                       return Err(PaymentSendFailure::ParameterError(APIError::RouteError{err: "There must be at least one path to send over"}));
+                       return Err(PaymentSendFailure::ParameterError(APIError::InvalidRoute{err: "There must be at least one path to send over"}));
                }
                if payment_secret.is_none() && route.paths.len() > 1 {
                        return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError{err: "Payment secret is required for multi-path payments".to_string()}));
@@ -2682,12 +2630,12 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                let mut path_errs = Vec::with_capacity(route.paths.len());
                'path_check: for path in route.paths.iter() {
                        if path.len() < 1 || path.len() > 20 {
-                               path_errs.push(Err(APIError::RouteError{err: "Path didn't go anywhere/had bogus size"}));
+                               path_errs.push(Err(APIError::InvalidRoute{err: "Path didn't go anywhere/had bogus size"}));
                                continue 'path_check;
                        }
                        for (idx, hop) in path.iter().enumerate() {
                                if idx != path.len() - 1 && hop.pubkey == our_node_id {
-                                       path_errs.push(Err(APIError::RouteError{err: "Path went through us but wasn't a simple rebalance loop to us"}));
+                                       path_errs.push(Err(APIError::InvalidRoute{err: "Path went through us but wasn't a simple rebalance loop to us"}));
                                        continue 'path_check;
                                }
                        }
@@ -3138,6 +3086,102 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                Ok(())
        }
 
+       /// Attempts to forward an intercepted HTLC over the provided channel id and with the provided
+       /// amount to forward. Should only be called in response to an [`HTLCIntercepted`] event.
+       ///
+       /// Intercepted HTLCs can be useful for Lightning Service Providers (LSPs) to open a just-in-time
+       /// channel to a receiving node if the node lacks sufficient inbound liquidity.
+       ///
+       /// To make use of intercepted HTLCs, set [`UserConfig::accept_intercept_htlcs`] and use
+       /// [`ChannelManager::get_intercept_scid`] to generate short channel id(s) to put in the
+       /// receiver's invoice route hints. These route hints will signal to LDK to generate an
+       /// [`HTLCIntercepted`] event when it receives the forwarded HTLC, and this method or
+       /// [`ChannelManager::fail_intercepted_htlc`] MUST be called in response to the event.
+       ///
+       /// Note that LDK does not enforce fee requirements in `amt_to_forward_msat`, and will not stop
+       /// you from forwarding more than you received.
+       ///
+       /// Errors if the event was not handled in time, in which case the HTLC was automatically failed
+       /// backwards.
+       ///
+       /// [`UserConfig::accept_intercept_htlcs`]: crate::util::config::UserConfig::accept_intercept_htlcs
+       /// [`HTLCIntercepted`]: events::Event::HTLCIntercepted
+       // TODO: when we move to deciding the best outbound channel at forward time, only take
+       // `next_node_id` and not `next_hop_channel_id`
+       pub fn forward_intercepted_htlc(&self, intercept_id: InterceptId, next_hop_channel_id: &[u8; 32], _next_node_id: PublicKey, amt_to_forward_msat: u64) -> Result<(), APIError> {
+               let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
+
+               let next_hop_scid = match self.channel_state.lock().unwrap().by_id.get(next_hop_channel_id) {
+                       Some(chan) => {
+                               if !chan.is_usable() {
+                                       return Err(APIError::ChannelUnavailable {
+                                               err: format!("Channel with id {} not fully established", log_bytes!(*next_hop_channel_id))
+                                       })
+                               }
+                               chan.get_short_channel_id().unwrap_or(chan.outbound_scid_alias())
+                       },
+                       None => return Err(APIError::ChannelUnavailable {
+                               err: format!("Channel with id {} not found", log_bytes!(*next_hop_channel_id))
+                       })
+               };
+
+               let payment = self.pending_intercepted_htlcs.lock().unwrap().remove(&intercept_id)
+                       .ok_or_else(|| APIError::APIMisuseError {
+                               err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0))
+                       })?;
+
+               let routing = match payment.forward_info.routing {
+                       PendingHTLCRouting::Forward { onion_packet, .. } => {
+                               PendingHTLCRouting::Forward { onion_packet, short_channel_id: next_hop_scid }
+                       },
+                       _ => unreachable!() // Only `PendingHTLCRouting::Forward`s are intercepted
+               };
+               let pending_htlc_info = PendingHTLCInfo {
+                       outgoing_amt_msat: amt_to_forward_msat, routing, ..payment.forward_info
+               };
+
+               let mut per_source_pending_forward = [(
+                       payment.prev_short_channel_id,
+                       payment.prev_funding_outpoint,
+                       payment.prev_user_channel_id,
+                       vec![(pending_htlc_info, payment.prev_htlc_id)]
+               )];
+               self.forward_htlcs(&mut per_source_pending_forward);
+               Ok(())
+       }
+
+       /// Fails the intercepted HTLC indicated by intercept_id. Should only be called in response to
+       /// an [`HTLCIntercepted`] event. See [`ChannelManager::forward_intercepted_htlc`].
+       ///
+       /// Errors if the event was not handled in time, in which case the HTLC was automatically failed
+       /// backwards.
+       ///
+       /// [`HTLCIntercepted`]: events::Event::HTLCIntercepted
+       pub fn fail_intercepted_htlc(&self, intercept_id: InterceptId) -> Result<(), APIError> {
+               let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
+
+               let payment = self.pending_intercepted_htlcs.lock().unwrap().remove(&intercept_id)
+                       .ok_or_else(|| APIError::APIMisuseError {
+                               err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0))
+                       })?;
+
+               if let PendingHTLCRouting::Forward { short_channel_id, .. } = payment.forward_info.routing {
+                       let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
+                               short_channel_id: payment.prev_short_channel_id,
+                               outpoint: payment.prev_funding_outpoint,
+                               htlc_id: payment.prev_htlc_id,
+                               incoming_packet_shared_secret: payment.forward_info.incoming_shared_secret,
+                               phantom_shared_secret: None,
+                       });
+
+                       let failure_reason = HTLCFailReason::from_failure_code(0x4000 | 10);
+                       let destination = HTLCDestination::UnknownNextHop { requested_forward_scid: short_channel_id };
+                       self.fail_htlc_backwards_internal(&htlc_source, &payment.forward_info.payment_hash, &failure_reason, destination);
+               } else { unreachable!() } // Only `PendingHTLCRouting::Forward`s are intercepted
+
+               Ok(())
+       }
+
        /// Processes HTLCs which are pending waiting on random forward delay.
        ///
        /// Should only really ever be called in response to a PendingHTLCsForwardable event.
@@ -3147,22 +3191,19 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
                let mut new_events = Vec::new();
                let mut failed_forwards = Vec::new();
-               let mut phantom_receives: Vec<(u64, OutPoint, Vec<(PendingHTLCInfo, u64)>)> = Vec::new();
-               let mut handle_errors = Vec::new();
+               let mut phantom_receives: Vec<(u64, OutPoint, u128, Vec<(PendingHTLCInfo, u64)>)> = Vec::new();
                {
                        let mut forward_htlcs = HashMap::new();
                        mem::swap(&mut forward_htlcs, &mut self.forward_htlcs.lock().unwrap());
 
                        for (short_chan_id, mut pending_forwards) in forward_htlcs {
-                               let mut channel_state_lock = self.channel_state.lock().unwrap();
-                               let channel_state = &mut *channel_state_lock;
                                if short_chan_id != 0 {
                                        macro_rules! forwarding_channel_not_found {
                                                () => {
                                                        for forward_info in pending_forwards.drain(..) {
                                                                match forward_info {
                                                                        HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
-                                                                               prev_short_channel_id, prev_htlc_id, prev_funding_outpoint,
+                                                                               prev_short_channel_id, prev_htlc_id, prev_funding_outpoint, prev_user_channel_id,
                                                                                forward_info: PendingHTLCInfo {
                                                                                        routing, incoming_shared_secret, payment_hash, outgoing_amt_msat,
                                                                                        outgoing_cltv_value, incoming_amt_msat: _
@@ -3187,7 +3228,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                                                };
 
                                                                                                failed_forwards.push((htlc_source, payment_hash,
-                                                                                                       HTLCFailReason::Reason { failure_code: $err_code, data: $err_data },
+                                                                                                       HTLCFailReason::reason($err_code, $err_data),
                                                                                                        reason
                                                                                                ));
                                                                                                continue;
@@ -3228,7 +3269,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                                                match next_hop {
                                                                                                        onion_utils::Hop::Receive(hop_data) => {
                                                                                                                match self.construct_recv_pending_htlc_info(hop_data, incoming_shared_secret, payment_hash, outgoing_amt_msat, outgoing_cltv_value, Some(phantom_shared_secret)) {
-                                                                                                                       Ok(info) => phantom_receives.push((prev_short_channel_id, prev_funding_outpoint, vec![(info, prev_htlc_id)])),
+                                                                                                                       Ok(info) => phantom_receives.push((prev_short_channel_id, prev_funding_outpoint, prev_user_channel_id, vec![(info, prev_htlc_id)])),
                                                                                                                        Err(ReceiveError { err_code, err_data, msg }) => failed_payment!(msg, err_code, err_data, Some(phantom_shared_secret))
                                                                                                                }
                                                                                                        },
@@ -3258,18 +3299,18 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                        continue;
                                                }
                                        };
+                                       let mut channel_state_lock = self.channel_state.lock().unwrap();
+                                       let channel_state = &mut *channel_state_lock;
                                        match channel_state.by_id.entry(forward_chan_id) {
                                                hash_map::Entry::Vacant(_) => {
                                                        forwarding_channel_not_found!();
                                                        continue;
                                                },
                                                hash_map::Entry::Occupied(mut chan) => {
-                                                       let mut add_htlc_msgs = Vec::new();
-                                                       let mut fail_htlc_msgs = Vec::new();
                                                        for forward_info in pending_forwards.drain(..) {
                                                                match forward_info {
                                                                        HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
-                                                                               prev_short_channel_id, prev_htlc_id, prev_funding_outpoint ,
+                                                                               prev_short_channel_id, prev_htlc_id, prev_funding_outpoint, prev_user_channel_id: _,
                                                                                forward_info: PendingHTLCInfo {
                                                                                        incoming_shared_secret, payment_hash, outgoing_amt_msat, outgoing_cltv_value,
                                                                                        routing: PendingHTLCRouting::Forward { onion_packet, .. }, incoming_amt_msat: _,
@@ -3284,34 +3325,21 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                                        // Phantom payments are only PendingHTLCRouting::Receive.
                                                                                        phantom_shared_secret: None,
                                                                                });
-                                                                               match chan.get_mut().send_htlc(outgoing_amt_msat, payment_hash, outgoing_cltv_value, htlc_source.clone(), onion_packet, &self.logger) {
-                                                                                       Err(e) => {
-                                                                                               if let ChannelError::Ignore(msg) = e {
-                                                                                                       log_trace!(self.logger, "Failed to forward HTLC with payment_hash {}: {}", log_bytes!(payment_hash.0), msg);
-                                                                                               } else {
-                                                                                                       panic!("Stated return value requirements in send_htlc() were not met");
-                                                                                               }
-                                                                                               let (failure_code, data) = self.get_htlc_temp_fail_err_and_data(0x1000|7, short_chan_id, chan.get());
-                                                                                               failed_forwards.push((htlc_source, payment_hash,
-                                                                                                       HTLCFailReason::Reason { failure_code, data },
-                                                                                                       HTLCDestination::NextHopChannel { node_id: Some(chan.get().get_counterparty_node_id()), channel_id: forward_chan_id }
-                                                                                               ));
-                                                                                               continue;
-                                                                                       },
-                                                                                       Ok(update_add) => {
-                                                                                               match update_add {
-                                                                                                       Some(msg) => { add_htlc_msgs.push(msg); },
-                                                                                                       None => {
-                                                                                                               // Nothing to do here...we're waiting on a remote
-                                                                                                               // revoke_and_ack before we can add anymore HTLCs. The Channel
-                                                                                                               // will automatically handle building the update_add_htlc and
-                                                                                                               // commitment_signed messages when we can.
-                                                                                                               // TODO: Do some kind of timer to set the channel as !is_live()
-                                                                                                               // as we don't really want others relying on us relaying through
-                                                                                                               // this channel currently :/.
-                                                                                                       }
-                                                                                               }
+                                                                               if let Err(e) = chan.get_mut().queue_add_htlc(outgoing_amt_msat,
+                                                                                       payment_hash, outgoing_cltv_value, htlc_source.clone(),
+                                                                                       onion_packet, &self.logger)
+                                                                               {
+                                                                                       if let ChannelError::Ignore(msg) = e {
+                                                                                               log_trace!(self.logger, "Failed to forward HTLC with payment_hash {}: {}", log_bytes!(payment_hash.0), msg);
+                                                                                       } else {
+                                                                                               panic!("Stated return value requirements in send_htlc() were not met");
                                                                                        }
+                                                                                       let (failure_code, data) = self.get_htlc_temp_fail_err_and_data(0x1000|7, short_chan_id, chan.get());
+                                                                                       failed_forwards.push((htlc_source, payment_hash,
+                                                                                               HTLCFailReason::reason(failure_code, data),
+                                                                                               HTLCDestination::NextHopChannel { node_id: Some(chan.get().get_counterparty_node_id()), channel_id: forward_chan_id }
+                                                                                       ));
+                                                                                       continue;
                                                                                }
                                                                        },
                                                                        HTLCForwardInfo::AddHTLC { .. } => {
@@ -3319,84 +3347,29 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                        },
                                                                        HTLCForwardInfo::FailHTLC { htlc_id, err_packet } => {
                                                                                log_trace!(self.logger, "Failing HTLC back to channel with short id {} (backward HTLC ID {}) after delay", short_chan_id, htlc_id);
-                                                                               match chan.get_mut().get_update_fail_htlc(htlc_id, err_packet, &self.logger) {
-                                                                                       Err(e) => {
-                                                                                               if let ChannelError::Ignore(msg) = e {
-                                                                                                       log_trace!(self.logger, "Failed to fail HTLC with ID {} backwards to short_id {}: {}", htlc_id, short_chan_id, msg);
-                                                                                               } else {
-                                                                                                       panic!("Stated return value requirements in get_update_fail_htlc() were not met");
-                                                                                               }
-                                                                                               // fail-backs are best-effort, we probably already have one
-                                                                                               // pending, and if not that's OK, if not, the channel is on
-                                                                                               // the chain and sending the HTLC-Timeout is their problem.
-                                                                                               continue;
-                                                                                       },
-                                                                                       Ok(Some(msg)) => { fail_htlc_msgs.push(msg); },
-                                                                                       Ok(None) => {
-                                                                                               // Nothing to do here...we're waiting on a remote
-                                                                                               // revoke_and_ack before we can update the commitment
-                                                                                               // transaction. The Channel will automatically handle
-                                                                                               // building the update_fail_htlc and commitment_signed
-                                                                                               // messages when we can.
-                                                                                               // We don't need any kind of timer here as they should fail
-                                                                                               // the channel onto the chain if they can't get our
-                                                                                               // update_fail_htlc in time, it's not our problem.
+                                                                               if let Err(e) = chan.get_mut().queue_fail_htlc(
+                                                                                       htlc_id, err_packet, &self.logger
+                                                                               ) {
+                                                                                       if let ChannelError::Ignore(msg) = e {
+                                                                                               log_trace!(self.logger, "Failed to fail HTLC with ID {} backwards to short_id {}: {}", htlc_id, short_chan_id, msg);
+                                                                                       } else {
+                                                                                               panic!("Stated return value requirements in queue_fail_htlc() were not met");
                                                                                        }
+                                                                                       // fail-backs are best-effort, we probably already have one
+                                                                                       // pending, and if not that's OK, if not, the channel is on
+                                                                                       // the chain and sending the HTLC-Timeout is their problem.
+                                                                                       continue;
                                                                                }
                                                                        },
                                                                }
                                                        }
-
-                                                       if !add_htlc_msgs.is_empty() || !fail_htlc_msgs.is_empty() {
-                                                               let (commitment_msg, monitor_update) = match chan.get_mut().send_commitment(&self.logger) {
-                                                                       Ok(res) => res,
-                                                                       Err(e) => {
-                                                                               // We surely failed send_commitment due to bad keys, in that case
-                                                                               // close channel and then send error message to peer.
-                                                                               let counterparty_node_id = chan.get().get_counterparty_node_id();
-                                                                               let err: Result<(), _>  = match e {
-                                                                                       ChannelError::Ignore(_) | ChannelError::Warn(_) => {
-                                                                                               panic!("Stated return value requirements in send_commitment() were not met");
-                                                                                       }
-                                                                                       ChannelError::Close(msg) => {
-                                                                                               log_trace!(self.logger, "Closing channel {} due to Close-required error: {}", log_bytes!(chan.key()[..]), msg);
-                                                                                               let mut channel = remove_channel!(self, chan);
-                                                                                               // ChannelClosed event is generated by handle_error for us.
-                                                                                               Err(MsgHandleErrInternal::from_finish_shutdown(msg, channel.channel_id(), channel.get_user_id(), channel.force_shutdown(true), self.get_channel_update_for_broadcast(&channel).ok()))
-                                                                                       },
-                                                                               };
-                                                                               handle_errors.push((counterparty_node_id, err));
-                                                                               continue;
-                                                                       }
-                                                               };
-                                                               match self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update) {
-                                                                       ChannelMonitorUpdateStatus::Completed => {},
-                                                                       e => {
-                                                                               handle_errors.push((chan.get().get_counterparty_node_id(), handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, false, true)));
-                                                                               continue;
-                                                                       }
-                                                               }
-                                                               log_debug!(self.logger, "Forwarding HTLCs resulted in a commitment update with {} HTLCs added and {} HTLCs failed for channel {}",
-                                                                       add_htlc_msgs.len(), fail_htlc_msgs.len(), log_bytes!(chan.get().channel_id()));
-                                                               channel_state.pending_msg_events.push(events::MessageSendEvent::UpdateHTLCs {
-                                                                       node_id: chan.get().get_counterparty_node_id(),
-                                                                       updates: msgs::CommitmentUpdate {
-                                                                               update_add_htlcs: add_htlc_msgs,
-                                                                               update_fulfill_htlcs: Vec::new(),
-                                                                               update_fail_htlcs: fail_htlc_msgs,
-                                                                               update_fail_malformed_htlcs: Vec::new(),
-                                                                               update_fee: None,
-                                                                               commitment_signed: commitment_msg,
-                                                                       },
-                                                               });
-                                                       }
                                                }
                                        }
                                } else {
                                        for forward_info in pending_forwards.drain(..) {
                                                match forward_info {
                                                        HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
-                                                               prev_short_channel_id, prev_htlc_id, prev_funding_outpoint,
+                                                               prev_short_channel_id, prev_htlc_id, prev_funding_outpoint, prev_user_channel_id,
                                                                forward_info: PendingHTLCInfo {
                                                                        routing, incoming_shared_secret, payment_hash, outgoing_amt_msat, ..
                                                                }
@@ -3429,9 +3402,9 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
                                                                macro_rules! fail_htlc {
                                                                        ($htlc: expr, $payment_hash: expr) => {
-                                                                               let mut htlc_msat_height_data = byte_utils::be64_to_array($htlc.value).to_vec();
+                                                                               let mut htlc_msat_height_data = $htlc.value.to_be_bytes().to_vec();
                                                                                htlc_msat_height_data.extend_from_slice(
-                                                                                       &byte_utils::be32_to_array(self.best_block.read().unwrap().height()),
+                                                                                       &self.best_block.read().unwrap().height().to_be_bytes(),
                                                                                );
                                                                                failed_forwards.push((HTLCSource::PreviousHopData(HTLCPreviousHopData {
                                                                                                short_channel_id: $htlc.prev_hop.short_channel_id,
@@ -3440,22 +3413,33 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                                                incoming_packet_shared_secret: $htlc.prev_hop.incoming_packet_shared_secret,
                                                                                                phantom_shared_secret,
                                                                                        }), payment_hash,
-                                                                                       HTLCFailReason::Reason { failure_code: 0x4000 | 15, data: htlc_msat_height_data },
+                                                                                       HTLCFailReason::reason(0x4000 | 15, htlc_msat_height_data),
                                                                                        HTLCDestination::FailedPayment { payment_hash: $payment_hash },
                                                                                ));
                                                                        }
                                                                }
+                                                               let phantom_shared_secret = claimable_htlc.prev_hop.phantom_shared_secret;
+                                                               let mut receiver_node_id = self.our_network_pubkey;
+                                                               if phantom_shared_secret.is_some() {
+                                                                       receiver_node_id = self.keys_manager.get_node_id(Recipient::PhantomNode)
+                                                                               .expect("Failed to get node_id for phantom node recipient");
+                                                               }
 
                                                                macro_rules! check_total_value {
                                                                        ($payment_data: expr, $payment_preimage: expr) => {{
-                                                                               let mut payment_received_generated = false;
+                                                                               let mut payment_claimable_generated = false;
                                                                                let purpose = || {
                                                                                        events::PaymentPurpose::InvoicePayment {
                                                                                                payment_preimage: $payment_preimage,
                                                                                                payment_secret: $payment_data.payment_secret,
                                                                                        }
                                                                                };
-                                                                               let (_, htlcs) = channel_state.claimable_htlcs.entry(payment_hash)
+                                                                               let mut claimable_payments = self.claimable_payments.lock().unwrap();
+                                                                               if claimable_payments.pending_claiming_payments.contains_key(&payment_hash) {
+                                                                                       fail_htlc!(claimable_htlc, payment_hash);
+                                                                                       continue
+                                                                               }
+                                                                               let (_, htlcs) = claimable_payments.claimable_htlcs.entry(payment_hash)
                                                                                        .or_insert_with(|| (purpose(), Vec::new()));
                                                                                if htlcs.len() == 1 {
                                                                                        if let OnionPayload::Spontaneous(_) = htlcs[0].onion_payload {
@@ -3484,20 +3468,24 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                                                log_bytes!(payment_hash.0), total_value, $payment_data.total_msat);
                                                                                        fail_htlc!(claimable_htlc, payment_hash);
                                                                                } else if total_value == $payment_data.total_msat {
+                                                                                       let prev_channel_id = prev_funding_outpoint.to_channel_id();
                                                                                        htlcs.push(claimable_htlc);
-                                                                                       new_events.push(events::Event::PaymentReceived {
+                                                                                       new_events.push(events::Event::PaymentClaimable {
+                                                                                               receiver_node_id: Some(receiver_node_id),
                                                                                                payment_hash,
                                                                                                purpose: purpose(),
                                                                                                amount_msat: total_value,
+                                                                                               via_channel_id: Some(prev_channel_id),
+                                                                                               via_user_channel_id: Some(prev_user_channel_id),
                                                                                        });
-                                                                                       payment_received_generated = true;
+                                                                                       payment_claimable_generated = true;
                                                                                } else {
                                                                                        // Nothing to do - we haven't reached the total
                                                                                        // payment value yet, wait until we receive more
                                                                                        // MPP parts.
                                                                                        htlcs.push(claimable_htlc);
                                                                                }
-                                                                               payment_received_generated
+                                                                               payment_claimable_generated
                                                                        }}
                                                                }
 
@@ -3523,14 +3511,23 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                                                check_total_value!(payment_data, payment_preimage);
                                                                                        },
                                                                                        OnionPayload::Spontaneous(preimage) => {
-                                                                                               match channel_state.claimable_htlcs.entry(payment_hash) {
+                                                                                               let mut claimable_payments = self.claimable_payments.lock().unwrap();
+                                                                                               if claimable_payments.pending_claiming_payments.contains_key(&payment_hash) {
+                                                                                                       fail_htlc!(claimable_htlc, payment_hash);
+                                                                                                       continue
+                                                                                               }
+                                                                                               match claimable_payments.claimable_htlcs.entry(payment_hash) {
                                                                                                        hash_map::Entry::Vacant(e) => {
                                                                                                                let purpose = events::PaymentPurpose::SpontaneousPayment(preimage);
                                                                                                                e.insert((purpose.clone(), vec![claimable_htlc]));
-                                                                                                               new_events.push(events::Event::PaymentReceived {
+                                                                                                               let prev_channel_id = prev_funding_outpoint.to_channel_id();
+                                                                                                               new_events.push(events::Event::PaymentClaimable {
+                                                                                                                       receiver_node_id: Some(receiver_node_id),
                                                                                                                        payment_hash,
                                                                                                                        amount_msat: outgoing_amt_msat,
                                                                                                                        purpose,
+                                                                                                                       via_channel_id: Some(prev_channel_id),
+                                                                                                                       via_user_channel_id: Some(prev_user_channel_id),
                                                                                                                });
                                                                                                        },
                                                                                                        hash_map::Entry::Occupied(_) => {
@@ -3556,8 +3553,8 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                                                log_bytes!(payment_hash.0), payment_data.total_msat, inbound_payment.get().min_value_msat.unwrap());
                                                                                        fail_htlc!(claimable_htlc, payment_hash);
                                                                                } else {
-                                                                                       let payment_received_generated = check_total_value!(payment_data, inbound_payment.get().payment_preimage);
-                                                                                       if payment_received_generated {
+                                                                                       let payment_claimable_generated = check_total_value!(payment_data, inbound_payment.get().payment_preimage);
+                                                                                       if payment_claimable_generated {
                                                                                                inbound_payment.remove_entry();
                                                                                        }
                                                                                }
@@ -3574,13 +3571,15 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                }
 
                for (htlc_source, payment_hash, failure_reason, destination) in failed_forwards.drain(..) {
-                       self.fail_htlc_backwards_internal(htlc_source, &payment_hash, failure_reason, destination);
+                       self.fail_htlc_backwards_internal(&htlc_source, &payment_hash, &failure_reason, destination);
                }
                self.forward_htlcs(&mut phantom_receives);
 
-               for (counterparty_node_id, err) in handle_errors.drain(..) {
-                       let _ = handle_error!(self, err, counterparty_node_id);
-               }
+               // Freeing the holding cell here is relatively redundant - in practice we'll do it when we
+               // next get a `get_and_clear_pending_msg_events` call, but some tests rely on it, and it's
+               // nice to do the work now if we can rather than while we're trying to get messages in the
+               // network stack.
+               self.check_free_holding_cells();
 
                if new_events.is_empty() { return }
                let mut events = self.pending_events.lock().unwrap();
@@ -3618,59 +3617,24 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                self.process_background_events();
        }
 
-       fn update_channel_fee(&self, pending_msg_events: &mut Vec<events::MessageSendEvent>, chan_id: &[u8; 32], chan: &mut Channel<<K::Target as KeysInterface>::Signer>, new_feerate: u32) -> (bool, NotifyOption, Result<(), MsgHandleErrInternal>) {
-               if !chan.is_outbound() { return (true, NotifyOption::SkipPersist, Ok(())); }
+       fn update_channel_fee(&self, chan_id: &[u8; 32], chan: &mut Channel<<K::Target as KeysInterface>::Signer>, new_feerate: u32) -> NotifyOption {
+               if !chan.is_outbound() { return NotifyOption::SkipPersist; }
                // If the feerate has decreased by less than half, don't bother
                if new_feerate <= chan.get_feerate() && new_feerate * 2 > chan.get_feerate() {
                        log_trace!(self.logger, "Channel {} does not qualify for a feerate change from {} to {}.",
                                log_bytes!(chan_id[..]), chan.get_feerate(), new_feerate);
-                       return (true, NotifyOption::SkipPersist, Ok(()));
+                       return NotifyOption::SkipPersist;
                }
                if !chan.is_live() {
                        log_trace!(self.logger, "Channel {} does not qualify for a feerate change from {} to {} as it cannot currently be updated (probably the peer is disconnected).",
                                log_bytes!(chan_id[..]), chan.get_feerate(), new_feerate);
-                       return (true, NotifyOption::SkipPersist, Ok(()));
+                       return NotifyOption::SkipPersist;
                }
                log_trace!(self.logger, "Channel {} qualifies for a feerate change from {} to {}.",
                        log_bytes!(chan_id[..]), chan.get_feerate(), new_feerate);
 
-               let mut retain_channel = true;
-               let res = match chan.send_update_fee_and_commit(new_feerate, &self.logger) {
-                       Ok(res) => Ok(res),
-                       Err(e) => {
-                               let (drop, res) = convert_chan_err!(self, e, chan, chan_id);
-                               if drop { retain_channel = false; }
-                               Err(res)
-                       }
-               };
-               let ret_err = match res {
-                       Ok(Some((update_fee, commitment_signed, monitor_update))) => {
-                               match self.chain_monitor.update_channel(chan.get_funding_txo().unwrap(), monitor_update) {
-                                       ChannelMonitorUpdateStatus::Completed => {
-                                               pending_msg_events.push(events::MessageSendEvent::UpdateHTLCs {
-                                                       node_id: chan.get_counterparty_node_id(),
-                                                       updates: msgs::CommitmentUpdate {
-                                                               update_add_htlcs: Vec::new(),
-                                                               update_fulfill_htlcs: Vec::new(),
-                                                               update_fail_htlcs: Vec::new(),
-                                                               update_fail_malformed_htlcs: Vec::new(),
-                                                               update_fee: Some(update_fee),
-                                                               commitment_signed,
-                                                       },
-                                               });
-                                               Ok(())
-                                       },
-                                       e => {
-                                               let (res, drop) = handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, chan_id, COMMITMENT_UPDATE_ONLY);
-                                               if drop { retain_channel = false; }
-                                               res
-                                       }
-                               }
-                       },
-                       Ok(None) => Ok(()),
-                       Err(e) => Err(e),
-               };
-               (retain_channel, NotifyOption::DoPersist, ret_err)
+               chan.queue_update_fee(new_feerate, &self.logger);
+               NotifyOption::DoPersist
        }
 
        #[cfg(fuzzing)]
@@ -3684,19 +3648,10 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
                        let new_feerate = self.fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::Normal);
 
-                       let mut handle_errors = Vec::new();
-                       {
-                               let mut channel_state_lock = self.channel_state.lock().unwrap();
-                               let channel_state = &mut *channel_state_lock;
-                               let pending_msg_events = &mut channel_state.pending_msg_events;
-                               channel_state.by_id.retain(|chan_id, chan| {
-                                       let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(pending_msg_events, chan_id, chan, new_feerate);
-                                       if chan_needs_persist == NotifyOption::DoPersist { should_persist = NotifyOption::DoPersist; }
-                                       if err.is_err() {
-                                               handle_errors.push(err);
-                                       }
-                                       retain_channel
-                               });
+                       let mut channel_state = self.channel_state.lock().unwrap();
+                       for (chan_id, chan) in channel_state.by_id.iter_mut() {
+                               let chan_needs_persist = self.update_channel_fee(chan_id, chan, new_feerate);
+                               if chan_needs_persist == NotifyOption::DoPersist { should_persist = NotifyOption::DoPersist; }
                        }
 
                        should_persist
@@ -3761,20 +3716,15 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
                        let new_feerate = self.fee_estimator.bounded_sat_per_1000_weight(ConfirmationTarget::Normal);
 
-                       let mut handle_errors = Vec::new();
+                       let mut handle_errors: Vec<(Result<(), _>, _)> = Vec::new();
                        let mut timed_out_mpp_htlcs = Vec::new();
                        {
                                let mut channel_state_lock = self.channel_state.lock().unwrap();
                                let channel_state = &mut *channel_state_lock;
                                let pending_msg_events = &mut channel_state.pending_msg_events;
                                channel_state.by_id.retain(|chan_id, chan| {
-                                       let counterparty_node_id = chan.get_counterparty_node_id();
-                                       let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(pending_msg_events, chan_id, chan, new_feerate);
+                                       let chan_needs_persist = self.update_channel_fee(chan_id, chan, new_feerate);
                                        if chan_needs_persist == NotifyOption::DoPersist { should_persist = NotifyOption::DoPersist; }
-                                       if err.is_err() {
-                                               handle_errors.push((err, counterparty_node_id));
-                                       }
-                                       if !retain_channel { return false; }
 
                                        if let Err(e) = chan.timer_check_closing_negotiation_progress() {
                                                let (needs_close, err) = convert_chan_err!(self, e, chan, chan_id);
@@ -3812,33 +3762,35 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
                                        true
                                });
+                       }
 
-                               channel_state.claimable_htlcs.retain(|payment_hash, (_, htlcs)| {
-                                       if htlcs.is_empty() {
-                                               // This should be unreachable
-                                               debug_assert!(false);
+                       self.claimable_payments.lock().unwrap().claimable_htlcs.retain(|payment_hash, (_, htlcs)| {
+                               if htlcs.is_empty() {
+                                       // This should be unreachable
+                                       debug_assert!(false);
+                                       return false;
+                               }
+                               if let OnionPayload::Invoice { .. } = htlcs[0].onion_payload {
+                                       // Check if we've received all the parts we need for an MPP (the value of the parts adds to total_msat).
+                                       // In this case we're not going to handle any timeouts of the parts here.
+                                       if htlcs[0].total_msat == htlcs.iter().fold(0, |total, htlc| total + htlc.value) {
+                                               return true;
+                                       } else if htlcs.into_iter().any(|htlc| {
+                                               htlc.timer_ticks += 1;
+                                               return htlc.timer_ticks >= MPP_TIMEOUT_TICKS
+                                       }) {
+                                               timed_out_mpp_htlcs.extend(htlcs.drain(..).map(|htlc: ClaimableHTLC| (htlc.prev_hop, *payment_hash)));
                                                return false;
                                        }
-                                       if let OnionPayload::Invoice { .. } = htlcs[0].onion_payload {
-                                               // Check if we've received all the parts we need for an MPP (the value of the parts adds to total_msat).
-                                               // In this case we're not going to handle any timeouts of the parts here.
-                                               if htlcs[0].total_msat == htlcs.iter().fold(0, |total, htlc| total + htlc.value) {
-                                                       return true;
-                                               } else if htlcs.into_iter().any(|htlc| {
-                                                       htlc.timer_ticks += 1;
-                                                       return htlc.timer_ticks >= MPP_TIMEOUT_TICKS
-                                               }) {
-                                                       timed_out_mpp_htlcs.extend(htlcs.into_iter().map(|htlc| (htlc.prev_hop.clone(), payment_hash.clone())));
-                                                       return false;
-                                               }
-                                       }
-                                       true
-                               });
-                       }
+                               }
+                               true
+                       });
 
                        for htlc_source in timed_out_mpp_htlcs.drain(..) {
+                               let source = HTLCSource::PreviousHopData(htlc_source.0.clone());
+                               let reason = HTLCFailReason::from_failure_code(23);
                                let receiver = HTLCDestination::FailedPayment { payment_hash: htlc_source.1 };
-                               self.fail_htlc_backwards_internal(HTLCSource::PreviousHopData(htlc_source.0.clone()), &htlc_source.1, HTLCFailReason::Reason { failure_code: 23, data: Vec::new() }, receiver );
+                               self.fail_htlc_backwards_internal(&source, &htlc_source.1, &reason, receiver);
                        }
 
                        for (err, counterparty_node_id) in handle_errors.drain(..) {
@@ -3847,17 +3799,24 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
                        self.remove_stale_resolved_payments();
 
+                       // Technically we don't need to do this here, but if we have holding cell entries in a
+                       // channel that need freeing, it's better to do that here and block a background task
+                       // than block the message queueing pipeline.
+                       if self.check_free_holding_cells() {
+                               should_persist = NotifyOption::DoPersist;
+                       }
+
                        should_persist
                });
        }
 
        /// Indicates that the preimage for payment_hash is unknown or the received amount is incorrect
-       /// after a PaymentReceived event, failing the HTLC back to its origin and freeing resources
+       /// after a PaymentClaimable event, failing the HTLC back to its origin and freeing resources
        /// along the path (including in our own channel on which we received it).
        ///
        /// Note that in some cases around unclean shutdown, it is possible the payment may have
        /// already been claimed by you via [`ChannelManager::claim_funds`] prior to you seeing (a
-       /// second copy of) the [`events::Event::PaymentReceived`] event. Alternatively, the payment
+       /// second copy of) the [`events::Event::PaymentClaimable`] event. Alternatively, the payment
        /// may have already been failed automatically by LDK if it was nearing its expiration time.
        ///
        /// While LDK will never claim a payment automatically on your behalf (i.e. without you calling
@@ -3867,19 +3826,15 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        pub fn fail_htlc_backwards(&self, payment_hash: &PaymentHash) {
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
 
-               let removed_source = {
-                       let mut channel_state = self.channel_state.lock().unwrap();
-                       channel_state.claimable_htlcs.remove(payment_hash)
-               };
+               let removed_source = self.claimable_payments.lock().unwrap().claimable_htlcs.remove(payment_hash);
                if let Some((_, mut sources)) = removed_source {
                        for htlc in sources.drain(..) {
-                               let mut htlc_msat_height_data = byte_utils::be64_to_array(htlc.value).to_vec();
-                               htlc_msat_height_data.extend_from_slice(&byte_utils::be32_to_array(
-                                               self.best_block.read().unwrap().height()));
-                               self.fail_htlc_backwards_internal(
-                                               HTLCSource::PreviousHopData(htlc.prev_hop), payment_hash,
-                                               HTLCFailReason::Reason { failure_code: 0x4000 | 15, data: htlc_msat_height_data },
-                                               HTLCDestination::FailedPayment { payment_hash: *payment_hash });
+                               let mut htlc_msat_height_data = htlc.value.to_be_bytes().to_vec();
+                               htlc_msat_height_data.extend_from_slice(&self.best_block.read().unwrap().height().to_be_bytes());
+                               let source = HTLCSource::PreviousHopData(htlc.prev_hop);
+                               let reason = HTLCFailReason::reason(0x4000 | 15, htlc_msat_height_data);
+                               let receiver = HTLCDestination::FailedPayment { payment_hash: *payment_hash };
+                               self.fail_htlc_backwards_internal(&source, &payment_hash, &reason, receiver);
                        }
                }
        }
@@ -3938,23 +3893,24 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                &self, mut htlcs_to_fail: Vec<(HTLCSource, PaymentHash)>, channel_id: [u8; 32],
                counterparty_node_id: &PublicKey
        ) {
-               for (htlc_src, payment_hash) in htlcs_to_fail.drain(..) {
-                       let (failure_code, onion_failure_data) =
-                               match self.channel_state.lock().unwrap().by_id.entry(channel_id) {
-                                       hash_map::Entry::Occupied(chan_entry) => {
-                                               self.get_htlc_inbound_temp_fail_err_and_data(0x1000|7, &chan_entry.get())
-                                       },
-                                       hash_map::Entry::Vacant(_) => (0x4000|10, Vec::new())
-                               };
+               let (failure_code, onion_failure_data) =
+                       match self.channel_state.lock().unwrap().by_id.entry(channel_id) {
+                               hash_map::Entry::Occupied(chan_entry) => {
+                                       self.get_htlc_inbound_temp_fail_err_and_data(0x1000|7, &chan_entry.get())
+                               },
+                               hash_map::Entry::Vacant(_) => (0x4000|10, Vec::new())
+                       };
 
+               for (htlc_src, payment_hash) in htlcs_to_fail.drain(..) {
+                       let reason = HTLCFailReason::reason(failure_code, onion_failure_data.clone());
                        let receiver = HTLCDestination::NextHopChannel { node_id: Some(counterparty_node_id.clone()), channel_id };
-                       self.fail_htlc_backwards_internal(htlc_src, &payment_hash, HTLCFailReason::Reason { failure_code, data: onion_failure_data }, receiver);
+                       self.fail_htlc_backwards_internal(&htlc_src, &payment_hash, &reason, receiver);
                }
        }
 
        /// Fails an HTLC backwards to the sender of it to us.
        /// Note that we do not assume that channels corresponding to failed HTLCs are still available.
-       fn fail_htlc_backwards_internal(&self, source: HTLCSource, payment_hash: &PaymentHash, onion_error: HTLCFailReason,destination: HTLCDestination) {
+       fn fail_htlc_backwards_internal(&self, source: &HTLCSource, payment_hash: &PaymentHash, onion_error: &HTLCFailReason, destination: HTLCDestination) {
                #[cfg(debug_assertions)]
                {
                        // Ensure that the `channel_state` lock is not held when calling this function.
@@ -3973,13 +3929,13 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                // from block_connected which may run during initialization prior to the chain_monitor
                // being fully configured. See the docs for `ChannelManagerReadArgs` for more.
                match source {
-                       HTLCSource::OutboundRoute { ref path, session_priv, payment_id, ref payment_params, .. } => {
+                       HTLCSource::OutboundRoute { ref path, ref session_priv, ref payment_id, ref payment_params, .. } => {
                                let mut session_priv_bytes = [0; 32];
                                session_priv_bytes.copy_from_slice(&session_priv[..]);
                                let mut outbounds = self.pending_outbound_payments.lock().unwrap();
                                let mut all_paths_failed = false;
                                let mut full_failure_ev = None;
-                               if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(payment_id) {
+                               if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(*payment_id) {
                                        if !payment.get_mut().remove(&session_priv_bytes, Some(&path)) {
                                                log_trace!(self.logger, "Received duplicative fail for HTLC with payment_hash {}", log_bytes!(payment_hash.0));
                                                return;
@@ -3992,7 +3948,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                all_paths_failed = true;
                                                if payment.get().abandoned() {
                                                        full_failure_ev = Some(events::Event::PaymentFailed {
-                                                               payment_id,
+                                                               payment_id: *payment_id,
                                                                payment_hash: payment.get().payment_hash().expect("PendingOutboundPayments::RetriesExceeded always has a payment hash set"),
                                                        });
                                                        payment.remove();
@@ -4012,126 +3968,69 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                } else { None };
                                log_trace!(self.logger, "Failing outbound payment HTLC with payment_hash {}", log_bytes!(payment_hash.0));
 
-                               let path_failure = match &onion_error {
-                                       &HTLCFailReason::LightningError { ref err } => {
+                               let path_failure = {
 #[cfg(test)]
-                                               let (network_update, short_channel_id, payment_retryable, onion_error_code, onion_error_data) = onion_utils::process_onion_failure(&self.secp_ctx, &self.logger, &source, err.data.clone());
+                                       let (network_update, short_channel_id, payment_retryable, onion_error_code, onion_error_data) = onion_error.decode_onion_failure(&self.secp_ctx, &self.logger, &source);
 #[cfg(not(test))]
-                                               let (network_update, short_channel_id, payment_retryable, _, _) = onion_utils::process_onion_failure(&self.secp_ctx, &self.logger, &source, err.data.clone());
+                                       let (network_update, short_channel_id, payment_retryable, _, _) = onion_error.decode_onion_failure(&self.secp_ctx, &self.logger, &source);
 
-                                               if self.payment_is_probe(payment_hash, &payment_id) {
-                                                       if !payment_retryable {
-                                                               events::Event::ProbeSuccessful {
-                                                                       payment_id,
-                                                                       payment_hash: payment_hash.clone(),
-                                                                       path: path.clone(),
-                                                               }
-                                                       } else {
-                                                               events::Event::ProbeFailed {
-                                                                       payment_id,
-                                                                       payment_hash: payment_hash.clone(),
-                                                                       path: path.clone(),
-                                                                       short_channel_id,
-                                                               }
-                                                       }
-                                               } else {
-                                                       // TODO: If we decided to blame ourselves (or one of our channels) in
-                                                       // process_onion_failure we should close that channel as it implies our
-                                                       // next-hop is needlessly blaming us!
-                                                       if let Some(scid) = short_channel_id {
-                                                               retry.as_mut().map(|r| r.payment_params.previously_failed_channels.push(scid));
-                                                       }
-                                                       events::Event::PaymentPathFailed {
-                                                               payment_id: Some(payment_id),
+                                       if self.payment_is_probe(payment_hash, &payment_id) {
+                                               if !payment_retryable {
+                                                       events::Event::ProbeSuccessful {
+                                                               payment_id: *payment_id,
                                                                payment_hash: payment_hash.clone(),
-                                                               payment_failed_permanently: !payment_retryable,
-                                                               network_update,
-                                                               all_paths_failed,
                                                                path: path.clone(),
-                                                               short_channel_id,
-                                                               retry,
-                                                               #[cfg(test)]
-                                                               error_code: onion_error_code,
-                                                               #[cfg(test)]
-                                                               error_data: onion_error_data
-                                                       }
-                                               }
-                                       },
-                                       &HTLCFailReason::Reason {
-#[cfg(test)]
-                                                       ref failure_code,
-#[cfg(test)]
-                                                       ref data,
-                                                       .. } => {
-                                               // we get a fail_malformed_htlc from the first hop
-                                               // TODO: We'd like to generate a NetworkUpdate for temporary
-                                               // failures here, but that would be insufficient as find_route
-                                               // generally ignores its view of our own channels as we provide them via
-                                               // ChannelDetails.
-                                               // TODO: For non-temporary failures, we really should be closing the
-                                               // channel here as we apparently can't relay through them anyway.
-                                               let scid = path.first().unwrap().short_channel_id;
-                                               retry.as_mut().map(|r| r.payment_params.previously_failed_channels.push(scid));
-
-                                               if self.payment_is_probe(payment_hash, &payment_id) {
-                                                       events::Event::ProbeFailed {
-                                                               payment_id,
-                                                               payment_hash: payment_hash.clone(),
-                                                               path: path.clone(),
-                                                               short_channel_id: Some(scid),
                                                        }
                                                } else {
-                                                       events::Event::PaymentPathFailed {
-                                                               payment_id: Some(payment_id),
+                                                       events::Event::ProbeFailed {
+                                                               payment_id: *payment_id,
                                                                payment_hash: payment_hash.clone(),
-                                                               payment_failed_permanently: false,
-                                                               network_update: None,
-                                                               all_paths_failed,
                                                                path: path.clone(),
-                                                               short_channel_id: Some(scid),
-                                                               retry,
-#[cfg(test)]
-                                                               error_code: Some(*failure_code),
-#[cfg(test)]
-                                                               error_data: Some(data.clone()),
+                                                               short_channel_id,
                                                        }
                                                }
+                                       } else {
+                                               // TODO: If we decided to blame ourselves (or one of our channels) in
+                                               // process_onion_failure we should close that channel as it implies our
+                                               // next-hop is needlessly blaming us!
+                                               if let Some(scid) = short_channel_id {
+                                                       retry.as_mut().map(|r| r.payment_params.previously_failed_channels.push(scid));
+                                               }
+                                               events::Event::PaymentPathFailed {
+                                                       payment_id: Some(*payment_id),
+                                                       payment_hash: payment_hash.clone(),
+                                                       payment_failed_permanently: !payment_retryable,
+                                                       network_update,
+                                                       all_paths_failed,
+                                                       path: path.clone(),
+                                                       short_channel_id,
+                                                       retry,
+                                                       #[cfg(test)]
+                                                       error_code: onion_error_code,
+                                                       #[cfg(test)]
+                                                       error_data: onion_error_data
+                                               }
                                        }
                                };
                                let mut pending_events = self.pending_events.lock().unwrap();
                                pending_events.push(path_failure);
                                if let Some(ev) = full_failure_ev { pending_events.push(ev); }
                        },
-                       HTLCSource::PreviousHopData(HTLCPreviousHopData { short_channel_id, htlc_id, incoming_packet_shared_secret, phantom_shared_secret, outpoint }) => {
-                               let err_packet = match onion_error {
-                                       HTLCFailReason::Reason { failure_code, data } => {
-                                               log_trace!(self.logger, "Failing HTLC with payment_hash {} backwards from us with code {}", log_bytes!(payment_hash.0), failure_code);
-                                               if let Some(phantom_ss) = phantom_shared_secret {
-                                                       let phantom_packet = onion_utils::build_failure_packet(&phantom_ss, failure_code, &data[..]).encode();
-                                                       let encrypted_phantom_packet = onion_utils::encrypt_failure_packet(&phantom_ss, &phantom_packet);
-                                                       onion_utils::encrypt_failure_packet(&incoming_packet_shared_secret, &encrypted_phantom_packet.data[..])
-                                               } else {
-                                                       let packet = onion_utils::build_failure_packet(&incoming_packet_shared_secret, failure_code, &data[..]).encode();
-                                                       onion_utils::encrypt_failure_packet(&incoming_packet_shared_secret, &packet)
-                                               }
-                                       },
-                                       HTLCFailReason::LightningError { err } => {
-                                               log_trace!(self.logger, "Failing HTLC with payment_hash {} backwards with pre-built LightningError", log_bytes!(payment_hash.0));
-                                               onion_utils::encrypt_failure_packet(&incoming_packet_shared_secret, &err.data)
-                                       }
-                               };
+                       HTLCSource::PreviousHopData(HTLCPreviousHopData { ref short_channel_id, ref htlc_id, ref incoming_packet_shared_secret, ref phantom_shared_secret, ref outpoint }) => {
+                               log_trace!(self.logger, "Failing HTLC with payment_hash {} backwards from us with {:?}", log_bytes!(payment_hash.0), onion_error);
+                               let err_packet = onion_error.get_encrypted_failure_packet(incoming_packet_shared_secret, phantom_shared_secret);
 
                                let mut forward_event = None;
                                let mut forward_htlcs = self.forward_htlcs.lock().unwrap();
                                if forward_htlcs.is_empty() {
                                        forward_event = Some(Duration::from_millis(MIN_HTLC_RELAY_HOLDING_CELL_MILLIS));
                                }
-                               match forward_htlcs.entry(short_channel_id) {
+                               match forward_htlcs.entry(*short_channel_id) {
                                        hash_map::Entry::Occupied(mut entry) => {
-                                               entry.get_mut().push(HTLCForwardInfo::FailHTLC { htlc_id, err_packet });
+                                               entry.get_mut().push(HTLCForwardInfo::FailHTLC { htlc_id: *htlc_id, err_packet });
                                        },
                                        hash_map::Entry::Vacant(entry) => {
-                                               entry.insert(vec!(HTLCForwardInfo::FailHTLC { htlc_id, err_packet }));
+                                               entry.insert(vec!(HTLCForwardInfo::FailHTLC { htlc_id: *htlc_id, err_packet }));
                                        }
                                }
                                mem::drop(forward_htlcs);
@@ -4143,13 +4042,13 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                }
                                pending_events.push(events::Event::HTLCHandlingFailed {
                                        prev_channel_id: outpoint.to_channel_id(),
-                                       failed_next_destination: destination
+                                       failed_next_destination: destination,
                                });
                        },
                }
        }
 
-       /// Provides a payment preimage in response to [`Event::PaymentReceived`], generating any
+       /// Provides a payment preimage in response to [`Event::PaymentClaimable`], generating any
        /// [`MessageSendEvent`]s needed to claim the payment.
        ///
        /// Note that calling this method does *not* guarantee that the payment has been claimed. You
@@ -4157,148 +4056,159 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        /// provided to your [`EventHandler`] when [`process_pending_events`] is next called.
        ///
        /// Note that if you did not set an `amount_msat` when calling [`create_inbound_payment`] or
-       /// [`create_inbound_payment_for_hash`] you must check that the amount in the `PaymentReceived`
+       /// [`create_inbound_payment_for_hash`] you must check that the amount in the `PaymentClaimable`
        /// event matches your expectation. If you fail to do so and call this method, you may provide
        /// the sender "proof-of-payment" when they did not fulfill the full expected payment.
        ///
-       /// [`Event::PaymentReceived`]: crate::util::events::Event::PaymentReceived
+       /// [`Event::PaymentClaimable`]: crate::util::events::Event::PaymentClaimable
        /// [`Event::PaymentClaimed`]: crate::util::events::Event::PaymentClaimed
        /// [`process_pending_events`]: EventsProvider::process_pending_events
        /// [`create_inbound_payment`]: Self::create_inbound_payment
        /// [`create_inbound_payment_for_hash`]: Self::create_inbound_payment_for_hash
-       /// [`get_and_clear_pending_msg_events`]: MessageSendEventsProvider::get_and_clear_pending_msg_events
        pub fn claim_funds(&self, payment_preimage: PaymentPreimage) {
                let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0).into_inner());
 
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
 
-               let removed_source = self.channel_state.lock().unwrap().claimable_htlcs.remove(&payment_hash);
-               if let Some((payment_purpose, mut sources)) = removed_source {
-                       assert!(!sources.is_empty());
-
-                       // If we are claiming an MPP payment, we have to take special care to ensure that each
-                       // channel exists before claiming all of the payments (inside one lock).
-                       // Note that channel existance is sufficient as we should always get a monitor update
-                       // which will take care of the real HTLC claim enforcement.
-                       //
-                       // If we find an HTLC which we would need to claim but for which we do not have a
-                       // channel, we will fail all parts of the MPP payment. While we could wait and see if
-                       // the sender retries the already-failed path(s), it should be a pretty rare case where
-                       // we got all the HTLCs and then a channel closed while we were waiting for the user to
-                       // provide the preimage, so worrying too much about the optimal handling isn't worth
-                       // it.
-                       let mut claimable_amt_msat = 0;
-                       let mut expected_amt_msat = None;
-                       let mut valid_mpp = true;
-                       let mut errs = Vec::new();
-                       let mut claimed_any_htlcs = false;
-                       let mut channel_state_lock = self.channel_state.lock().unwrap();
-                       let channel_state = &mut *channel_state_lock;
-                       for htlc in sources.iter() {
-                               let chan_id = match self.short_to_chan_info.read().unwrap().get(&htlc.prev_hop.short_channel_id) {
-                                       Some((_cp_id, chan_id)) => chan_id.clone(),
-                                       None => {
-                                               valid_mpp = false;
+               let mut sources = {
+                       let mut claimable_payments = self.claimable_payments.lock().unwrap();
+                       if let Some((payment_purpose, sources)) = claimable_payments.claimable_htlcs.remove(&payment_hash) {
+                               let mut receiver_node_id = self.our_network_pubkey;
+                               for htlc in sources.iter() {
+                                       if htlc.prev_hop.phantom_shared_secret.is_some() {
+                                               let phantom_pubkey = self.keys_manager.get_node_id(Recipient::PhantomNode)
+                                                       .expect("Failed to get node_id for phantom node recipient");
+                                               receiver_node_id = phantom_pubkey;
                                                break;
                                        }
-                               };
+                               }
 
-                               if let None = channel_state.by_id.get(&chan_id) {
-                                       valid_mpp = false;
-                                       break;
+                               let dup_purpose = claimable_payments.pending_claiming_payments.insert(payment_hash,
+                                       ClaimingPayment { amount_msat: sources.iter().map(|source| source.value).sum(),
+                                       payment_purpose, receiver_node_id,
+                               });
+                               if dup_purpose.is_some() {
+                                       debug_assert!(false, "Shouldn't get a duplicate pending claim event ever");
+                                       log_error!(self.logger, "Got a duplicate pending claimable event on payment hash {}! Please report this bug",
+                                               log_bytes!(payment_hash.0));
                                }
+                               sources
+                       } else { return; }
+               };
+               debug_assert!(!sources.is_empty());
 
-                               if expected_amt_msat.is_some() && expected_amt_msat != Some(htlc.total_msat) {
-                                       log_error!(self.logger, "Somehow ended up with an MPP payment with different total amounts - this should not be reachable!");
-                                       debug_assert!(false);
+               // If we are claiming an MPP payment, we check that all channels which contain a claimable
+               // HTLC still exist. While this isn't guaranteed to remain true if a channel closes while
+               // we're claiming (or even after we claim, before the commitment update dance completes),
+               // it should be a relatively rare race, and we'd rather not claim HTLCs that require us to
+               // go on-chain (and lose the on-chain fee to do so) than just reject the payment.
+               //
+               // Note that we'll still always get our funds - as long as the generated
+               // `ChannelMonitorUpdate` makes it out to the relevant monitor we can claim on-chain.
+               //
+               // If we find an HTLC which we would need to claim but for which we do not have a
+               // channel, we will fail all parts of the MPP payment. While we could wait and see if
+               // the sender retries the already-failed path(s), it should be a pretty rare case where
+               // we got all the HTLCs and then a channel closed while we were waiting for the user to
+               // provide the preimage, so worrying too much about the optimal handling isn't worth
+               // it.
+               let mut claimable_amt_msat = 0;
+               let mut expected_amt_msat = None;
+               let mut valid_mpp = true;
+               let mut errs = Vec::new();
+               let mut channel_state = Some(self.channel_state.lock().unwrap());
+               for htlc in sources.iter() {
+                       let chan_id = match self.short_to_chan_info.read().unwrap().get(&htlc.prev_hop.short_channel_id) {
+                               Some((_cp_id, chan_id)) => chan_id.clone(),
+                               None => {
                                        valid_mpp = false;
                                        break;
                                }
-                               expected_amt_msat = Some(htlc.total_msat);
-                               if let OnionPayload::Spontaneous(_) = &htlc.onion_payload {
-                                       // We don't currently support MPP for spontaneous payments, so just check
-                                       // that there's one payment here and move on.
-                                       if sources.len() != 1 {
-                                               log_error!(self.logger, "Somehow ended up with an MPP spontaneous payment - this should not be reachable!");
-                                               debug_assert!(false);
-                                               valid_mpp = false;
-                                               break;
-                                       }
-                               }
+                       };
 
-                               claimable_amt_msat += htlc.value;
-                       }
-                       if sources.is_empty() || expected_amt_msat.is_none() {
-                               log_info!(self.logger, "Attempted to claim an incomplete payment which no longer had any available HTLCs!");
-                               return;
+                       if let None = channel_state.as_ref().unwrap().by_id.get(&chan_id) {
+                               valid_mpp = false;
+                               break;
                        }
-                       if claimable_amt_msat != expected_amt_msat.unwrap() {
-                               log_info!(self.logger, "Attempted to claim an incomplete payment, expected {} msat, had {} available to claim.",
-                                       expected_amt_msat.unwrap(), claimable_amt_msat);
-                               return;
+
+                       if expected_amt_msat.is_some() && expected_amt_msat != Some(htlc.total_msat) {
+                               log_error!(self.logger, "Somehow ended up with an MPP payment with different total amounts - this should not be reachable!");
+                               debug_assert!(false);
+                               valid_mpp = false;
+                               break;
                        }
-                       if valid_mpp {
-                               for htlc in sources.drain(..) {
-                                       match self.claim_funds_from_hop(&mut channel_state_lock, htlc.prev_hop, payment_preimage) {
-                                               ClaimFundsFromHop::MonitorUpdateFail(pk, err, _) => {
-                                                       if let msgs::ErrorAction::IgnoreError = err.err.action {
-                                                               // We got a temporary failure updating monitor, but will claim the
-                                                               // HTLC when the monitor updating is restored (or on chain).
-                                                               log_error!(self.logger, "Temporary failure claiming HTLC, treating as success: {}", err.err.err);
-                                                               claimed_any_htlcs = true;
-                                                       } else { errs.push((pk, err)); }
-                                               },
-                                               ClaimFundsFromHop::PrevHopForceClosed => unreachable!("We already checked for channel existence, we can't fail here!"),
-                                               ClaimFundsFromHop::DuplicateClaim => {
-                                                       // While we should never get here in most cases, if we do, it likely
-                                                       // indicates that the HTLC was timed out some time ago and is no longer
-                                                       // available to be claimed. Thus, it does not make sense to set
-                                                       // `claimed_any_htlcs`.
-                                               },
-                                               ClaimFundsFromHop::Success(_) => claimed_any_htlcs = true,
-                                       }
+                       expected_amt_msat = Some(htlc.total_msat);
+                       if let OnionPayload::Spontaneous(_) = &htlc.onion_payload {
+                               // We don't currently support MPP for spontaneous payments, so just check
+                               // that there's one payment here and move on.
+                               if sources.len() != 1 {
+                                       log_error!(self.logger, "Somehow ended up with an MPP spontaneous payment - this should not be reachable!");
+                                       debug_assert!(false);
+                                       valid_mpp = false;
+                                       break;
                                }
                        }
-                       mem::drop(channel_state_lock);
-                       if !valid_mpp {
-                               for htlc in sources.drain(..) {
-                                       let mut htlc_msat_height_data = byte_utils::be64_to_array(htlc.value).to_vec();
-                                       htlc_msat_height_data.extend_from_slice(&byte_utils::be32_to_array(
-                                               self.best_block.read().unwrap().height()));
-                                       self.fail_htlc_backwards_internal(
-                                               HTLCSource::PreviousHopData(htlc.prev_hop), &payment_hash,
-                                               HTLCFailReason::Reason { failure_code: 0x4000|15, data: htlc_msat_height_data },
-                                               HTLCDestination::FailedPayment { payment_hash } );
+
+                       claimable_amt_msat += htlc.value;
+               }
+               if sources.is_empty() || expected_amt_msat.is_none() {
+                       mem::drop(channel_state);
+                       self.claimable_payments.lock().unwrap().pending_claiming_payments.remove(&payment_hash);
+                       log_info!(self.logger, "Attempted to claim an incomplete payment which no longer had any available HTLCs!");
+                       return;
+               }
+               if claimable_amt_msat != expected_amt_msat.unwrap() {
+                       mem::drop(channel_state);
+                       self.claimable_payments.lock().unwrap().pending_claiming_payments.remove(&payment_hash);
+                       log_info!(self.logger, "Attempted to claim an incomplete payment, expected {} msat, had {} available to claim.",
+                               expected_amt_msat.unwrap(), claimable_amt_msat);
+                       return;
+               }
+               if valid_mpp {
+                       for htlc in sources.drain(..) {
+                               if channel_state.is_none() { channel_state = Some(self.channel_state.lock().unwrap()); }
+                               if let Err((pk, err)) = self.claim_funds_from_hop(channel_state.take().unwrap(), htlc.prev_hop,
+                                       payment_preimage,
+                                       |_| Some(MonitorUpdateCompletionAction::PaymentClaimed { payment_hash }))
+                               {
+                                       if let msgs::ErrorAction::IgnoreError = err.err.action {
+                                               // We got a temporary failure updating monitor, but will claim the
+                                               // HTLC when the monitor updating is restored (or on chain).
+                                               log_error!(self.logger, "Temporary failure claiming HTLC, treating as success: {}", err.err.err);
+                                       } else { errs.push((pk, err)); }
                                }
                        }
-
-                       if claimed_any_htlcs {
-                               self.pending_events.lock().unwrap().push(events::Event::PaymentClaimed {
-                                       payment_hash,
-                                       purpose: payment_purpose,
-                                       amount_msat: claimable_amt_msat,
-                               });
+               }
+               mem::drop(channel_state);
+               if !valid_mpp {
+                       for htlc in sources.drain(..) {
+                               let mut htlc_msat_height_data = htlc.value.to_be_bytes().to_vec();
+                               htlc_msat_height_data.extend_from_slice(&self.best_block.read().unwrap().height().to_be_bytes());
+                               let source = HTLCSource::PreviousHopData(htlc.prev_hop);
+                               let reason = HTLCFailReason::reason(0x4000 | 15, htlc_msat_height_data);
+                               let receiver = HTLCDestination::FailedPayment { payment_hash };
+                               self.fail_htlc_backwards_internal(&source, &payment_hash, &reason, receiver);
                        }
+                       self.claimable_payments.lock().unwrap().pending_claiming_payments.remove(&payment_hash);
+               }
 
-                       // Now we can handle any errors which were generated.
-                       for (counterparty_node_id, err) in errs.drain(..) {
-                               let res: Result<(), _> = Err(err);
-                               let _ = handle_error!(self, res, counterparty_node_id);
-                       }
+               // Now we can handle any errors which were generated.
+               for (counterparty_node_id, err) in errs.drain(..) {
+                       let res: Result<(), _> = Err(err);
+                       let _ = handle_error!(self, res, counterparty_node_id);
                }
        }
 
-       fn claim_funds_from_hop(&self, channel_state_lock: &mut MutexGuard<ChannelHolder<<K::Target as KeysInterface>::Signer>>, prev_hop: HTLCPreviousHopData, payment_preimage: PaymentPreimage) -> ClaimFundsFromHop {
+       fn claim_funds_from_hop<ComplFunc: FnOnce(Option<u64>) -> Option<MonitorUpdateCompletionAction>>(&self,
+               mut channel_state_lock: MutexGuard<ChannelHolder<<K::Target as KeysInterface>::Signer>>,
+               prev_hop: HTLCPreviousHopData, payment_preimage: PaymentPreimage, completion_action: ComplFunc)
+       -> Result<(), (PublicKey, MsgHandleErrInternal)> {
                //TODO: Delay the claimed_funds relaying just like we do outbound relay!
-               let channel_state = &mut **channel_state_lock;
-               let chan_id = match self.short_to_chan_info.read().unwrap().get(&prev_hop.short_channel_id) {
-                       Some((_cp_id, chan_id)) => chan_id.clone(),
-                       None => {
-                               return ClaimFundsFromHop::PrevHopForceClosed
-                       }
-               };
 
+               let chan_id = prev_hop.outpoint.to_channel_id();
+               let channel_state = &mut *channel_state_lock;
                if let hash_map::Entry::Occupied(mut chan) = channel_state.by_id.entry(chan_id) {
+                       let counterparty_node_id = chan.get().get_counterparty_node_id();
                        match chan.get_mut().get_update_fulfill_htlc_and_commit(prev_hop.htlc_id, payment_preimage, &self.logger) {
                                Ok(msgs_monitor_option) => {
                                        if let UpdateFulfillCommitFetch::NewClaim { msgs, htlc_value_msat, monitor_update } = msgs_monitor_option {
@@ -4308,11 +4218,10 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                log_given_level!(self.logger, if e == ChannelMonitorUpdateStatus::PermanentFailure { Level::Error } else { Level::Debug },
                                                                        "Failed to update channel monitor with preimage {:?}: {:?}",
                                                                        payment_preimage, e);
-                                                               return ClaimFundsFromHop::MonitorUpdateFail(
-                                                                       chan.get().get_counterparty_node_id(),
-                                                                       handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, false, msgs.is_some()).unwrap_err(),
-                                                                       Some(htlc_value_msat)
-                                                               );
+                                                               let err = handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, false, msgs.is_some()).unwrap_err();
+                                                               mem::drop(channel_state_lock);
+                                                               self.handle_monitor_update_completion_actions(completion_action(Some(htlc_value_msat)));
+                                                               return Err((counterparty_node_id, err));
                                                        }
                                                }
                                                if let Some((msg, commitment_signed)) = msgs {
@@ -4330,29 +4239,62 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                }
                                                        });
                                                }
-                                               return ClaimFundsFromHop::Success(htlc_value_msat);
+                                               mem::drop(channel_state_lock);
+                                               self.handle_monitor_update_completion_actions(completion_action(Some(htlc_value_msat)));
+                                               Ok(())
                                        } else {
-                                               return ClaimFundsFromHop::DuplicateClaim;
+                                               Ok(())
                                        }
                                },
                                Err((e, monitor_update)) => {
                                        match self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update) {
                                                ChannelMonitorUpdateStatus::Completed => {},
                                                e => {
+                                                       // TODO: This needs to be handled somehow - if we receive a monitor update
+                                                       // with a preimage we *must* somehow manage to propagate it to the upstream
+                                                       // channel, or we must have an ability to receive the same update and try
+                                                       // again on restart.
                                                        log_given_level!(self.logger, if e == ChannelMonitorUpdateStatus::PermanentFailure { Level::Error } else { Level::Info },
                                                                "Failed to update channel monitor with preimage {:?} immediately prior to force-close: {:?}",
                                                                payment_preimage, e);
                                                },
                                        }
-                                       let counterparty_node_id = chan.get().get_counterparty_node_id();
                                        let (drop, res) = convert_chan_err!(self, e, chan.get_mut(), &chan_id);
                                        if drop {
                                                chan.remove_entry();
                                        }
-                                       return ClaimFundsFromHop::MonitorUpdateFail(counterparty_node_id, res, None);
+                                       mem::drop(channel_state_lock);
+                                       self.handle_monitor_update_completion_actions(completion_action(None));
+                                       Err((counterparty_node_id, res))
                                },
                        }
-               } else { return ClaimFundsFromHop::PrevHopForceClosed }
+               } else {
+                       let preimage_update = ChannelMonitorUpdate {
+                               update_id: CLOSED_CHANNEL_UPDATE_ID,
+                               updates: vec![ChannelMonitorUpdateStep::PaymentPreimage {
+                                       payment_preimage,
+                               }],
+                       };
+                       // We update the ChannelMonitor on the backward link, after
+                       // receiving an `update_fulfill_htlc` from the forward link.
+                       let update_res = self.chain_monitor.update_channel(prev_hop.outpoint, preimage_update);
+                       if update_res != ChannelMonitorUpdateStatus::Completed {
+                               // TODO: This needs to be handled somehow - if we receive a monitor update
+                               // with a preimage we *must* somehow manage to propagate it to the upstream
+                               // channel, or we must have an ability to receive the same event and try
+                               // again on restart.
+                               log_error!(self.logger, "Critical error: failed to update channel monitor with preimage {:?}: {:?}",
+                                       payment_preimage, update_res);
+                       }
+                       mem::drop(channel_state_lock);
+                       // Note that we do process the completion action here. This totally could be a
+                       // duplicate claim, but we have no way of knowing without interrogating the
+                       // `ChannelMonitor` we've provided the above update to. Instead, note that `Event`s are
+                       // generally always allowed to be duplicative (and it's specifically noted in
+                       // `PaymentForwarded`).
+                       self.handle_monitor_update_completion_actions(completion_action(None));
+                       Ok(())
+               }
        }
 
        fn finalize_claims(&self, mut sources: Vec<HTLCSource>) {
@@ -4378,7 +4320,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                }
        }
 
-       fn claim_funds_internal(&self, mut channel_state_lock: MutexGuard<ChannelHolder<<K::Target as KeysInterface>::Signer>>, source: HTLCSource, payment_preimage: PaymentPreimage, forwarded_htlc_value_msat: Option<u64>, from_onchain: bool, next_channel_id: [u8; 32]) {
+       fn claim_funds_internal(&self, channel_state_lock: MutexGuard<ChannelHolder<<K::Target as KeysInterface>::Signer>>, source: HTLCSource, payment_preimage: PaymentPreimage, forwarded_htlc_value_msat: Option<u64>, from_onchain: bool, next_channel_id: [u8; 32]) {
                match source {
                        HTLCSource::OutboundRoute { session_priv, payment_id, path, .. } => {
                                mem::drop(channel_state_lock);
@@ -4425,62 +4367,28 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                        },
                        HTLCSource::PreviousHopData(hop_data) => {
                                let prev_outpoint = hop_data.outpoint;
-                               let res = self.claim_funds_from_hop(&mut channel_state_lock, hop_data, payment_preimage);
-                               let claimed_htlc = if let ClaimFundsFromHop::DuplicateClaim = res { false } else { true };
-                               let htlc_claim_value_msat = match res {
-                                       ClaimFundsFromHop::MonitorUpdateFail(_, _, amt_opt) => amt_opt,
-                                       ClaimFundsFromHop::Success(amt) => Some(amt),
-                                       _ => None,
-                               };
-                               if let ClaimFundsFromHop::PrevHopForceClosed = res {
-                                       let preimage_update = ChannelMonitorUpdate {
-                                               update_id: CLOSED_CHANNEL_UPDATE_ID,
-                                               updates: vec![ChannelMonitorUpdateStep::PaymentPreimage {
-                                                       payment_preimage: payment_preimage.clone(),
-                                               }],
-                                       };
-                                       // We update the ChannelMonitor on the backward link, after
-                                       // receiving an offchain preimage event from the forward link (the
-                                       // event being update_fulfill_htlc).
-                                       let update_res = self.chain_monitor.update_channel(prev_outpoint, preimage_update);
-                                       if update_res != ChannelMonitorUpdateStatus::Completed {
-                                               // TODO: This needs to be handled somehow - if we receive a monitor update
-                                               // with a preimage we *must* somehow manage to propagate it to the upstream
-                                               // channel, or we must have an ability to receive the same event and try
-                                               // again on restart.
-                                               log_error!(self.logger, "Critical error: failed to update channel monitor with preimage {:?}: {:?}",
-                                                       payment_preimage, update_res);
-                                       }
-                                       // Note that we do *not* set `claimed_htlc` to false here. In fact, this
-                                       // totally could be a duplicate claim, but we have no way of knowing
-                                       // without interrogating the `ChannelMonitor` we've provided the above
-                                       // update to. Instead, we simply document in `PaymentForwarded` that this
-                                       // can happen.
-                               }
-                               mem::drop(channel_state_lock);
-                               if let ClaimFundsFromHop::MonitorUpdateFail(pk, err, _) = res {
+                               let res = self.claim_funds_from_hop(channel_state_lock, hop_data, payment_preimage,
+                                       |htlc_claim_value_msat| {
+                                               if let Some(forwarded_htlc_value) = forwarded_htlc_value_msat {
+                                                       let fee_earned_msat = if let Some(claimed_htlc_value) = htlc_claim_value_msat {
+                                                               Some(claimed_htlc_value - forwarded_htlc_value)
+                                                       } else { None };
+
+                                                       let prev_channel_id = Some(prev_outpoint.to_channel_id());
+                                                       let next_channel_id = Some(next_channel_id);
+
+                                                       Some(MonitorUpdateCompletionAction::EmitEvent { event: events::Event::PaymentForwarded {
+                                                               fee_earned_msat,
+                                                               claim_from_onchain_tx: from_onchain,
+                                                               prev_channel_id,
+                                                               next_channel_id,
+                                                       }})
+                                               } else { None }
+                                       });
+                               if let Err((pk, err)) = res {
                                        let result: Result<(), _> = Err(err);
                                        let _ = handle_error!(self, result, pk);
                                }
-
-                               if claimed_htlc {
-                                       if let Some(forwarded_htlc_value) = forwarded_htlc_value_msat {
-                                               let fee_earned_msat = if let Some(claimed_htlc_value) = htlc_claim_value_msat {
-                                                       Some(claimed_htlc_value - forwarded_htlc_value)
-                                               } else { None };
-
-                                               let mut pending_events = self.pending_events.lock().unwrap();
-                                               let prev_channel_id = Some(prev_outpoint.to_channel_id());
-                                               let next_channel_id = Some(next_channel_id);
-
-                                               pending_events.push(events::Event::PaymentForwarded {
-                                                       fee_earned_msat,
-                                                       claim_from_onchain_tx: from_onchain,
-                                                       prev_channel_id,
-                                                       next_channel_id,
-                                               });
-                                       }
-                               }
                        },
                }
        }
@@ -4490,10 +4398,91 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                self.our_network_pubkey.clone()
        }
 
+       fn handle_monitor_update_completion_actions<I: IntoIterator<Item=MonitorUpdateCompletionAction>>(&self, actions: I) {
+               for action in actions.into_iter() {
+                       match action {
+                               MonitorUpdateCompletionAction::PaymentClaimed { payment_hash } => {
+                                       let payment = self.claimable_payments.lock().unwrap().pending_claiming_payments.remove(&payment_hash);
+                                       if let Some(ClaimingPayment { amount_msat, payment_purpose: purpose, receiver_node_id }) = payment {
+                                               self.pending_events.lock().unwrap().push(events::Event::PaymentClaimed {
+                                                       payment_hash, purpose, amount_msat, receiver_node_id: Some(receiver_node_id),
+                                               });
+                                       }
+                               },
+                               MonitorUpdateCompletionAction::EmitEvent { event } => {
+                                       self.pending_events.lock().unwrap().push(event);
+                               },
+                       }
+               }
+       }
+
+       /// Handles a channel reentering a functional state, either due to reconnect or a monitor
+       /// update completion.
+       fn handle_channel_resumption(&self, pending_msg_events: &mut Vec<MessageSendEvent>,
+               channel: &mut Channel<<K::Target as KeysInterface>::Signer>, raa: Option<msgs::RevokeAndACK>,
+               commitment_update: Option<msgs::CommitmentUpdate>, order: RAACommitmentOrder,
+               pending_forwards: Vec<(PendingHTLCInfo, u64)>, funding_broadcastable: Option<Transaction>,
+               channel_ready: Option<msgs::ChannelReady>, announcement_sigs: Option<msgs::AnnouncementSignatures>)
+       -> Option<(u64, OutPoint, u128, Vec<(PendingHTLCInfo, u64)>)> {
+               let mut htlc_forwards = None;
+
+               let counterparty_node_id = channel.get_counterparty_node_id();
+               if !pending_forwards.is_empty() {
+                       htlc_forwards = Some((channel.get_short_channel_id().unwrap_or(channel.outbound_scid_alias()),
+                               channel.get_funding_txo().unwrap(), channel.get_user_id(), pending_forwards));
+               }
+
+               if let Some(msg) = channel_ready {
+                       send_channel_ready!(self, pending_msg_events, channel, msg);
+               }
+               if let Some(msg) = announcement_sigs {
+                       pending_msg_events.push(events::MessageSendEvent::SendAnnouncementSignatures {
+                               node_id: counterparty_node_id,
+                               msg,
+                       });
+               }
+
+               emit_channel_ready_event!(self, channel);
+
+               macro_rules! handle_cs { () => {
+                       if let Some(update) = commitment_update {
+                               pending_msg_events.push(events::MessageSendEvent::UpdateHTLCs {
+                                       node_id: counterparty_node_id,
+                                       updates: update,
+                               });
+                       }
+               } }
+               macro_rules! handle_raa { () => {
+                       if let Some(revoke_and_ack) = raa {
+                               pending_msg_events.push(events::MessageSendEvent::SendRevokeAndACK {
+                                       node_id: counterparty_node_id,
+                                       msg: revoke_and_ack,
+                               });
+                       }
+               } }
+               match order {
+                       RAACommitmentOrder::CommitmentFirst => {
+                               handle_cs!();
+                               handle_raa!();
+                       },
+                       RAACommitmentOrder::RevokeAndACKFirst => {
+                               handle_raa!();
+                               handle_cs!();
+                       },
+               }
+
+               if let Some(tx) = funding_broadcastable {
+                       log_info!(self.logger, "Broadcasting funding transaction with txid {}", tx.txid());
+                       self.tx_broadcaster.broadcast_transaction(&tx);
+               }
+
+               htlc_forwards
+       }
+
        fn channel_monitor_updated(&self, funding_txo: &OutPoint, highest_applied_update_id: u64) {
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
 
-               let chan_restoration_res;
+               let htlc_forwards;
                let (mut pending_failures, finalized_claims, counterparty_node_id) = {
                        let mut channel_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_lock;
@@ -4520,18 +4509,20 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        })
                                } else { None }
                        } else { None };
-                       chan_restoration_res = handle_chan_restoration_locked!(self, channel_lock, channel_state, channel, updates.raa, updates.commitment_update, updates.order, None, updates.accepted_htlcs, updates.funding_broadcastable, updates.channel_ready, updates.announcement_sigs);
+                       htlc_forwards = self.handle_channel_resumption(&mut channel_state.pending_msg_events, channel.get_mut(), updates.raa, updates.commitment_update, updates.order, updates.accepted_htlcs, updates.funding_broadcastable, updates.channel_ready, updates.announcement_sigs);
                        if let Some(upd) = channel_update {
                                channel_state.pending_msg_events.push(upd);
                        }
 
                        (updates.failed_htlcs, updates.finalized_claimed_htlcs, counterparty_node_id)
                };
-               post_handle_chan_restoration!(self, chan_restoration_res);
+               if let Some(forwards) = htlc_forwards {
+                       self.forward_htlcs(&mut [forwards][..]);
+               }
                self.finalize_claims(finalized_claims);
                for failure in pending_failures.drain(..) {
                        let receiver = HTLCDestination::NextHopChannel { node_id: Some(counterparty_node_id), channel_id: funding_txo.to_channel_id() };
-                       self.fail_htlc_backwards_internal(failure.0, &failure.1, failure.2, receiver);
+                       self.fail_htlc_backwards_internal(&failure.0, &failure.1, &failure.2, receiver);
                }
        }
 
@@ -4711,7 +4702,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.temporary_channel_id));
                                        }
-                                       (try_chan_entry!(self, chan.get_mut().funding_created(msg, best_block, &self.logger), chan), chan.remove())
+                                       (try_chan_entry!(self, chan.get_mut().funding_created(msg, best_block, &self.keys_manager, &self.logger), chan), chan.remove())
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.temporary_channel_id))
                        }
@@ -4782,7 +4773,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                        }
-                                       let (monitor, funding_tx, channel_ready) = match chan.get_mut().funding_signed(&msg, best_block, &self.logger) {
+                                       let (monitor, funding_tx, channel_ready) = match chan.get_mut().funding_signed(&msg, best_block, &self.keys_manager, &self.logger) {
                                                Ok(update) => update,
                                                Err(e) => try_chan_entry!(self, Err(e), chan),
                                        };
@@ -4899,7 +4890,8 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                };
                for htlc_source in dropped_htlcs.drain(..) {
                        let receiver = HTLCDestination::NextHopChannel { node_id: Some(counterparty_node_id.clone()), channel_id: msg.channel_id };
-                       self.fail_htlc_backwards_internal(htlc_source.0, &htlc_source.1, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }, receiver);
+                       let reason = HTLCFailReason::from_failure_code(0x4000 | 8);
+                       self.fail_htlc_backwards_internal(&htlc_source.0, &htlc_source.1, &reason, receiver);
                }
 
                let _ = handle_error!(self, result, *counterparty_node_id);
@@ -4978,10 +4970,10 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                PendingHTLCStatus::Forward(PendingHTLCInfo { ref incoming_shared_secret, .. }) => {
                                                        let reason = if (error_code & 0x1000) != 0 {
                                                                let (real_code, error_data) = self.get_htlc_inbound_temp_fail_err_and_data(error_code, chan);
-                                                               onion_utils::build_first_hop_failure_packet(incoming_shared_secret, real_code, &error_data)
+                                                               HTLCFailReason::reason(real_code, error_data)
                                                        } else {
-                                                               onion_utils::build_first_hop_failure_packet(incoming_shared_secret, error_code, &[])
-                                                       };
+                                                               HTLCFailReason::from_failure_code(error_code)
+                                                       }.get_encrypted_failure_packet(incoming_shared_secret, &None);
                                                        let msg = msgs::UpdateFailHTLC {
                                                                channel_id: msg.channel_id,
                                                                htlc_id: msg.htlc_id,
@@ -5025,7 +5017,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
-                               try_chan_entry!(self, chan.get_mut().update_fail_htlc(&msg, HTLCFailReason::LightningError { err: msg.reason.clone() }), chan);
+                               try_chan_entry!(self, chan.get_mut().update_fail_htlc(&msg, HTLCFailReason::from_msg(msg)), chan);
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                }
@@ -5044,7 +5036,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        let chan_err: ChannelError = ChannelError::Close("Got update_fail_malformed_htlc with BADONION not set".to_owned());
                                        try_chan_entry!(self, Err(chan_err), chan);
                                }
-                               try_chan_entry!(self, chan.get_mut().update_fail_malformed_htlc(&msg, HTLCFailReason::Reason { failure_code: msg.failure_code, data: Vec::new() }), chan);
+                               try_chan_entry!(self, chan.get_mut().update_fail_malformed_htlc(&msg, HTLCFailReason::reason(msg.failure_code, msg.sha256_of_onion.to_vec())), chan);
                                Ok(())
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
@@ -5099,31 +5091,85 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        }
 
        #[inline]
-       fn forward_htlcs(&self, per_source_pending_forwards: &mut [(u64, OutPoint, Vec<(PendingHTLCInfo, u64)>)]) {
-               for &mut (prev_short_channel_id, prev_funding_outpoint, ref mut pending_forwards) in per_source_pending_forwards {
+       fn forward_htlcs(&self, per_source_pending_forwards: &mut [(u64, OutPoint, u128, Vec<(PendingHTLCInfo, u64)>)]) {
+               for &mut (prev_short_channel_id, prev_funding_outpoint, prev_user_channel_id, ref mut pending_forwards) in per_source_pending_forwards {
                        let mut forward_event = None;
+                       let mut new_intercept_events = Vec::new();
+                       let mut failed_intercept_forwards = Vec::new();
                        if !pending_forwards.is_empty() {
-                               let mut forward_htlcs = self.forward_htlcs.lock().unwrap();
-                               if forward_htlcs.is_empty() {
-                                       forward_event = Some(Duration::from_millis(MIN_HTLC_RELAY_HOLDING_CELL_MILLIS))
-                               }
                                for (forward_info, prev_htlc_id) in pending_forwards.drain(..) {
-                                       match forward_htlcs.entry(match forward_info.routing {
-                                                       PendingHTLCRouting::Forward { short_channel_id, .. } => short_channel_id,
-                                                       PendingHTLCRouting::Receive { .. } => 0,
-                                                       PendingHTLCRouting::ReceiveKeysend { .. } => 0,
-                                       }) {
+                                       let scid = match forward_info.routing {
+                                               PendingHTLCRouting::Forward { short_channel_id, .. } => short_channel_id,
+                                               PendingHTLCRouting::Receive { .. } => 0,
+                                               PendingHTLCRouting::ReceiveKeysend { .. } => 0,
+                                       };
+                                       // Pull this now to avoid introducing a lock order with `forward_htlcs`.
+                                       let is_our_scid = self.short_to_chan_info.read().unwrap().contains_key(&scid);
+
+                                       let mut forward_htlcs = self.forward_htlcs.lock().unwrap();
+                                       let forward_htlcs_empty = forward_htlcs.is_empty();
+                                       match forward_htlcs.entry(scid) {
                                                hash_map::Entry::Occupied(mut entry) => {
                                                        entry.get_mut().push(HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
-                                                               prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, forward_info }));
+                                                               prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, prev_user_channel_id, forward_info }));
                                                },
                                                hash_map::Entry::Vacant(entry) => {
-                                                       entry.insert(vec!(HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
-                                                               prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, forward_info })));
+                                                       if !is_our_scid && forward_info.incoming_amt_msat.is_some() &&
+                                                          fake_scid::is_valid_intercept(&self.fake_scid_rand_bytes, scid, &self.genesis_hash)
+                                                       {
+                                                               let intercept_id = InterceptId(Sha256::hash(&forward_info.incoming_shared_secret).into_inner());
+                                                               let mut pending_intercepts = self.pending_intercepted_htlcs.lock().unwrap();
+                                                               match pending_intercepts.entry(intercept_id) {
+                                                                       hash_map::Entry::Vacant(entry) => {
+                                                                               new_intercept_events.push(events::Event::HTLCIntercepted {
+                                                                                       requested_next_hop_scid: scid,
+                                                                                       payment_hash: forward_info.payment_hash,
+                                                                                       inbound_amount_msat: forward_info.incoming_amt_msat.unwrap(),
+                                                                                       expected_outbound_amount_msat: forward_info.outgoing_amt_msat,
+                                                                                       intercept_id
+                                                                               });
+                                                                               entry.insert(PendingAddHTLCInfo {
+                                                                                       prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, prev_user_channel_id, forward_info });
+                                                                       },
+                                                                       hash_map::Entry::Occupied(_) => {
+                                                                               log_info!(self.logger, "Failed to forward incoming HTLC: detected duplicate intercepted payment over short channel id {}", scid);
+                                                                               let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
+                                                                                       short_channel_id: prev_short_channel_id,
+                                                                                       outpoint: prev_funding_outpoint,
+                                                                                       htlc_id: prev_htlc_id,
+                                                                                       incoming_packet_shared_secret: forward_info.incoming_shared_secret,
+                                                                                       phantom_shared_secret: None,
+                                                                               });
+
+                                                                               failed_intercept_forwards.push((htlc_source, forward_info.payment_hash,
+                                                                                               HTLCFailReason::from_failure_code(0x4000 | 10),
+                                                                                               HTLCDestination::InvalidForward { requested_forward_scid: scid },
+                                                                               ));
+                                                                       }
+                                                               }
+                                                       } else {
+                                                               // We don't want to generate a PendingHTLCsForwardable event if only intercepted
+                                                               // payments are being processed.
+                                                               if forward_htlcs_empty {
+                                                                       forward_event = Some(Duration::from_millis(MIN_HTLC_RELAY_HOLDING_CELL_MILLIS));
+                                                               }
+                                                               entry.insert(vec!(HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
+                                                                       prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, prev_user_channel_id, forward_info })));
+                                                       }
                                                }
                                        }
                                }
                        }
+
+                       for (htlc_source, payment_hash, failure_reason, destination) in failed_intercept_forwards.drain(..) {
+                               self.fail_htlc_backwards_internal(&htlc_source, &payment_hash, &failure_reason, destination);
+                       }
+
+                       if !new_intercept_events.is_empty() {
+                               let mut events = self.pending_events.lock().unwrap();
+                               events.append(&mut new_intercept_events);
+                       }
+
                        match forward_event {
                                Some(time) => {
                                        let mut pending_events = self.pending_events.lock().unwrap();
@@ -5178,7 +5224,8 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                        raa_updates.finalized_claimed_htlcs,
                                                        chan.get().get_short_channel_id()
                                                                .unwrap_or(chan.get().outbound_scid_alias()),
-                                                       chan.get().get_funding_txo().unwrap()))
+                                                       chan.get().get_funding_txo().unwrap(),
+                                                       chan.get().get_user_id()))
                                },
                                hash_map::Entry::Vacant(_) => break Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                        }
@@ -5186,13 +5233,13 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                self.fail_holding_cell_htlcs(htlcs_to_fail, msg.channel_id, counterparty_node_id);
                match res {
                        Ok((pending_forwards, mut pending_failures, finalized_claim_htlcs,
-                               short_channel_id, channel_outpoint)) =>
+                               short_channel_id, channel_outpoint, user_channel_id)) =>
                        {
                                for failure in pending_failures.drain(..) {
                                        let receiver = HTLCDestination::NextHopChannel { node_id: Some(*counterparty_node_id), channel_id: channel_outpoint.to_channel_id() };
-                                       self.fail_htlc_backwards_internal(failure.0, &failure.1, failure.2, receiver);
+                                       self.fail_htlc_backwards_internal(&failure.0, &failure.1, &failure.2, receiver);
                                }
-                               self.forward_htlcs(&mut [(short_channel_id, channel_outpoint, pending_forwards)]);
+                               self.forward_htlcs(&mut [(short_channel_id, channel_outpoint, user_channel_id, pending_forwards)]);
                                self.finalize_claims(finalized_claim_htlcs);
                                Ok(())
                        },
@@ -5278,8 +5325,8 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        }
 
        fn internal_channel_reestablish(&self, counterparty_node_id: &PublicKey, msg: &msgs::ChannelReestablish) -> Result<(), MsgHandleErrInternal> {
-               let chan_restoration_res;
-               let (htlcs_failed_forward, need_lnd_workaround) = {
+               let htlc_forwards;
+               let need_lnd_workaround = {
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
 
@@ -5313,19 +5360,21 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                }
                                        }
                                        let need_lnd_workaround = chan.get_mut().workaround_lnd_bug_4006.take();
-                                       chan_restoration_res = handle_chan_restoration_locked!(
-                                               self, channel_state_lock, channel_state, chan, responses.raa, responses.commitment_update, responses.order,
-                                               responses.mon_update, Vec::new(), None, responses.channel_ready, responses.announcement_sigs);
+                                       htlc_forwards = self.handle_channel_resumption(
+                                               &mut channel_state.pending_msg_events, chan.get_mut(), responses.raa, responses.commitment_update, responses.order,
+                                               Vec::new(), None, responses.channel_ready, responses.announcement_sigs);
                                        if let Some(upd) = channel_update {
                                                channel_state.pending_msg_events.push(upd);
                                        }
-                                       (responses.holding_cell_failed_htlcs, need_lnd_workaround)
+                                       need_lnd_workaround
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                        }
                };
-               post_handle_chan_restoration!(self, chan_restoration_res);
-               self.fail_holding_cell_htlcs(htlcs_failed_forward, msg.channel_id, counterparty_node_id);
+
+               if let Some(forwards) = htlc_forwards {
+                       self.forward_htlcs(&mut [forwards][..]);
+               }
 
                if let Some(channel_ready_msg) = need_lnd_workaround {
                        self.internal_channel_ready(counterparty_node_id, &channel_ready_msg)?;
@@ -5348,7 +5397,8 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                } else {
                                                        log_trace!(self.logger, "Failing HTLC with hash {} from our monitor", log_bytes!(htlc_update.payment_hash.0));
                                                        let receiver = HTLCDestination::NextHopChannel { node_id: counterparty_node_id, channel_id: funding_outpoint.to_channel_id() };
-                                                       self.fail_htlc_backwards_internal(htlc_update.source, &htlc_update.payment_hash, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }, receiver);
+                                                       let reason = HTLCFailReason::from_failure_code(0x4000 | 8);
+                                                       self.fail_htlc_backwards_internal(&htlc_update.source, &htlc_update.payment_hash, &reason, receiver);
                                                }
                                        },
                                        MonitorEvent::CommitmentTxConfirmed(funding_outpoint) |
@@ -5404,11 +5454,6 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        /// Check the holding cell in each channel and free any pending HTLCs in them if possible.
        /// Returns whether there were any updates such as if pending HTLCs were freed or a monitor
        /// update was applied.
-       ///
-       /// This should only apply to HTLCs which were added to the holding cell because we were
-       /// waiting on a monitor update to finish. In that case, we don't want to free the holding cell
-       /// directly in `channel_monitor_updated` as it may introduce deadlocks calling back into user
-       /// code to inform them of a channel monitor update.
        fn check_free_holding_cells(&self) -> bool {
                let mut has_monitor_update = false;
                let mut failed_htlcs = Vec::new();
@@ -5584,8 +5629,8 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        /// This differs from [`create_inbound_payment_for_hash`] only in that it generates the
        /// [`PaymentHash`] and [`PaymentPreimage`] for you.
        ///
-       /// The [`PaymentPreimage`] will ultimately be returned to you in the [`PaymentReceived`], which
-       /// will have the [`PaymentReceived::payment_preimage`] field filled in. That should then be
+       /// The [`PaymentPreimage`] will ultimately be returned to you in the [`PaymentClaimable`], which
+       /// will have the [`PaymentClaimable::payment_preimage`] field filled in. That should then be
        /// passed directly to [`claim_funds`].
        ///
        /// See [`create_inbound_payment_for_hash`] for detailed documentation on behavior and requirements.
@@ -5601,8 +5646,8 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        /// Errors if `min_value_msat` is greater than total bitcoin supply.
        ///
        /// [`claim_funds`]: Self::claim_funds
-       /// [`PaymentReceived`]: events::Event::PaymentReceived
-       /// [`PaymentReceived::payment_preimage`]: events::Event::PaymentReceived::payment_preimage
+       /// [`PaymentClaimable`]: events::Event::PaymentClaimable
+       /// [`PaymentClaimable::payment_preimage`]: events::Event::PaymentClaimable::payment_preimage
        /// [`create_inbound_payment_for_hash`]: Self::create_inbound_payment_for_hash
        pub fn create_inbound_payment(&self, min_value_msat: Option<u64>, invoice_expiry_delta_secs: u32) -> Result<(PaymentHash, PaymentSecret), ()> {
                inbound_payment::create(&self.inbound_payment_key, min_value_msat, invoice_expiry_delta_secs, &self.keys_manager, self.highest_seen_timestamp.load(Ordering::Acquire) as u64)
@@ -5628,7 +5673,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        /// Gets a [`PaymentSecret`] for a given [`PaymentHash`], for which the payment preimage is
        /// stored external to LDK.
        ///
-       /// A [`PaymentReceived`] event will only be generated if the [`PaymentSecret`] matches a
+       /// A [`PaymentClaimable`] event will only be generated if the [`PaymentSecret`] matches a
        /// payment secret fetched via this method or [`create_inbound_payment`], and which is at least
        /// the `min_value_msat` provided here, if one is provided.
        ///
@@ -5638,7 +5683,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        ///
        /// `min_value_msat` should be set if the invoice being generated contains a value. Any payment
        /// received for the returned [`PaymentHash`] will be required to be at least `min_value_msat`
-       /// before a [`PaymentReceived`] event will be generated, ensuring that we do not provide the
+       /// before a [`PaymentClaimable`] event will be generated, ensuring that we do not provide the
        /// sender "proof-of-payment" unless they have paid the required amount.
        ///
        /// `invoice_expiry_delta_secs` describes the number of seconds that the invoice is valid for
@@ -5649,9 +5694,9 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        ///
        /// Note that we use block header time to time-out pending inbound payments (with some margin
        /// to compensate for the inaccuracy of block header timestamps). Thus, in practice we will
-       /// accept a payment and generate a [`PaymentReceived`] event for some time after the expiry.
+       /// accept a payment and generate a [`PaymentClaimable`] event for some time after the expiry.
        /// If you need exact expiry semantics, you should enforce them upon receipt of
-       /// [`PaymentReceived`].
+       /// [`PaymentClaimable`].
        ///
        /// Note that invoices generated for inbound payments should have their `min_final_cltv_expiry`
        /// set to at least [`MIN_FINAL_CLTV_EXPIRY`].
@@ -5667,7 +5712,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        /// Errors if `min_value_msat` is greater than total bitcoin supply.
        ///
        /// [`create_inbound_payment`]: Self::create_inbound_payment
-       /// [`PaymentReceived`]: events::Event::PaymentReceived
+       /// [`PaymentClaimable`]: events::Event::PaymentClaimable
        pub fn create_inbound_payment_for_hash(&self, payment_hash: PaymentHash, min_value_msat: Option<u64>, invoice_expiry_delta_secs: u32) -> Result<PaymentSecret, ()> {
                inbound_payment::create_from_hash(&self.inbound_payment_key, min_value_msat, payment_hash, invoice_expiry_delta_secs, self.highest_seen_timestamp.load(Ordering::Acquire) as u64)
        }
@@ -5722,6 +5767,39 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                }
        }
 
+       /// Gets a fake short channel id for use in receiving intercepted payments. These fake scids are
+       /// used when constructing the route hints for HTLCs intended to be intercepted. See
+       /// [`ChannelManager::forward_intercepted_htlc`].
+       ///
+       /// Note that this method is not guaranteed to return unique values, you may need to call it a few
+       /// times to get a unique scid.
+       pub fn get_intercept_scid(&self) -> u64 {
+               let best_block_height = self.best_block.read().unwrap().height();
+               let short_to_chan_info = self.short_to_chan_info.read().unwrap();
+               loop {
+                       let scid_candidate = fake_scid::Namespace::Intercept.get_fake_scid(best_block_height, &self.genesis_hash, &self.fake_scid_rand_bytes, &self.keys_manager);
+                       // Ensure the generated scid doesn't conflict with a real channel.
+                       if short_to_chan_info.contains_key(&scid_candidate) { continue }
+                       return scid_candidate
+               }
+       }
+
+       /// Gets inflight HTLC information by processing pending outbound payments that are in
+       /// our channels. May be used during pathfinding to account for in-use channel liquidity.
+       pub fn compute_inflight_htlcs(&self) -> InFlightHtlcs {
+               let mut inflight_htlcs = InFlightHtlcs::new();
+
+               for chan in self.channel_state.lock().unwrap().by_id.values() {
+                       for (htlc_source, _) in chan.inflight_htlc_sources() {
+                               if let HTLCSource::OutboundRoute { path, .. } = htlc_source {
+                                       inflight_htlcs.process_path(path, self.get_our_node_id());
+                               }
+                       }
+               }
+
+               inflight_htlcs
+       }
+
        #[cfg(any(test, fuzzing, feature = "_test_utils"))]
        pub fn get_and_clear_pending_events(&self) -> Vec<events::Event> {
                let events = core::cell::RefCell::new(Vec::new());
@@ -5730,6 +5808,12 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                events.into_inner()
        }
 
+       #[cfg(test)]
+       pub fn pop_pending_event(&self) -> Option<events::Event> {
+               let mut events = self.pending_events.lock().unwrap();
+               if events.is_empty() { None } else { Some(events.remove(0)) }
+       }
+
        #[cfg(test)]
        pub fn has_pending_payments(&self) -> bool {
                !self.pending_outbound_payments.lock().unwrap().is_empty()
@@ -6001,9 +6085,8 @@ where
                                if let Ok((channel_ready_opt, mut timed_out_pending_htlcs, announcement_sigs)) = res {
                                        for (source, payment_hash) in timed_out_pending_htlcs.drain(..) {
                                                let (failure_code, data) = self.get_htlc_inbound_temp_fail_err_and_data(0x1000|14 /* expiry_too_soon */, &channel);
-                                               timed_out_htlcs.push((source, payment_hash, HTLCFailReason::Reason {
-                                                       failure_code, data,
-                                               }, HTLCDestination::NextHopChannel { node_id: Some(channel.get_counterparty_node_id()), channel_id: channel.channel_id() }));
+                                               timed_out_htlcs.push((source, payment_hash, HTLCFailReason::reason(failure_code, data),
+                                                       HTLCDestination::NextHopChannel { node_id: Some(channel.get_counterparty_node_id()), channel_id: channel.channel_id() }));
                                        }
                                        if let Some(channel_ready) = channel_ready_opt {
                                                send_channel_ready!(self, pending_msg_events, channel, channel_ready);
@@ -6077,34 +6160,56 @@ where
                                }
                                true
                        });
+               }
 
-                       if let Some(height) = height_opt {
-                               channel_state.claimable_htlcs.retain(|payment_hash, (_, htlcs)| {
-                                       htlcs.retain(|htlc| {
-                                               // If height is approaching the number of blocks we think it takes us to get
-                                               // our commitment transaction confirmed before the HTLC expires, plus the
-                                               // number of blocks we generally consider it to take to do a commitment update,
-                                               // just give up on it and fail the HTLC.
-                                               if height >= htlc.cltv_expiry - HTLC_FAIL_BACK_BUFFER {
-                                                       let mut htlc_msat_height_data = byte_utils::be64_to_array(htlc.value).to_vec();
-                                                       htlc_msat_height_data.extend_from_slice(&byte_utils::be32_to_array(height));
-
-                                                       timed_out_htlcs.push((HTLCSource::PreviousHopData(htlc.prev_hop.clone()), payment_hash.clone(), HTLCFailReason::Reason {
-                                                               failure_code: 0x4000 | 15,
-                                                               data: htlc_msat_height_data
-                                                       }, HTLCDestination::FailedPayment { payment_hash: payment_hash.clone() }));
-                                                       false
-                                               } else { true }
-                                       });
-                                       !htlcs.is_empty() // Only retain this entry if htlcs has at least one entry.
+               if let Some(height) = height_opt {
+                       self.claimable_payments.lock().unwrap().claimable_htlcs.retain(|payment_hash, (_, htlcs)| {
+                               htlcs.retain(|htlc| {
+                                       // If height is approaching the number of blocks we think it takes us to get
+                                       // our commitment transaction confirmed before the HTLC expires, plus the
+                                       // number of blocks we generally consider it to take to do a commitment update,
+                                       // just give up on it and fail the HTLC.
+                                       if height >= htlc.cltv_expiry - HTLC_FAIL_BACK_BUFFER {
+                                               let mut htlc_msat_height_data = htlc.value.to_be_bytes().to_vec();
+                                               htlc_msat_height_data.extend_from_slice(&height.to_be_bytes());
+
+                                               timed_out_htlcs.push((HTLCSource::PreviousHopData(htlc.prev_hop.clone()), payment_hash.clone(),
+                                                       HTLCFailReason::reason(0x4000 | 15, htlc_msat_height_data),
+                                                       HTLCDestination::FailedPayment { payment_hash: payment_hash.clone() }));
+                                               false
+                                       } else { true }
                                });
-                       }
+                               !htlcs.is_empty() // Only retain this entry if htlcs has at least one entry.
+                       });
+
+                       let mut intercepted_htlcs = self.pending_intercepted_htlcs.lock().unwrap();
+                       intercepted_htlcs.retain(|_, htlc| {
+                               if height >= htlc.forward_info.outgoing_cltv_value - HTLC_FAIL_BACK_BUFFER {
+                                       let prev_hop_data = HTLCSource::PreviousHopData(HTLCPreviousHopData {
+                                               short_channel_id: htlc.prev_short_channel_id,
+                                               htlc_id: htlc.prev_htlc_id,
+                                               incoming_packet_shared_secret: htlc.forward_info.incoming_shared_secret,
+                                               phantom_shared_secret: None,
+                                               outpoint: htlc.prev_funding_outpoint,
+                                       });
+
+                                       let requested_forward_scid /* intercept scid */ = match htlc.forward_info.routing {
+                                               PendingHTLCRouting::Forward { short_channel_id, .. } => short_channel_id,
+                                               _ => unreachable!(),
+                                       };
+                                       timed_out_htlcs.push((prev_hop_data, htlc.forward_info.payment_hash,
+                                                       HTLCFailReason::from_failure_code(0x2000 | 2),
+                                                       HTLCDestination::InvalidForward { requested_forward_scid }));
+                                       log_trace!(self.logger, "Timing out intercepted HTLC with requested forward scid {}", requested_forward_scid);
+                                       false
+                               } else { true }
+                       });
                }
 
                self.handle_init_event_channel_failures(failed_channels);
 
                for (source, payment_hash, reason, destination) in timed_out_htlcs.drain(..) {
-                       self.fail_htlc_backwards_internal(source, &payment_hash, reason, destination);
+                       self.fail_htlc_backwards_internal(&source, &payment_hash, &reason, destination);
                }
        }
 
@@ -6152,7 +6257,7 @@ where
        }
 }
 
-impl<M: Deref , T: Deref , K: Deref , F: Deref , L: Deref >
+impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref >
        ChannelMessageHandler for ChannelManager<M, T, K, F, L>
        where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
         T::Target: BroadcasterInterface,
@@ -6484,6 +6589,7 @@ impl Writeable for ChannelDetails {
                        (6, self.funding_txo, option),
                        (7, self.config, option),
                        (8, self.short_channel_id, option),
+                       (9, self.confirmations, option),
                        (10, self.channel_value_satoshis, required),
                        (12, self.unspendable_punishment_reserve, option),
                        (14, user_channel_id_low, required),
@@ -6518,6 +6624,7 @@ impl Readable for ChannelDetails {
                        (6, funding_txo, option),
                        (7, config, option),
                        (8, short_channel_id, option),
+                       (9, confirmations, option),
                        (10, channel_value_satoshis, required),
                        (12, unspendable_punishment_reserve, option),
                        (14, user_channel_id_low, required),
@@ -6561,6 +6668,7 @@ impl Readable for ChannelDetails {
                        next_outbound_htlc_limit_msat: next_outbound_htlc_limit_msat.0.unwrap(),
                        inbound_capacity_msat: inbound_capacity_msat.0.unwrap(),
                        confirmations_required,
+                       confirmations,
                        force_close_spend_delay,
                        is_outbound: is_outbound.0.unwrap(),
                        is_channel_ready: is_channel_ready.0.unwrap(),
@@ -6812,18 +6920,9 @@ impl Writeable for HTLCSource {
        }
 }
 
-impl_writeable_tlv_based_enum!(HTLCFailReason,
-       (0, LightningError) => {
-               (0, err, required),
-       },
-       (1, Reason) => {
-               (0, failure_code, required),
-               (2, data, vec_type),
-       },
-;);
-
 impl_writeable_tlv_based!(PendingAddHTLCInfo, {
        (0, forward_info, required),
+       (1, prev_user_channel_id, (default_value, 0)),
        (2, prev_short_channel_id, required),
        (4, prev_htlc_id, required),
        (6, prev_funding_outpoint, required),
@@ -6918,10 +7017,13 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Writeable for ChannelMana
                        }
                }
 
-               let channel_state = self.channel_state.lock().unwrap();
+               let pending_inbound_payments = self.pending_inbound_payments.lock().unwrap();
+               let claimable_payments = self.claimable_payments.lock().unwrap();
+               let pending_outbound_payments = self.pending_outbound_payments.lock().unwrap();
+
                let mut htlc_purposes: Vec<&events::PaymentPurpose> = Vec::new();
-               (channel_state.claimable_htlcs.len() as u64).write(writer)?;
-               for (payment_hash, (purpose, previous_hops)) in channel_state.claimable_htlcs.iter() {
+               (claimable_payments.claimable_htlcs.len() as u64).write(writer)?;
+               for (payment_hash, (purpose, previous_hops)) in claimable_payments.claimable_htlcs.iter() {
                        payment_hash.write(writer)?;
                        (previous_hops.len() as u64).write(writer)?;
                        for htlc in previous_hops.iter() {
@@ -6938,8 +7040,6 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Writeable for ChannelMana
                        peer_state.latest_features.write(writer)?;
                }
 
-               let pending_inbound_payments = self.pending_inbound_payments.lock().unwrap();
-               let pending_outbound_payments = self.pending_outbound_payments.lock().unwrap();
                let events = self.pending_events.lock().unwrap();
                (events.len() as u64).write(writer)?;
                for event in events.iter() {
@@ -7002,9 +7102,27 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Writeable for ChannelMana
                                _ => {},
                        }
                }
+
+               let mut pending_intercepted_htlcs = None;
+               let our_pending_intercepts = self.pending_intercepted_htlcs.lock().unwrap();
+               if our_pending_intercepts.len() != 0 {
+                       pending_intercepted_htlcs = Some(our_pending_intercepts);
+               }
+
+               let mut pending_claiming_payments = Some(&claimable_payments.pending_claiming_payments);
+               if pending_claiming_payments.as_ref().unwrap().is_empty() {
+                       // LDK versions prior to 0.0.113 do not know how to read the pending claimed payments
+                       // map. Thus, if there are no entries we skip writing a TLV for it.
+                       pending_claiming_payments = None;
+               } else {
+                       debug_assert!(false, "While we have code to serialize pending_claiming_payments, the map should always be empty until a later PR");
+               }
+
                write_tlv_fields!(writer, {
                        (1, pending_outbound_payments_no_retry, required),
+                       (2, pending_intercepted_htlcs, option),
                        (3, pending_outbound_payments, required),
+                       (4, pending_claiming_payments, option),
                        (5, self.our_network_pubkey, required),
                        (7, self.fake_scid_rand_bytes, required),
                        (9, htlc_purposes, vec_type),
@@ -7190,6 +7308,25 @@ impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                                                user_channel_id: channel.get_user_id(),
                                                reason: ClosureReason::OutdatedChannelManager
                                        });
+                                       for (channel_htlc_source, payment_hash) in channel.inflight_htlc_sources() {
+                                               let mut found_htlc = false;
+                                               for (monitor_htlc_source, _) in monitor.get_all_current_outbound_htlcs() {
+                                                       if *channel_htlc_source == monitor_htlc_source { found_htlc = true; break; }
+                                               }
+                                               if !found_htlc {
+                                                       // If we have some HTLCs in the channel which are not present in the newer
+                                                       // ChannelMonitor, they have been removed and should be failed back to
+                                                       // ensure we don't forget them entirely. Note that if the missing HTLC(s)
+                                                       // were actually claimed we'd have generated and ensured the previous-hop
+                                                       // claim update ChannelMonitor updates were persisted prior to persising
+                                                       // the ChannelMonitor update for the forward leg, so attempting to fail the
+                                                       // backwards leg of the HTLC will simply be rejected.
+                                                       log_info!(args.logger,
+                                                               "Failing HTLC with hash {} as it is missing in the ChannelMonitor for channel {} but was present in the (stale) ChannelManager",
+                                                               log_bytes!(channel.channel_id()), log_bytes!(payment_hash.0));
+                                                       failed_htlcs.push((channel_htlc_source.clone(), *payment_hash, channel.get_counterparty_node_id(), channel.channel_id()));
+                                               }
+                                       }
                                } else {
                                        log_info!(args.logger, "Successfully loaded channel {}", log_bytes!(channel.channel_id()));
                                        if let Some(short_channel_id) = channel.get_short_channel_id() {
@@ -7270,16 +7407,6 @@ impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                                None => continue,
                        }
                }
-               if forward_htlcs_count > 0 {
-                       // If we have pending HTLCs to forward, assume we either dropped a
-                       // `PendingHTLCsForwardable` or the user received it but never processed it as they
-                       // shut down before the timer hit. Either way, set the time_forwardable to a small
-                       // constant as enough time has likely passed that we should simply handle the forwards
-                       // now, or at least after the user gets a chance to reconnect to our peers.
-                       pending_events_read.push(events::Event::PendingHTLCsForwardable {
-                               time_forwardable: Duration::from_secs(2),
-                       });
-               }
 
                let background_event_count: u64 = Readable::read(reader)?;
                let mut pending_background_events_read: Vec<BackgroundEvent> = Vec::with_capacity(cmp::min(background_event_count as usize, MAX_ALLOC_SIZE/mem::size_of::<BackgroundEvent>()));
@@ -7317,13 +7444,17 @@ impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                // pending_outbound_payments_no_retry is for compatibility with 0.0.101 clients.
                let mut pending_outbound_payments_no_retry: Option<HashMap<PaymentId, HashSet<[u8; 32]>>> = None;
                let mut pending_outbound_payments = None;
+               let mut pending_intercepted_htlcs: Option<HashMap<InterceptId, PendingAddHTLCInfo>> = Some(HashMap::new());
                let mut received_network_pubkey: Option<PublicKey> = None;
                let mut fake_scid_rand_bytes: Option<[u8; 32]> = None;
                let mut probing_cookie_secret: Option<[u8; 32]> = None;
                let mut claimable_htlc_purposes = None;
+               let mut pending_claiming_payments = Some(HashMap::new());
                read_tlv_fields!(reader, {
                        (1, pending_outbound_payments_no_retry, option),
+                       (2, pending_intercepted_htlcs, option),
                        (3, pending_outbound_payments, option),
+                       (4, pending_claiming_payments, option),
                        (5, received_network_pubkey, option),
                        (7, fake_scid_rand_bytes, option),
                        (9, claimable_htlc_purposes, vec_type),
@@ -7388,10 +7519,44 @@ impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                                                        }
                                                }
                                        }
+                                       for (htlc_source, htlc) in monitor.get_all_current_outbound_htlcs() {
+                                               if let HTLCSource::PreviousHopData(prev_hop_data) = htlc_source {
+                                                       // The ChannelMonitor is now responsible for this HTLC's
+                                                       // failure/success and will let us know what its outcome is. If we
+                                                       // still have an entry for this HTLC in `forward_htlcs`, we were
+                                                       // apparently not persisted after the monitor was when forwarding
+                                                       // the payment.
+                                                       forward_htlcs.retain(|_, forwards| {
+                                                               forwards.retain(|forward| {
+                                                                       if let HTLCForwardInfo::AddHTLC(htlc_info) = forward {
+                                                                               if htlc_info.prev_short_channel_id == prev_hop_data.short_channel_id &&
+                                                                                       htlc_info.prev_htlc_id == prev_hop_data.htlc_id
+                                                                               {
+                                                                                       log_info!(args.logger, "Removing pending to-forward HTLC with hash {} as it was forwarded to the closed channel {}",
+                                                                                               log_bytes!(htlc.payment_hash.0), log_bytes!(monitor.get_funding_txo().0.to_channel_id()));
+                                                                                       false
+                                                                               } else { true }
+                                                                       } else { true }
+                                                               });
+                                                               !forwards.is_empty()
+                                                       })
+                                               }
+                                       }
                                }
                        }
                }
 
+               if !forward_htlcs.is_empty() {
+                       // If we have pending HTLCs to forward, assume we either dropped a
+                       // `PendingHTLCsForwardable` or the user received it but never processed it as they
+                       // shut down before the timer hit. Either way, set the time_forwardable to a small
+                       // constant as enough time has likely passed that we should simply handle the forwards
+                       // now, or at least after the user gets a chance to reconnect to our peers.
+                       pending_events_read.push(events::Event::PendingHTLCsForwardable {
+                               time_forwardable: Duration::from_secs(2),
+                       });
+               }
+
                let inbound_pmt_key_material = args.keys_manager.get_inbound_payment_key_material();
                let expanded_inbound_key = inbound_payment::ExpandedKey::new(&inbound_pmt_key_material);
 
@@ -7487,6 +7652,13 @@ impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                                if let Some((payment_purpose, claimable_htlcs)) = claimable_htlcs.remove(&payment_hash) {
                                        log_info!(args.logger, "Re-claiming HTLCs with payment hash {} as we've released the preimage to a ChannelMonitor!", log_bytes!(payment_hash.0));
                                        let mut claimable_amt_msat = 0;
+                                       let mut receiver_node_id = Some(our_network_pubkey);
+                                       let phantom_shared_secret = claimable_htlcs[0].prev_hop.phantom_shared_secret;
+                                       if phantom_shared_secret.is_some() {
+                                               let phantom_pubkey = args.keys_manager.get_node_id(Recipient::PhantomNode)
+                                                       .expect("Failed to get node_id for phantom node recipient");
+                                               receiver_node_id = Some(phantom_pubkey)
+                                       }
                                        for claimable_htlc in claimable_htlcs {
                                                claimable_amt_msat += claimable_htlc.value;
 
@@ -7514,6 +7686,7 @@ impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                                                }
                                        }
                                        pending_events_read.push(events::Event::PaymentClaimed {
+                                               receiver_node_id,
                                                payment_hash,
                                                purpose: payment_purpose,
                                                amount_msat: claimable_amt_msat,
@@ -7532,14 +7705,15 @@ impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 
                        channel_state: Mutex::new(ChannelHolder {
                                by_id,
-                               claimable_htlcs,
                                pending_msg_events: Vec::new(),
                        }),
                        inbound_payment_key: expanded_inbound_key,
                        pending_inbound_payments: Mutex::new(pending_inbound_payments),
                        pending_outbound_payments: Mutex::new(pending_outbound_payments.unwrap()),
+                       pending_intercepted_htlcs: Mutex::new(pending_intercepted_htlcs.unwrap()),
 
                        forward_htlcs: Mutex::new(forward_htlcs),
+                       claimable_payments: Mutex::new(ClaimablePayments { claimable_htlcs, pending_claiming_payments: pending_claiming_payments.unwrap() }),
                        outbound_scid_aliases: Mutex::new(outbound_scid_aliases),
                        id_to_peer: Mutex::new(id_to_peer),
                        short_to_chan_info: FairRwLock::new(short_to_chan_info),
@@ -7568,7 +7742,8 @@ impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                for htlc_source in failed_htlcs.drain(..) {
                        let (source, payment_hash, counterparty_node_id, channel_id) = htlc_source;
                        let receiver = HTLCDestination::NextHopChannel { node_id: Some(counterparty_node_id), channel_id };
-                       channel_manager.fail_htlc_backwards_internal(source, &payment_hash, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() }, receiver);
+                       let reason = HTLCFailReason::from_failure_code(0x4000 | 8);
+                       channel_manager.fail_htlc_backwards_internal(&source, &payment_hash, &reason, receiver);
                }
 
                //TODO: Broadcast channel update for closed channels, but only after we've made a
@@ -8315,7 +8490,7 @@ pub mod bench {
                                $node_b.handle_revoke_and_ack(&$node_a.get_our_node_id(), &get_event_msg!(NodeHolder { node: &$node_a }, MessageSendEvent::SendRevokeAndACK, $node_b.get_our_node_id()));
 
                                expect_pending_htlcs_forwardable!(NodeHolder { node: &$node_b });
-                               expect_payment_received!(NodeHolder { node: &$node_b }, payment_hash, payment_secret, 10_000);
+                               expect_payment_claimable!(NodeHolder { node: &$node_b }, payment_hash, payment_secret, 10_000);
                                $node_b.claim_funds(payment_preimage);
                                expect_payment_claimed!(NodeHolder { node: &$node_b }, payment_hash, 10_000);
 
index 677fdccfd371819b7692cf90176716fd0b20d409..bdf40f466e7450b476c4678b343751f70f919337 100644 (file)
@@ -107,6 +107,14 @@ pub enum ConnectStyle {
        /// The same as `TransactionsFirst`, however when we have multiple blocks to connect, we only
        /// make a single `best_block_updated` call.
        TransactionsFirstSkippingBlocks,
+       /// The same as `TransactionsFirst`, however when we have multiple blocks to connect, we only
+       /// make a single `best_block_updated` call. Further, we call `transactions_confirmed` multiple
+       /// times to ensure it's idempotent.
+       TransactionsDuplicativelyFirstSkippingBlocks,
+       /// The same as `TransactionsFirst`, however when we have multiple blocks to connect, we only
+       /// make a single `best_block_updated` call. Further, we call `transactions_confirmed` multiple
+       /// times to ensure it's idempotent.
+       HighlyRedundantTransactionsFirstSkippingBlocks,
        /// The same as `TransactionsFirst` when connecting blocks. During disconnection only
        /// `transaction_unconfirmed` is called.
        TransactionsFirstReorgsOnlyTip,
@@ -121,14 +129,16 @@ impl ConnectStyle {
                        use core::hash::{BuildHasher, Hasher};
                        // Get a random value using the only std API to do so - the DefaultHasher
                        let rand_val = std::collections::hash_map::RandomState::new().build_hasher().finish();
-                       let res = match rand_val % 7 {
+                       let res = match rand_val % 9 {
                                0 => ConnectStyle::BestBlockFirst,
                                1 => ConnectStyle::BestBlockFirstSkippingBlocks,
                                2 => ConnectStyle::BestBlockFirstReorgsOnlyTip,
                                3 => ConnectStyle::TransactionsFirst,
                                4 => ConnectStyle::TransactionsFirstSkippingBlocks,
-                               5 => ConnectStyle::TransactionsFirstReorgsOnlyTip,
-                               6 => ConnectStyle::FullBlockViaListen,
+                               5 => ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks,
+                               6 => ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks,
+                               7 => ConnectStyle::TransactionsFirstReorgsOnlyTip,
+                               8 => ConnectStyle::FullBlockViaListen,
                                _ => unreachable!(),
                        };
                        eprintln!("Using Block Connection Style: {:?}", res);
@@ -143,6 +153,7 @@ impl ConnectStyle {
 pub fn connect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, depth: u32) -> BlockHash {
        let skip_intermediaries = match *node.connect_style.borrow() {
                ConnectStyle::BestBlockFirstSkippingBlocks|ConnectStyle::TransactionsFirstSkippingBlocks|
+                       ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks|ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks|
                        ConnectStyle::BestBlockFirstReorgsOnlyTip|ConnectStyle::TransactionsFirstReorgsOnlyTip => true,
                _ => false,
        };
@@ -193,8 +204,32 @@ fn do_connect_block<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, block: Block, sk
                                node.node.best_block_updated(&block.header, height);
                                node.node.transactions_confirmed(&block.header, &txdata, height);
                        },
-                       ConnectStyle::TransactionsFirst|ConnectStyle::TransactionsFirstSkippingBlocks|ConnectStyle::TransactionsFirstReorgsOnlyTip => {
+                       ConnectStyle::TransactionsFirst|ConnectStyle::TransactionsFirstSkippingBlocks|
+                       ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks|ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks|
+                       ConnectStyle::TransactionsFirstReorgsOnlyTip => {
+                               if *node.connect_style.borrow() == ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks {
+                                       let mut connections = Vec::new();
+                                       for (block, height) in node.blocks.lock().unwrap().iter() {
+                                               if !block.txdata.is_empty() {
+                                                       // Reconnect all transactions we've ever seen to ensure transaction connection
+                                                       // is *really* idempotent. This is a somewhat likely deployment for some
+                                                       // esplora implementations of chain sync which try to reduce state and
+                                                       // complexity as much as possible.
+                                                       //
+                                                       // Sadly we have to clone the block here to maintain lockorder. In the
+                                                       // future we should consider Arc'ing the blocks to avoid this.
+                                                       connections.push((block.clone(), *height));
+                                               }
+                                       }
+                                       for (old_block, height) in connections {
+                                               node.chain_monitor.chain_monitor.transactions_confirmed(&old_block.header,
+                                                       &old_block.txdata.iter().enumerate().collect::<Vec<_>>(), height);
+                                       }
+                               }
                                node.chain_monitor.chain_monitor.transactions_confirmed(&block.header, &txdata, height);
+                               if *node.connect_style.borrow() == ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks {
+                                       node.chain_monitor.chain_monitor.transactions_confirmed(&block.header, &txdata, height);
+                               }
                                call_claimable_balances(node);
                                node.chain_monitor.chain_monitor.best_block_updated(&block.header, height);
                                node.node.transactions_confirmed(&block.header, &txdata, height);
@@ -226,7 +261,8 @@ pub fn disconnect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, count: u32)
                                node.chain_monitor.chain_monitor.block_disconnected(&orig.0.header, orig.1);
                                Listen::block_disconnected(node.node, &orig.0.header, orig.1);
                        },
-                       ConnectStyle::BestBlockFirstSkippingBlocks|ConnectStyle::TransactionsFirstSkippingBlocks => {
+                       ConnectStyle::BestBlockFirstSkippingBlocks|ConnectStyle::TransactionsFirstSkippingBlocks|
+                       ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks|ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks => {
                                if i == count - 1 {
                                        node.chain_monitor.chain_monitor.best_block_updated(&prev.0.header, prev.1);
                                        node.node.best_block_updated(&prev.0.header, prev.1);
@@ -1091,7 +1127,7 @@ macro_rules! check_closed_event {
                use $crate::util::events::Event;
 
                let events = $node.node.get_and_clear_pending_events();
-               assert_eq!(events.len(), $events);
+               assert_eq!(events.len(), $events, "{:?}", events);
                let expected_reason = $reason;
                let mut issues_discard_funding = false;
                for event in events {
@@ -1350,7 +1386,7 @@ macro_rules! expect_pending_htlcs_forwardable_conditions {
                let events = $node.node.get_and_clear_pending_events();
                match events[0] {
                        $crate::util::events::Event::PendingHTLCsForwardable { .. } => { },
-                       _ => panic!("Unexpected event"),
+                       _ => panic!("Unexpected event {:?}", events),
                };
 
                let count = expected_failures.len() + 1;
@@ -1433,20 +1469,20 @@ macro_rules! expect_pending_htlcs_forwardable_from_events {
                }
        }}
 }
-
 #[macro_export]
 #[cfg(any(test, feature = "_bench_unstable", feature = "_test_utils"))]
-macro_rules! expect_payment_received {
+macro_rules! expect_payment_claimable {
        ($node: expr, $expected_payment_hash: expr, $expected_payment_secret: expr, $expected_recv_value: expr) => {
-               expect_payment_received!($node, $expected_payment_hash, $expected_payment_secret, $expected_recv_value, None)
+               expect_payment_claimable!($node, $expected_payment_hash, $expected_payment_secret, $expected_recv_value, None, $node.node.get_our_node_id())
        };
-       ($node: expr, $expected_payment_hash: expr, $expected_payment_secret: expr, $expected_recv_value: expr, $expected_payment_preimage: expr) => {
+       ($node: expr, $expected_payment_hash: expr, $expected_payment_secret: expr, $expected_recv_value: expr, $expected_payment_preimage: expr, $expected_receiver_node_id: expr) => {
                let events = $node.node.get_and_clear_pending_events();
                assert_eq!(events.len(), 1);
                match events[0] {
-                       $crate::util::events::Event::PaymentReceived { ref payment_hash, ref purpose, amount_msat } => {
+                       $crate::util::events::Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id: _, via_user_channel_id: _ } => {
                                assert_eq!($expected_payment_hash, *payment_hash);
                                assert_eq!($expected_recv_value, amount_msat);
+                               assert_eq!($expected_receiver_node_id, receiver_node_id.unwrap());
                                match purpose {
                                        $crate::util::events::PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                                assert_eq!(&$expected_payment_preimage, payment_preimage);
@@ -1560,7 +1596,7 @@ macro_rules! expect_payment_forwarded {
                                if !$downstream_force_closed {
                                        assert!($node.node.list_channels().iter().any(|x| x.counterparty.node_id == $next_node.node.get_our_node_id() && x.channel_id == next_channel_id.unwrap()));
                                }
-                               assert_eq!(claim_from_onchain_tx, $upstream_force_closed);
+                               assert_eq!(claim_from_onchain_tx, $downstream_force_closed);
                        },
                        _ => panic!("Unexpected event"),
                }
@@ -1720,7 +1756,7 @@ pub fn send_along_route_with_secret<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>,
        payment_id
 }
 
-pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_received_expected: bool, clear_recipient_events: bool, expected_preimage: Option<PaymentPreimage>) {
+pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_claimable_expected: bool, clear_recipient_events: bool, expected_preimage: Option<PaymentPreimage>) {
        let mut payment_event = SendEvent::from_event(ev);
        let mut prev_node = origin_node;
 
@@ -1735,11 +1771,12 @@ pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_p
 
                if idx == expected_path.len() - 1 && clear_recipient_events {
                        let events_2 = node.node.get_and_clear_pending_events();
-                       if payment_received_expected {
+                       if payment_claimable_expected {
                                assert_eq!(events_2.len(), 1);
                                match events_2[0] {
-                                       Event::PaymentReceived { ref payment_hash, ref purpose, amount_msat } => {
+                                       Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, ref via_channel_id, ref via_user_channel_id } => {
                                                assert_eq!(our_payment_hash, *payment_hash);
+                                               assert_eq!(node.node.get_our_node_id(), receiver_node_id.unwrap());
                                                match &purpose {
                                                        PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                                                assert_eq!(expected_preimage, *payment_preimage);
@@ -1751,6 +1788,8 @@ pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_p
                                                        },
                                                }
                                                assert_eq!(amount_msat, recv_value);
+                                               assert!(node.node.list_channels().iter().any(|details| details.channel_id == via_channel_id.unwrap()));
+                                               assert!(node.node.list_channels().iter().any(|details| details.user_channel_id == via_user_channel_id.unwrap()));
                                        },
                                        _ => panic!("Unexpected event"),
                                }
@@ -1769,8 +1808,8 @@ pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_p
        }
 }
 
-pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_received_expected: bool, expected_preimage: Option<PaymentPreimage>) {
-       do_pass_along_path(origin_node, expected_path, recv_value, our_payment_hash, our_payment_secret, ev, payment_received_expected, true, expected_preimage);
+pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_claimable_expected: bool, expected_preimage: Option<PaymentPreimage>) {
+       do_pass_along_path(origin_node, expected_path, recv_value, our_payment_hash, our_payment_secret, ev, payment_claimable_expected, true, expected_preimage);
 }
 
 pub fn pass_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&[&Node<'a, 'b, 'c>]], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: PaymentSecret) {
@@ -1778,7 +1817,7 @@ pub fn pass_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_rou
        assert_eq!(events.len(), expected_route.len());
        for (path_idx, (ev, expected_path)) in events.drain(..).zip(expected_route.iter()).enumerate() {
                // Once we've gotten through all the HTLCs, the last one should result in a
-               // PaymentReceived (but each previous one should not!), .
+               // PaymentClaimable (but each previous one should not!), .
                let expect_payment = path_idx == expected_route.len() - 1;
                pass_along_path(origin_node, expected_path, recv_value, our_payment_hash.clone(), Some(our_payment_secret), ev, expect_payment, None);
        }
@@ -2413,6 +2452,14 @@ macro_rules! handle_chan_reestablish_msgs {
                                assert_eq!(*node_id, $dst_node.node.get_our_node_id());
                        }
 
+                       let mut had_channel_update = false; // ChannelUpdate may be now or later, but not both
+                       if let Some(&MessageSendEvent::SendChannelUpdate { ref node_id, ref msg }) = msg_events.get(idx) {
+                               assert_eq!(*node_id, $dst_node.node.get_our_node_id());
+                               idx += 1;
+                               assert_eq!(msg.contents.flags & 2, 0); // "disabled" flag must not be set as we just reconnected.
+                               had_channel_update = true;
+                       }
+
                        let mut revoke_and_ack = None;
                        let mut commitment_update = None;
                        let order = if let Some(ev) = msg_events.get(idx) {
@@ -2457,6 +2504,7 @@ macro_rules! handle_chan_reestablish_msgs {
                                assert_eq!(*node_id, $dst_node.node.get_our_node_id());
                                idx += 1;
                                assert_eq!(msg.contents.flags & 2, 0); // "disabled" flag must not be set as we just reconnected.
+                               assert!(!had_channel_update);
                        }
 
                        assert_eq!(msg_events.len(), idx);
index e3ae9c7a20f10a51a65ce0a2b3a0e50dac626bd1..79c85e75447d5bb959ef391c5941d0d62db933ff 100644 (file)
@@ -30,7 +30,7 @@ use crate::ln::features::{ChannelFeatures, NodeFeatures};
 use crate::ln::msgs;
 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
 use crate::util::enforcing_trait_impls::EnforcingSigner;
-use crate::util::{byte_utils, test_utils};
+use crate::util::test_utils;
 use crate::util::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose, ClosureReason, HTLCDestination};
 use crate::util::errors::APIError;
 use crate::util::ser::{Writeable, ReadableArgs};
@@ -705,7 +705,7 @@ fn test_update_fee_that_funder_cannot_afford() {
 
        // Assemble the set of keys we can use for signatures for our commitment_signed message.
        let commit_tx_keys = chan_utils::TxCreationKeys::derive_new(&secp_ctx, &remote_point, &remote_delayed_payment_basepoint,
-               &remote_htlc_basepoint, &local_revocation_basepoint, &local_htlc_basepoint).unwrap();
+               &remote_htlc_basepoint, &local_revocation_basepoint, &local_htlc_basepoint);
 
        let res = {
                let local_chan_lock = nodes[0].node.channel_state.lock().unwrap();
@@ -834,7 +834,7 @@ fn test_update_fee_with_fundee_update_add_htlc() {
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
        match events[0] {
-               Event::PaymentReceived { .. } => { },
+               Event::PaymentClaimable { .. } => { },
                _ => panic!("Unexpected event"),
        };
 
@@ -1179,7 +1179,7 @@ fn holding_cell_htlc_counting() {
        assert_eq!(events.len(), payments.len());
        for (event, &(_, ref hash)) in events.iter().zip(payments.iter()) {
                match event {
-                       &Event::PaymentReceived { ref payment_hash, .. } => {
+                       &Event::PaymentClaimable { ref payment_hash, .. } => {
                                assert_eq!(*payment_hash, *hash);
                        },
                        _ => panic!("Unexpected event"),
@@ -1412,7 +1412,7 @@ fn test_fee_spike_violation_fails_htlc() {
 
        // Assemble the set of keys we can use for signatures for our commitment_signed message.
        let commit_tx_keys = chan_utils::TxCreationKeys::derive_new(&secp_ctx, &remote_point, &remote_delayed_payment_basepoint,
-               &remote_htlc_basepoint, &local_revocation_basepoint, &local_htlc_basepoint).unwrap();
+               &remote_htlc_basepoint, &local_revocation_basepoint, &local_htlc_basepoint);
 
        // Build the remote commitment transaction so we can sign it, and then later use the
        // signature for the commitment_signed message.
@@ -1936,7 +1936,7 @@ fn test_channel_reserve_holding_cell_htlcs() {
        commitment_signed_dance!(nodes[2], nodes[1], payment_event_11.commitment_msg, false);
 
        expect_pending_htlcs_forwardable!(nodes[2]);
-       expect_payment_received!(nodes[2], our_payment_hash_1, our_payment_secret_1, recv_value_1);
+       expect_payment_claimable!(nodes[2], our_payment_hash_1, our_payment_secret_1, recv_value_1);
 
        // flush the htlcs in the holding cell
        assert_eq!(commitment_update_2.update_add_htlcs.len(), 2);
@@ -1956,9 +1956,11 @@ fn test_channel_reserve_holding_cell_htlcs() {
        let events = nodes[2].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 2);
        match events[0] {
-               Event::PaymentReceived { ref payment_hash, ref purpose, amount_msat } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
                        assert_eq!(our_payment_hash_21, *payment_hash);
                        assert_eq!(recv_value_21, amount_msat);
+                       assert_eq!(nodes[2].node.get_our_node_id(), receiver_node_id.unwrap());
+                       assert_eq!(via_channel_id, Some(chan_2.2));
                        match &purpose {
                                PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                        assert!(payment_preimage.is_none());
@@ -1970,9 +1972,11 @@ fn test_channel_reserve_holding_cell_htlcs() {
                _ => panic!("Unexpected event"),
        }
        match events[1] {
-               Event::PaymentReceived { ref payment_hash, ref purpose, amount_msat } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
                        assert_eq!(our_payment_hash_22, *payment_hash);
                        assert_eq!(recv_value_22, amount_msat);
+                       assert_eq!(nodes[2].node.get_our_node_id(), receiver_node_id.unwrap());
+                       assert_eq!(via_channel_id, Some(chan_2.2));
                        match &purpose {
                                PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                        assert!(payment_preimage.is_none());
@@ -2102,7 +2106,7 @@ fn channel_reserve_in_flight_removes() {
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash_3, payment_secret_3, 100000);
+       expect_payment_claimable!(nodes[1], payment_hash_3, payment_secret_3, 100000);
 
        // Note that as this RAA was generated before the delivery of the update_fulfill it shouldn't
        // resolve the second HTLC from A's point of view.
@@ -2149,7 +2153,7 @@ fn channel_reserve_in_flight_removes() {
        check_added_monitors!(nodes[0], 1);
 
        expect_pending_htlcs_forwardable!(nodes[0]);
-       expect_payment_received!(nodes[0], payment_hash_4, payment_secret_4, 10000);
+       expect_payment_claimable!(nodes[0], payment_hash_4, payment_secret_4, 10000);
 
        claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_4);
        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3);
@@ -2814,12 +2818,17 @@ fn test_htlc_on_chain_success() {
        check_added_monitors!(nodes[1], 1);
        check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
-       assert_eq!(node_txn.len(), 6); // ChannelManager : 3 (commitment tx + HTLC-Sucess * 2), ChannelMonitor : 3 (HTLC-Success, 2* RBF bumps of above HTLC txn)
+       assert!(node_txn.len() == 4 || node_txn.len() == 6); // ChannelManager : 3 (commitment tx + HTLC-Sucess * 2), ChannelMonitor : 3 (HTLC-Success, 2* RBF bumps of above HTLC txn)
        let commitment_spend =
                if node_txn[0].input[0].previous_output.txid == node_a_commitment_tx[0].txid() {
-                       check_spends!(node_txn[1], commitment_tx[0]);
-                       check_spends!(node_txn[2], commitment_tx[0]);
-                       assert_ne!(node_txn[1].input[0].previous_output.vout, node_txn[2].input[0].previous_output.vout);
+                       if node_txn.len() == 6 {
+                               // In some block `ConnectionStyle`s we may avoid broadcasting the double-spending
+                               // transactions spending the HTLC outputs of C's commitment transaction. Otherwise,
+                               // check that the extra broadcasts (double-)spend those here.
+                               check_spends!(node_txn[1], commitment_tx[0]);
+                               check_spends!(node_txn[2], commitment_tx[0]);
+                               assert_ne!(node_txn[1].input[0].previous_output.vout, node_txn[2].input[0].previous_output.vout);
+                       }
                        &node_txn[0]
                } else {
                        check_spends!(node_txn[0], commitment_tx[0]);
@@ -2834,10 +2843,11 @@ fn test_htlc_on_chain_success() {
        assert_eq!(commitment_spend.input[1].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT);
        assert_eq!(commitment_spend.lock_time.0, 0);
        assert!(commitment_spend.output[0].script_pubkey.is_v0_p2wpkh()); // direct payment
-       check_spends!(node_txn[3], chan_1.3);
-       assert_eq!(node_txn[3].input[0].witness.clone().last().unwrap().len(), 71);
-       check_spends!(node_txn[4], node_txn[3]);
-       check_spends!(node_txn[5], node_txn[3]);
+       let funding_spend_offset = if node_txn.len() == 6 { 3 } else { 1 };
+       check_spends!(node_txn[funding_spend_offset], chan_1.3);
+       assert_eq!(node_txn[funding_spend_offset].input[0].witness.clone().last().unwrap().len(), 71);
+       check_spends!(node_txn[funding_spend_offset + 1], node_txn[funding_spend_offset]);
+       check_spends!(node_txn[funding_spend_offset + 2], node_txn[funding_spend_offset]);
        // We don't bother to check that B can claim the HTLC output on its commitment tx here as
        // we already checked the same situation with A.
 
@@ -3370,6 +3380,12 @@ fn test_htlc_ignore_latest_remote_commitment() {
        let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
        let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
        let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+       if *nodes[1].connect_style.borrow() == ConnectStyle::FullBlockViaListen {
+               // We rely on the ability to connect a block redundantly, which isn't allowed via
+               // `chain::Listen`, so we never run the test if we randomly get assigned that
+               // connect_style.
+               return;
+       }
        create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 
        route_payment(&nodes[0], &[&nodes[1]], 10000000);
@@ -3391,7 +3407,6 @@ fn test_htlc_ignore_latest_remote_commitment() {
 
        // Duplicate the connect_block call since this may happen due to other listeners
        // registering new transactions
-       header.prev_blockhash = header.block_hash();
        connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[0].clone(), node_txn[2].clone()]});
 }
 
@@ -3612,15 +3627,16 @@ fn do_test_drop_messages_peer_disconnect(messages_delivered: u8, simulate_broken
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 
        let mut as_channel_ready = None;
-       if messages_delivered == 0 {
-               let (channel_ready, _, _) = create_chan_between_nodes_with_value_a(&nodes[0], &nodes[1], 100000, 10001, channelmanager::provided_init_features(), channelmanager::provided_init_features());
+       let channel_id = if messages_delivered == 0 {
+               let (channel_ready, chan_id, _) = create_chan_between_nodes_with_value_a(&nodes[0], &nodes[1], 100000, 10001, channelmanager::provided_init_features(), channelmanager::provided_init_features());
                as_channel_ready = Some(channel_ready);
                // nodes[1] doesn't receive the channel_ready message (it'll be re-sent on reconnect)
                // Note that we store it so that if we're running with `simulate_broken_lnd` we can deliver
                // it before the channel_reestablish message.
+               chan_id
        } else {
-               create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-       }
+               create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2
+       };
 
        let (route, payment_hash_1, payment_preimage_1, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1_000_000);
 
@@ -3723,9 +3739,11 @@ fn do_test_drop_messages_peer_disconnect(messages_delivered: u8, simulate_broken
        let events_2 = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events_2.len(), 1);
        match events_2[0] {
-               Event::PaymentReceived { ref payment_hash, ref purpose, amount_msat } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
                        assert_eq!(payment_hash_1, *payment_hash);
                        assert_eq!(amount_msat, 1_000_000);
+                       assert_eq!(receiver_node_id.unwrap(), nodes[1].node.get_our_node_id());
+                       assert_eq!(via_channel_id, Some(channel_id));
                        match &purpose {
                                PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                        assert!(payment_preimage.is_none());
@@ -4007,7 +4025,7 @@ fn test_drop_messages_peer_disconnect_dual_htlc() {
        let events_5 = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events_5.len(), 1);
        match events_5[0] {
-               Event::PaymentReceived { ref payment_hash, ref purpose, .. } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, .. } => {
                        assert_eq!(payment_hash_2, *payment_hash);
                        match &purpose {
                                PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
@@ -4050,7 +4068,7 @@ fn do_test_htlc_timeout(send_partial_mpp: bool) {
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
                // Now do the relevant commitment_signed/RAA dances along the path, noting that the final
-               // hop should *not* yet generate any PaymentReceived event(s).
+               // hop should *not* yet generate any PaymentClaimable event(s).
                pass_along_path(&nodes[0], &[&nodes[1]], 100000, our_payment_hash, Some(payment_secret), events.drain(..).next().unwrap(), false, None);
                our_payment_hash
        } else {
@@ -4082,8 +4100,8 @@ fn do_test_htlc_timeout(send_partial_mpp: bool) {
        nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_timeout_updates.update_fail_htlcs[0]);
        commitment_signed_dance!(nodes[0], nodes[1], htlc_timeout_updates.commitment_signed, false);
        // 100_000 msat as u64, followed by the height at which we failed back above
-       let mut expected_failure_data = byte_utils::be64_to_array(100_000).to_vec();
-       expected_failure_data.extend_from_slice(&byte_utils::be32_to_array(block_count - 1));
+       let mut expected_failure_data = (100_000 as u64).to_be_bytes().to_vec();
+       expected_failure_data.extend_from_slice(&(block_count - 1).to_be_bytes());
        expect_payment_failed!(nodes[0], our_payment_hash, true, 0x4000 | 15, &expected_failure_data[..]);
 }
 
@@ -4785,7 +4803,7 @@ fn test_duplicate_payment_hash_one_failure_one_success() {
        assert_eq!(htlc_success_txn[2], commitment_txn[0]);
        assert_eq!(htlc_success_txn[3], htlc_success_txn[0]);
        assert_eq!(htlc_success_txn[4], htlc_success_txn[1]);
-       assert_ne!(htlc_success_txn[0].input[0].previous_output, htlc_timeout_tx.input[0].previous_output);
+       assert_ne!(htlc_success_txn[1].input[0].previous_output, htlc_timeout_tx.input[0].previous_output);
 
        mine_transaction(&nodes[1], &htlc_timeout_tx);
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
@@ -4808,7 +4826,7 @@ fn test_duplicate_payment_hash_one_failure_one_success() {
        // Solve 2nd HTLC by broadcasting on B's chain HTLC-Success Tx from C
        // Note that the fee paid is effectively double as the HTLC value (including the nodes[1] fee
        // and nodes[2] fee) is rounded down and then claimed in full.
-       mine_transaction(&nodes[1], &htlc_success_txn[0]);
+       mine_transaction(&nodes[1], &htlc_success_txn[1]);
        expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(196*2), true, true);
        let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
        assert!(updates.update_add_htlcs.is_empty());
@@ -5782,7 +5800,7 @@ fn test_free_and_fail_holding_cell_htlcs() {
        let events = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
        match events[0] {
-               Event::PaymentReceived { .. } => {},
+               Event::PaymentClaimable { .. } => {},
                _ => panic!("Unexpected event"),
        }
        nodes[1].node.claim_funds(payment_preimage_1);
@@ -6005,7 +6023,7 @@ fn test_update_add_htlc_bolt2_sender_cltv_expiry_too_high() {
                .with_features(channelmanager::provided_invoice_features());
        let (mut route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], payment_params, 100000000, 0);
        route.paths[0].last_mut().unwrap().cltv_expiry_delta = 500000001;
-       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::RouteError { ref err },
+       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::InvalidRoute { ref err },
                assert_eq!(err, &"Channel CLTV overflowed?"));
 }
 
@@ -6041,7 +6059,7 @@ fn test_update_add_htlc_bolt2_sender_exceed_max_htlc_num_and_htlc_id_increment()
                commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 
                expect_pending_htlcs_forwardable!(nodes[1]);
-               expect_payment_received!(nodes[1], our_payment_hash, our_payment_secret, 100000);
+               expect_payment_claimable!(nodes[1], our_payment_hash, our_payment_secret, 100000);
        }
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
        unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
@@ -6956,8 +6974,8 @@ fn test_check_htlc_underpaying() {
        commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false, true);
 
        // 10_000 msat as u64, followed by a height of CHAN_CONFIRM_DEPTH as u32
-       let mut expected_failure_data = byte_utils::be64_to_array(10_000).to_vec();
-       expected_failure_data.extend_from_slice(&byte_utils::be32_to_array(CHAN_CONFIRM_DEPTH));
+       let mut expected_failure_data = (10_000 as u64).to_be_bytes().to_vec();
+       expected_failure_data.extend_from_slice(&CHAN_CONFIRM_DEPTH.to_be_bytes());
        expect_payment_failed!(nodes[0], our_payment_hash, true, 0x4000|15, &expected_failure_data[..]);
 }
 
@@ -7936,7 +7954,7 @@ fn test_preimage_storage() {
        let events = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
        match events[0] {
-               Event::PaymentReceived { ref purpose, .. } => {
+               Event::PaymentClaimable { ref purpose, .. } => {
                        match &purpose {
                                PaymentPurpose::InvoicePayment { payment_preimage, .. } => {
                                        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage.unwrap());
@@ -8006,7 +8024,7 @@ fn test_secret_timeout() {
        let events = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
        match events[0] {
-               Event::PaymentReceived { purpose: PaymentPurpose::InvoicePayment { payment_preimage, payment_secret }, .. } => {
+               Event::PaymentClaimable { purpose: PaymentPurpose::InvoicePayment { payment_preimage, payment_secret }, .. } => {
                        assert!(payment_preimage.is_none());
                        assert_eq!(payment_secret, our_payment_secret);
                        // We don't actually have the payment preimage with which to claim this payment!
@@ -8113,7 +8131,7 @@ fn test_update_err_monitor_lockdown() {
                let mut w = test_utils::TestVecWriter(Vec::new());
                monitor.write(&mut w).unwrap();
                let new_monitor = <(BlockHash, channelmonitor::ChannelMonitor<EnforcingSigner>)>::read(
-                               &mut io::Cursor::new(&w.0), &test_utils::OnlyReadsKeysInterface {}).unwrap().1;
+                               &mut io::Cursor::new(&w.0), nodes[0].keys_manager).unwrap().1;
                assert!(new_monitor == *monitor);
                let watchtower = test_utils::TestChainMonitor::new(Some(&chain_source), &chanmon_cfgs[0].tx_broadcaster, &logger, &chanmon_cfgs[0].fee_estimator, &persister, &node_cfgs[0].keys_manager);
                assert_eq!(watchtower.watch_channel(outpoint, new_monitor), ChannelMonitorUpdateStatus::Completed);
@@ -8177,7 +8195,7 @@ fn test_concurrent_monitor_claim() {
                let mut w = test_utils::TestVecWriter(Vec::new());
                monitor.write(&mut w).unwrap();
                let new_monitor = <(BlockHash, channelmonitor::ChannelMonitor<EnforcingSigner>)>::read(
-                               &mut io::Cursor::new(&w.0), &test_utils::OnlyReadsKeysInterface {}).unwrap().1;
+                               &mut io::Cursor::new(&w.0), nodes[0].keys_manager).unwrap().1;
                assert!(new_monitor == *monitor);
                let watchtower = test_utils::TestChainMonitor::new(Some(&chain_source), &chanmon_cfgs[0].tx_broadcaster, &logger, &chanmon_cfgs[0].fee_estimator, &persister, &node_cfgs[0].keys_manager);
                assert_eq!(watchtower.watch_channel(outpoint, new_monitor), ChannelMonitorUpdateStatus::Completed);
@@ -8206,7 +8224,7 @@ fn test_concurrent_monitor_claim() {
                let mut w = test_utils::TestVecWriter(Vec::new());
                monitor.write(&mut w).unwrap();
                let new_monitor = <(BlockHash, channelmonitor::ChannelMonitor<EnforcingSigner>)>::read(
-                               &mut io::Cursor::new(&w.0), &test_utils::OnlyReadsKeysInterface {}).unwrap().1;
+                               &mut io::Cursor::new(&w.0), nodes[0].keys_manager).unwrap().1;
                assert!(new_monitor == *monitor);
                let watchtower = test_utils::TestChainMonitor::new(Some(&chain_source), &chanmon_cfgs[0].tx_broadcaster, &logger, &chanmon_cfgs[0].fee_estimator, &persister, &node_cfgs[0].keys_manager);
                assert_eq!(watchtower.watch_channel(outpoint, new_monitor), ChannelMonitorUpdateStatus::Completed);
@@ -8941,7 +8959,7 @@ fn do_test_dup_htlc_second_rejected(test_for_second_fail_panic: bool) {
                commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
        }
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], our_payment_hash, our_payment_secret, 10_000);
+       expect_payment_claimable!(nodes[1], our_payment_hash, our_payment_secret, 10_000);
 
        {
                // Note that we use a different PaymentId here to allow us to duplicativly pay
@@ -9003,7 +9021,7 @@ fn do_test_dup_htlc_second_rejected(test_for_second_fail_panic: bool) {
 #[test]
 fn test_dup_htlc_second_fail_panic() {
        // Previously, if we received two HTLCs back-to-back, where the second overran the expected
-       // value for the payment, we'd fail back both HTLCs after generating a `PaymentReceived` event.
+       // value for the payment, we'd fail back both HTLCs after generating a `PaymentClaimable` event.
        // Then, if the user failed the second payment, they'd hit a "tried to fail an already failed
        // HTLC" debug panic. This tests for this behavior, checking that only one HTLC is auto-failed.
        do_test_dup_htlc_second_rejected(true);
@@ -9190,9 +9208,9 @@ fn test_keysend_payments_to_private_node() {
 
 #[test]
 fn test_double_partial_claim() {
-       // Test what happens if a node receives a payment, generates a PaymentReceived event, the HTLCs
+       // Test what happens if a node receives a payment, generates a PaymentClaimable event, the HTLCs
        // time out, the sender resends only some of the MPP parts, then the user processes the
-       // PaymentReceived event, ensuring they don't inadvertently claim only part of the full payment
+       // PaymentClaimable event, ensuring they don't inadvertently claim only part of the full payment
        // amount.
        let chanmon_cfgs = create_chanmon_cfgs(4);
        let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
@@ -9213,7 +9231,7 @@ fn test_double_partial_claim() {
        });
 
        send_along_route_with_secret(&nodes[0], route.clone(), &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 15_000_000, payment_hash, payment_secret);
-       // nodes[3] has now received a PaymentReceived event...which it will take some (exorbitant)
+       // nodes[3] has now received a PaymentClaimable event...which it will take some (exorbitant)
        // amount of time to respond to.
 
        // Connect some blocks to time out the payment
@@ -9237,7 +9255,7 @@ fn test_double_partial_claim() {
        pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), events.drain(..).next().unwrap(), false, None);
 
        // At this point nodes[3] has received one half of the payment, and the user goes to handle
-       // that PaymentReceived event they got hours ago and never handled...we should refuse to claim.
+       // that PaymentClaimable event they got hours ago and never handled...we should refuse to claim.
        nodes[3].node.claim_funds(payment_preimage);
        check_added_monitors!(nodes[3], 0);
        assert!(nodes[3].node.get_and_clear_pending_msg_events().is_empty());
index 2916855c0ab34d27b1352ac57e594ad31b090cc9..8726c6a996204b99e0fa653c3240a348c2752fcf 100644 (file)
@@ -567,6 +567,16 @@ fn do_test_claim_value_force_close(prev_commitment_tx: bool) {
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
        assert_eq!(Vec::<Balance>::new(),
                nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances());
+
+       // Ensure that even if we connect more blocks, potentially replaying the entire chain if we're
+       // using `ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks`, we don't get new
+       // monitor events or claimable balances.
+       for node in nodes.iter() {
+               connect_blocks(node, 6);
+               connect_blocks(node, 6);
+               assert!(node.chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+               assert!(node.chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty());
+       }
 }
 
 #[test]
@@ -606,7 +616,7 @@ fn test_balances_on_local_commitment_htlcs() {
        commitment_signed_dance!(nodes[1], nodes[0], updates.commitment_signed, false);
 
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash, payment_secret, 10_000_000);
+       expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 10_000_000);
 
        let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 20_000_000);
        nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
@@ -617,7 +627,7 @@ fn test_balances_on_local_commitment_htlcs() {
        commitment_signed_dance!(nodes[1], nodes[0], updates.commitment_signed, false);
 
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash_2, payment_secret_2, 20_000_000);
+       expect_payment_claimable!(nodes[1], payment_hash_2, payment_secret_2, 20_000_000);
        nodes[1].node.claim_funds(payment_preimage_2);
        get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
        check_added_monitors!(nodes[1], 1);
@@ -750,6 +760,14 @@ fn test_balances_on_local_commitment_htlcs() {
        connect_blocks(&nodes[0], node_a_htlc_claimable - nodes[0].best_block_info().1);
        assert!(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty());
        test_spendable_output(&nodes[0], &as_txn[1]);
+
+       // Ensure that even if we connect more blocks, potentially replaying the entire chain if we're
+       // using `ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks`, we don't get new
+       // monitor events or claimable balances.
+       connect_blocks(&nodes[0], 6);
+       connect_blocks(&nodes[0], 6);
+       assert!(nodes[0].chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+       assert!(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty());
 }
 
 #[test]
@@ -982,6 +1000,14 @@ fn test_no_preimage_inbound_htlc_balances() {
 
        connect_blocks(&nodes[1], 1);
        assert!(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty());
+
+       // Ensure that even if we connect more blocks, potentially replaying the entire chain if we're
+       // using `ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks`, we don't get new
+       // monitor events or claimable balances.
+       connect_blocks(&nodes[1], 6);
+       connect_blocks(&nodes[1], 6);
+       assert!(nodes[1].chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+       assert!(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty());
 }
 
 fn sorted_vec_with_additions<T: Ord + Clone>(v_orig: &Vec<T>, extra_ts: &[&T]) -> Vec<T> {
@@ -1231,6 +1257,14 @@ fn do_test_revoked_counterparty_commitment_balances(confirm_htlc_spend_first: bo
        test_spendable_output(&nodes[1], &claim_txn[1]);
        expect_payment_failed!(nodes[1], timeout_payment_hash, false);
        assert_eq!(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances(), Vec::new());
+
+       // Ensure that even if we connect more blocks, potentially replaying the entire chain if we're
+       // using `ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks`, we don't get new
+       // monitor events or claimable balances.
+       connect_blocks(&nodes[1], 6);
+       connect_blocks(&nodes[1], 6);
+       assert!(nodes[1].chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+       assert!(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty());
 }
 
 #[test]
@@ -1437,6 +1471,14 @@ fn test_revoked_counterparty_htlc_tx_balances() {
        test_spendable_output(&nodes[0], &as_second_htlc_claim_tx[1]);
 
        assert_eq!(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances(), Vec::new());
+
+       // Ensure that even if we connect more blocks, potentially replaying the entire chain if we're
+       // using `ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks`, we don't get new
+       // monitor events or claimable balances.
+       connect_blocks(&nodes[0], 6);
+       connect_blocks(&nodes[0], 6);
+       assert!(nodes[0].chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+       assert!(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty());
 }
 
 #[test]
@@ -1628,4 +1670,12 @@ fn test_revoked_counterparty_aggregated_claims() {
        expect_payment_failed!(nodes[1], revoked_payment_hash, false);
        test_spendable_output(&nodes[1], &claim_txn_2[0]);
        assert!(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty());
+
+       // Ensure that even if we connect more blocks, potentially replaying the entire chain if we're
+       // using `ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks`, we don't get new
+       // monitor events or claimable balances.
+       connect_blocks(&nodes[1], 6);
+       connect_blocks(&nodes[1], 6);
+       assert!(nodes[1].chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+       assert!(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty());
 }
index c9422a6e07205f260d277ee24578a5b92a43ce5c..ca2fd76584174a5e1be7b2c405c28a0cf29541d3 100644 (file)
@@ -25,7 +25,7 @@ use crate::ln::msgs::{ChannelMessageHandler, ChannelUpdate};
 use crate::ln::wire::Encode;
 use crate::util::events::{Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
 use crate::util::ser::{Writeable, Writer};
-use crate::util::{byte_utils, test_utils};
+use crate::util::test_utils;
 use crate::util::config::{UserConfig, ChannelConfig};
 use crate::util::errors::APIError;
 
@@ -125,7 +125,7 @@ fn run_onion_failure_test_with_fail_intercept<F1,F2,F3>(_name: &str, test_case:
 
                        if test_case == 2 || test_case == 200 {
                                expect_htlc_forward!(&nodes[2]);
-                               expect_event!(&nodes[2], Event::PaymentReceived);
+                               expect_event!(&nodes[2], Event::PaymentClaimable);
                                callback_node();
                                expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], vec![HTLCDestination::FailedPayment { payment_hash: payment_hash.clone() }]);
                        }
@@ -548,7 +548,7 @@ fn test_onion_failure() {
                connect_blocks(&nodes[0], height - nodes[0].best_block_info().1);
                connect_blocks(&nodes[1], height - nodes[1].best_block_info().1);
                connect_blocks(&nodes[2], height - nodes[2].best_block_info().1);
-       }, || {}, true, Some(17), None, None);
+       }, || {}, false, Some(0x4000 | 15), None, None);
 
        run_onion_failure_test("final_incorrect_cltv_expiry", 1, &nodes, &route, &payment_hash, &payment_secret, |_| {}, || {
                for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
@@ -1053,8 +1053,8 @@ fn test_phantom_final_incorrect_cltv_expiry() {
        commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
 
        // Ensure the payment fails with the expected error.
-       let expected_cltv = 82;
-       let error_data = byte_utils::be32_to_array(expected_cltv).to_vec();
+       let expected_cltv: u32 = 82;
+       let error_data = expected_cltv.to_be_bytes().to_vec();
        let mut fail_conditions = PaymentFailedConditions::new()
                .blamed_scid(phantom_scid)
                .expected_htlc_error_data(18, &error_data);
@@ -1099,10 +1099,93 @@ fn test_phantom_failure_too_low_cltv() {
        commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
 
        // Ensure the payment fails with the expected error.
-       let error_data = Vec::new();
+       let mut error_data = recv_value_msat.to_be_bytes().to_vec();
+       error_data.extend_from_slice(
+               &nodes[0].node.best_block.read().unwrap().height().to_be_bytes(),
+       );
+       let mut fail_conditions = PaymentFailedConditions::new()
+               .blamed_scid(phantom_scid)
+               .expected_htlc_error_data(0x4000 | 15, &error_data);
+       expect_payment_failed_conditions(&nodes[0], payment_hash, true, fail_conditions);
+}
+
+#[test]
+fn test_phantom_failure_modified_cltv() {
+       // Test that we fail back phantoms if the upstream node fiddled with the CLTV too much with the
+       // correct error code.
+       let chanmon_cfgs = create_chanmon_cfgs(2);
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+       let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+       let channel = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
+
+       // Get the route.
+       let recv_value_msat = 10_000;
+       let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[1], Some(recv_value_msat));
+       let (mut route, phantom_scid) = get_phantom_route!(nodes, recv_value_msat, channel);
+
+       // Route the HTLC through to the destination.
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       check_added_monitors!(nodes[0], 1);
+       let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
+       let mut update_add = update_0.update_add_htlcs[0].clone();
+
+       // Modify the route to have a too-low cltv.
+       update_add.cltv_expiry -= 10;
+
+       nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
+       commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
+
+       let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
+       assert!(update_1.update_fail_htlcs.len() == 1);
+       let fail_msg = update_1.update_fail_htlcs[0].clone();
+       nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
+       commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
+
+       // Ensure the payment fails with the expected error.
        let mut fail_conditions = PaymentFailedConditions::new()
                .blamed_scid(phantom_scid)
-               .expected_htlc_error_data(17, &error_data);
+               .expected_htlc_error_data(0x2000 | 2, &[]);
+       expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
+}
+
+#[test]
+fn test_phantom_failure_expires_too_soon() {
+       // Test that we fail back phantoms if the HTLC got delayed and we got blocks in between with
+       // the correct error code.
+       let chanmon_cfgs = create_chanmon_cfgs(2);
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+       let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+       let channel = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
+
+       // Get the route.
+       let recv_value_msat = 10_000;
+       let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[1], Some(recv_value_msat));
+       let (mut route, phantom_scid) = get_phantom_route!(nodes, recv_value_msat, channel);
+
+       // Route the HTLC through to the destination.
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       check_added_monitors!(nodes[0], 1);
+       let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
+       let mut update_add = update_0.update_add_htlcs[0].clone();
+
+       connect_blocks(&nodes[1], CLTV_FAR_FAR_AWAY);
+       nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add);
+       commitment_signed_dance!(nodes[1], nodes[0], &update_0.commitment_signed, false, true);
+
+       let update_1 = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
+       assert!(update_1.update_fail_htlcs.len() == 1);
+       let fail_msg = update_1.update_fail_htlcs[0].clone();
+       nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
+       commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
+
+       // Ensure the payment fails with the expected error.
+       let mut fail_conditions = PaymentFailedConditions::new()
+               .blamed_scid(phantom_scid)
+               .expected_htlc_error_data(0x2000 | 2, &[]);
        expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
 }
 
@@ -1144,10 +1227,8 @@ fn test_phantom_failure_too_low_recv_amt() {
        commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
 
        // Ensure the payment fails with the expected error.
-       let mut error_data = byte_utils::be64_to_array(bad_recv_amt_msat).to_vec();
-       error_data.extend_from_slice(
-               &byte_utils::be32_to_array(nodes[1].node.best_block.read().unwrap().height()),
-       );
+       let mut error_data = bad_recv_amt_msat.to_be_bytes().to_vec();
+       error_data.extend_from_slice(&nodes[1].node.best_block.read().unwrap().height().to_be_bytes());
        let mut fail_conditions = PaymentFailedConditions::new()
                .blamed_scid(phantom_scid)
                .expected_htlc_error_data(0x4000 | 15, &error_data);
@@ -1229,7 +1310,7 @@ fn test_phantom_failure_reject_payment() {
        nodes[1].node.process_pending_htlc_forwards();
        expect_pending_htlcs_forwardable_ignore!(nodes[1]);
        nodes[1].node.process_pending_htlc_forwards();
-       expect_payment_received!(nodes[1], payment_hash, payment_secret, recv_amt_msat);
+       expect_payment_claimable!(nodes[1], payment_hash, payment_secret, recv_amt_msat, None, route.paths[0].last().unwrap().pubkey);
        nodes[1].node.fail_htlc_backwards(&payment_hash);
        expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]);
        nodes[1].node.process_pending_htlc_forwards();
@@ -1242,10 +1323,8 @@ fn test_phantom_failure_reject_payment() {
        commitment_signed_dance!(nodes[0], nodes[1], update_1.commitment_signed, false);
 
        // Ensure the payment fails with the expected error.
-       let mut error_data = byte_utils::be64_to_array(recv_amt_msat).to_vec();
-       error_data.extend_from_slice(
-               &byte_utils::be32_to_array(nodes[1].node.best_block.read().unwrap().height()),
-       );
+       let mut error_data = recv_amt_msat.to_be_bytes().to_vec();
+       error_data.extend_from_slice(&nodes[1].node.best_block.read().unwrap().height().to_be_bytes());
        let mut fail_conditions = PaymentFailedConditions::new()
                .blamed_scid(phantom_scid)
                .expected_htlc_error_data(0x4000 | 15, &error_data);
index 23dc556cfacca4f129d7968323f997a0f3c63692..1abaf5920a51d8476d56b6f65968f725bd1a781a 100644 (file)
@@ -15,7 +15,7 @@ use crate::routing::gossip::NetworkUpdate;
 use crate::routing::router::RouteHop;
 use crate::util::chacha20::{ChaCha20, ChaChaReader};
 use crate::util::errors::{self, APIError};
-use crate::util::ser::{Readable, ReadableArgs, Writeable, LengthCalculatingWriter};
+use crate::util::ser::{Readable, ReadableArgs, Writeable, Writer, LengthCalculatingWriter};
 use crate::util::logger::Logger;
 
 use bitcoin::hashes::{Hash, HashEngine};
@@ -182,11 +182,11 @@ pub(super) fn build_onion_payloads(path: &Vec<RouteHop>, total_msat: u64, paymen
                });
                cur_value_msat += hop.fee_msat;
                if cur_value_msat >= 21000000 * 100000000 * 1000 {
-                       return Err(APIError::RouteError{err: "Channel fees overflowed?"});
+                       return Err(APIError::InvalidRoute{err: "Channel fees overflowed?"});
                }
                cur_cltv += hop.cltv_expiry_delta as u32;
                if cur_cltv >= 500000000 {
-                       return Err(APIError::RouteError{err: "Channel CLTV overflowed?"});
+                       return Err(APIError::InvalidRoute{err: "Channel CLTV overflowed?"});
                }
                last_short_channel_id = hop.short_channel_id;
        }
@@ -382,7 +382,7 @@ pub(super) fn build_failure_packet(shared_secret: &[u8], failure_type: u16, fail
        packet
 }
 
-#[inline]
+#[cfg(test)]
 pub(super) fn build_first_hop_failure_packet(shared_secret: &[u8], failure_type: u16, failure_data: &[u8]) -> msgs::OnionErrorPacket {
        let failure_packet = build_failure_packet(shared_secret, failure_type, failure_data);
        encrypt_failure_packet(shared_secret, &failure_packet.encode()[..])
@@ -592,6 +592,146 @@ pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &
        } else { unreachable!(); }
 }
 
+#[derive(Clone)] // See Channel::revoke_and_ack for why, tl;dr: Rust bug
+pub(super) struct HTLCFailReason(HTLCFailReasonRepr);
+
+#[derive(Clone)] // See Channel::revoke_and_ack for why, tl;dr: Rust bug
+enum HTLCFailReasonRepr {
+       LightningError {
+               err: msgs::OnionErrorPacket,
+       },
+       Reason {
+               failure_code: u16,
+               data: Vec<u8>,
+       }
+}
+
+impl core::fmt::Debug for HTLCFailReason {
+       fn fmt(&self, f: &mut core::fmt::Formatter) -> Result<(), core::fmt::Error> {
+               match self.0 {
+                       HTLCFailReasonRepr::Reason { ref failure_code, .. } => {
+                               write!(f, "HTLC error code {}", failure_code)
+                       },
+                       HTLCFailReasonRepr::LightningError { .. } => {
+                               write!(f, "pre-built LightningError")
+                       }
+               }
+       }
+}
+
+impl Writeable for HTLCFailReason {
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), crate::io::Error> {
+               self.0.write(writer)
+       }
+}
+impl Readable for HTLCFailReason {
+       fn read<R: Read>(reader: &mut R) -> Result<Self, msgs::DecodeError> {
+               Ok(Self(Readable::read(reader)?))
+       }
+}
+
+impl_writeable_tlv_based_enum!(HTLCFailReasonRepr,
+       (0, LightningError) => {
+               (0, err, required),
+       },
+       (1, Reason) => {
+               (0, failure_code, required),
+               (2, data, vec_type),
+       },
+;);
+
+impl HTLCFailReason {
+       pub(super) fn reason(failure_code: u16, data: Vec<u8>) -> Self {
+               const BADONION: u16 = 0x8000;
+               const PERM: u16 = 0x4000;
+               const NODE: u16 = 0x2000;
+               const UPDATE: u16 = 0x1000;
+
+                    if failure_code == 1  | PERM { debug_assert!(data.is_empty()) }
+               else if failure_code == 2  | NODE { debug_assert!(data.is_empty()) }
+               else if failure_code == 2  | PERM | NODE { debug_assert!(data.is_empty()) }
+               else if failure_code == 3  | PERM | NODE { debug_assert!(data.is_empty()) }
+               else if failure_code == 4  | BADONION | PERM { debug_assert_eq!(data.len(), 32) }
+               else if failure_code == 5  | BADONION | PERM { debug_assert_eq!(data.len(), 32) }
+               else if failure_code == 6  | BADONION | PERM { debug_assert_eq!(data.len(), 32) }
+               else if failure_code == 7  | UPDATE {
+                       debug_assert_eq!(data.len() - 2, u16::from_be_bytes(data[0..2].try_into().unwrap()) as usize) }
+               else if failure_code == 8  | PERM { debug_assert!(data.is_empty()) }
+               else if failure_code == 9  | PERM { debug_assert!(data.is_empty()) }
+               else if failure_code == 10 | PERM { debug_assert!(data.is_empty()) }
+               else if failure_code == 11 | UPDATE {
+                       debug_assert_eq!(data.len() - 2 - 8, u16::from_be_bytes(data[8..10].try_into().unwrap()) as usize) }
+               else if failure_code == 12 | UPDATE {
+                       debug_assert_eq!(data.len() - 2 - 8, u16::from_be_bytes(data[8..10].try_into().unwrap()) as usize) }
+               else if failure_code == 13 | UPDATE {
+                       debug_assert_eq!(data.len() - 2 - 4, u16::from_be_bytes(data[4..6].try_into().unwrap()) as usize) }
+               else if failure_code == 14 | UPDATE {
+                       debug_assert_eq!(data.len() - 2, u16::from_be_bytes(data[0..2].try_into().unwrap()) as usize) }
+               else if failure_code == 15 | PERM { debug_assert_eq!(data.len(), 12) }
+               else if failure_code == 18 { debug_assert_eq!(data.len(), 4) }
+               else if failure_code == 19 { debug_assert_eq!(data.len(), 8) }
+               else if failure_code == 20 | UPDATE {
+                       debug_assert_eq!(data.len() - 2 - 2, u16::from_be_bytes(data[2..4].try_into().unwrap()) as usize) }
+               else if failure_code == 21 { debug_assert!(data.is_empty()) }
+               else if failure_code == 22 | PERM { debug_assert!(data.len() <= 11) }
+               else if failure_code == 23 { debug_assert!(data.is_empty()) }
+               else if failure_code & BADONION != 0 {
+                       // We set some bogus BADONION failure codes in test, so ignore unknown ones.
+               }
+               else { debug_assert!(false, "Unknown failure code: {}", failure_code) }
+
+               Self(HTLCFailReasonRepr::Reason { failure_code, data })
+       }
+
+       pub(super) fn from_failure_code(failure_code: u16) -> Self {
+               Self::reason(failure_code, Vec::new())
+       }
+
+       pub(super) fn from_msg(msg: &msgs::UpdateFailHTLC) -> Self {
+               Self(HTLCFailReasonRepr::LightningError { err: msg.reason.clone() })
+       }
+
+       pub(super) fn get_encrypted_failure_packet(&self, incoming_packet_shared_secret: &[u8; 32], phantom_shared_secret: &Option<[u8; 32]>)
+       -> msgs::OnionErrorPacket {
+               match self.0 {
+                       HTLCFailReasonRepr::Reason { ref failure_code, ref data } => {
+                               if let Some(phantom_ss) = phantom_shared_secret {
+                                       let phantom_packet = build_failure_packet(phantom_ss, *failure_code, &data[..]).encode();
+                                       let encrypted_phantom_packet = encrypt_failure_packet(phantom_ss, &phantom_packet);
+                                       encrypt_failure_packet(incoming_packet_shared_secret, &encrypted_phantom_packet.data[..])
+                               } else {
+                                       let packet = build_failure_packet(incoming_packet_shared_secret, *failure_code, &data[..]).encode();
+                                       encrypt_failure_packet(incoming_packet_shared_secret, &packet)
+                               }
+                       },
+                       HTLCFailReasonRepr::LightningError { ref err } => {
+                               encrypt_failure_packet(incoming_packet_shared_secret, &err.data)
+                       }
+               }
+       }
+
+       pub(super) fn decode_onion_failure<T: secp256k1::Signing, L: Deref>(
+               &self, secp_ctx: &Secp256k1<T>, logger: &L, htlc_source: &HTLCSource
+       ) -> (Option<NetworkUpdate>, Option<u64>, bool, Option<u16>, Option<Vec<u8>>)
+       where L::Target: Logger {
+               match self.0 {
+                       HTLCFailReasonRepr::LightningError { ref err } => {
+                               process_onion_failure(secp_ctx, logger, &htlc_source, err.data.clone())
+                       },
+                       HTLCFailReasonRepr::Reason { ref failure_code, ref data, .. } => {
+                               // we get a fail_malformed_htlc from the first hop
+                               // TODO: We'd like to generate a NetworkUpdate for temporary
+                               // failures here, but that would be insufficient as find_route
+                               // generally ignores its view of our own channels as we provide them via
+                               // ChannelDetails.
+                               if let &HTLCSource::OutboundRoute { ref path, .. } = htlc_source {
+                                       (None, Some(path.first().unwrap().short_channel_id), true, Some(*failure_code), Some(data.clone()))
+                               } else { unreachable!(); }
+                       }
+               }
+       }
+}
+
 /// Allows `decode_next_hop` to return the next hop packet bytes for either payments or onion
 /// message forwards.
 pub(crate) trait NextPacketBytes: AsMut<[u8]> {
index 27dc67a0dadf738c46f9cd5fda36db554b639a73..ff65bac4bc8b9e7e18832ae067ab5bbf1769b57c 100644 (file)
@@ -19,7 +19,8 @@ use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
 use crate::ln::channelmanager::{self, BREAKDOWN_TIMEOUT, ChannelManager, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS};
 use crate::ln::msgs;
 use crate::ln::msgs::ChannelMessageHandler;
-use crate::routing::router::{PaymentParameters, get_route};
+use crate::routing::gossip::RoutingFees;
+use crate::routing::router::{get_route, PaymentParameters, RouteHint, RouteHintHop, RouteParameters};
 use crate::util::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
 use crate::util::test_utils;
 use crate::util::errors::APIError;
@@ -32,6 +33,7 @@ use bitcoin::network::constants::Network;
 use crate::prelude::*;
 
 use crate::ln::functional_test_utils::*;
+use crate::routing::gossip::NodeId;
 
 #[test]
 fn retry_single_path_payment() {
@@ -785,8 +787,9 @@ fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, co
        let funding_txo = OutPoint { txid: funding_tx.txid(), index: 0 };
        let mon_updates: Vec<_> = chanmon_cfgs[0].persister.chain_sync_monitor_persistences.lock().unwrap()
                .get_mut(&funding_txo).unwrap().drain().collect();
-       // If we are using chain::Confirm instead of chain::Listen, we will get the same update twice
-       assert!(mon_updates.len() == 1 || mon_updates.len() == 2);
+       // If we are using chain::Confirm instead of chain::Listen, we will get the same update twice.
+       // If we're testing connection idempotency we may get substantially more.
+       assert!(mon_updates.len() >= 1);
        assert!(nodes[0].chain_monitor.release_pending_monitor_events().is_empty());
        assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 
@@ -1239,3 +1242,327 @@ fn abandoned_send_payment_idempotent() {
        pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
        claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
 }
+
+#[derive(PartialEq)]
+enum InterceptTest {
+       Forward,
+       Fail,
+       Timeout,
+}
+
+#[test]
+fn test_trivial_inflight_htlc_tracking(){
+       // In this test, we test three scenarios:
+       // (1) Sending + claiming a payment successfully should return `None` when querying InFlightHtlcs
+       // (2) Sending a payment without claiming it should return the payment's value (500000) when querying InFlightHtlcs
+       // (3) After we claim the payment sent in (2), InFlightHtlcs should return `None` for the query.
+       let chanmon_cfgs = create_chanmon_cfgs(3);
+       let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
+       let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
+
+       let (_, _, chan_1_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
+       let (_, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features());
+
+       // Send and claim the payment. Inflight HTLCs should be empty.
+       send_payment(&nodes[0], &vec!(&nodes[1], &nodes[2])[..], 500000);
+       {
+               let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
+
+               let node_0_channel_lock = nodes[0].node.channel_state.lock().unwrap();
+               let node_1_channel_lock = nodes[1].node.channel_state.lock().unwrap();
+               let channel_1 = node_0_channel_lock.by_id.get(&chan_1_id).unwrap();
+               let channel_2 = node_1_channel_lock.by_id.get(&chan_2_id).unwrap();
+
+               let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
+                       &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
+                       &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
+                       channel_1.get_short_channel_id().unwrap()
+               );
+               let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
+                       &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
+                       &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
+                       channel_2.get_short_channel_id().unwrap()
+               );
+
+               assert_eq!(chan_1_used_liquidity, None);
+               assert_eq!(chan_2_used_liquidity, None);
+       }
+
+       // Send the payment, but do not claim it. Our inflight HTLCs should contain the pending payment.
+       let (payment_preimage, _,  _) = route_payment(&nodes[0], &vec!(&nodes[1], &nodes[2])[..], 500000);
+       {
+               let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
+
+               let node_0_channel_lock = nodes[0].node.channel_state.lock().unwrap();
+               let node_1_channel_lock = nodes[1].node.channel_state.lock().unwrap();
+               let channel_1 = node_0_channel_lock.by_id.get(&chan_1_id).unwrap();
+               let channel_2 = node_1_channel_lock.by_id.get(&chan_2_id).unwrap();
+
+               let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
+                       &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
+                       &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
+                       channel_1.get_short_channel_id().unwrap()
+               );
+               let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
+                       &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
+                       &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
+                       channel_2.get_short_channel_id().unwrap()
+               );
+
+               // First hop accounts for expected 1000 msat fee
+               assert_eq!(chan_1_used_liquidity, Some(501000));
+               assert_eq!(chan_2_used_liquidity, Some(500000));
+       }
+
+       // Now, let's claim the payment. This should result in the used liquidity to return `None`.
+       claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
+       {
+               let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
+
+               let node_0_channel_lock = nodes[0].node.channel_state.lock().unwrap();
+               let node_1_channel_lock = nodes[1].node.channel_state.lock().unwrap();
+               let channel_1 = node_0_channel_lock.by_id.get(&chan_1_id).unwrap();
+               let channel_2 = node_1_channel_lock.by_id.get(&chan_2_id).unwrap();
+
+               let chan_1_used_liquidity = inflight_htlcs.used_liquidity_msat(
+                       &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
+                       &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
+                       channel_1.get_short_channel_id().unwrap()
+               );
+               let chan_2_used_liquidity = inflight_htlcs.used_liquidity_msat(
+                       &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()) ,
+                       &NodeId::from_pubkey(&nodes[2].node.get_our_node_id()),
+                       channel_2.get_short_channel_id().unwrap()
+               );
+
+               assert_eq!(chan_1_used_liquidity, None);
+               assert_eq!(chan_2_used_liquidity, None);
+       }
+}
+
+#[test]
+fn test_holding_cell_inflight_htlcs() {
+       let chanmon_cfgs = create_chanmon_cfgs(2);
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+       let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+       let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
+
+       let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
+       let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
+
+       // Queue up two payments - one will be delivered right away, one immediately goes into the
+       // holding cell as nodes[0] is AwaitingRAA.
+       {
+               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
+               check_added_monitors!(nodes[0], 1);
+               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+               check_added_monitors!(nodes[0], 0);
+       }
+
+       let inflight_htlcs = node_chanmgrs[0].compute_inflight_htlcs();
+
+       {
+               let channel_lock = nodes[0].node.channel_state.lock().unwrap();
+               let channel = channel_lock.by_id.get(&channel_id).unwrap();
+
+               let used_liquidity = inflight_htlcs.used_liquidity_msat(
+                       &NodeId::from_pubkey(&nodes[0].node.get_our_node_id()) ,
+                       &NodeId::from_pubkey(&nodes[1].node.get_our_node_id()),
+                       channel.get_short_channel_id().unwrap()
+               );
+
+               assert_eq!(used_liquidity, Some(2000000));
+       }
+
+       // Clear pending events so test doesn't throw a "Had excess message on node..." error
+       nodes[0].node.get_and_clear_pending_msg_events();
+}
+
+#[test]
+fn intercepted_payment() {
+       // Test that detecting an intercept scid on payment forward will signal LDK to generate an
+       // intercept event, which the LSP can then use to either (a) open a JIT channel to forward the
+       // payment or (b) fail the payment.
+       do_test_intercepted_payment(InterceptTest::Forward);
+       do_test_intercepted_payment(InterceptTest::Fail);
+       // Make sure that intercepted payments will be automatically failed back if too many blocks pass.
+       do_test_intercepted_payment(InterceptTest::Timeout);
+}
+
+fn do_test_intercepted_payment(test: InterceptTest) {
+       let chanmon_cfgs = create_chanmon_cfgs(3);
+       let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
+
+       let mut zero_conf_chan_config = test_default_channel_config();
+       zero_conf_chan_config.manually_accept_inbound_channels = true;
+       let mut intercept_forwards_config = test_default_channel_config();
+       intercept_forwards_config.accept_intercept_htlcs = true;
+       let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), Some(zero_conf_chan_config)]);
+
+       let nodes = create_network(3, &node_cfgs, &node_chanmgrs);
+       let scorer = test_utils::TestScorer::with_penalty(0);
+       let random_seed_bytes = chanmon_cfgs[0].keys_manager.get_secure_random_bytes();
+
+       let _ = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
+
+       let amt_msat = 100_000;
+       let intercept_scid = nodes[1].node.get_intercept_scid();
+       let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id())
+               .with_route_hints(vec![
+                       RouteHint(vec![RouteHintHop {
+                               src_node_id: nodes[1].node.get_our_node_id(),
+                               short_channel_id: intercept_scid,
+                               fees: RoutingFees {
+                                       base_msat: 1000,
+                                       proportional_millionths: 0,
+                               },
+                               cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA,
+                               htlc_minimum_msat: None,
+                               htlc_maximum_msat: None,
+                       }])
+               ])
+               .with_features(channelmanager::provided_invoice_features());
+       let route_params = RouteParameters {
+               payment_params,
+               final_value_msat: amt_msat,
+               final_cltv_expiry_delta: TEST_FINAL_CLTV,
+       };
+       let route = get_route(
+               &nodes[0].node.get_our_node_id(), &route_params.payment_params,
+               &nodes[0].network_graph.read_only(), None, route_params.final_value_msat,
+               route_params.final_cltv_expiry_delta, nodes[0].logger, &scorer, &random_seed_bytes
+       ).unwrap();
+
+       let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       let payment_event = {
+               {
+                       let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
+                       assert_eq!(added_monitors.len(), 1);
+                       added_monitors.clear();
+               }
+               let mut events = nodes[0].node.get_and_clear_pending_msg_events();
+               assert_eq!(events.len(), 1);
+               SendEvent::from_event(events.remove(0))
+       };
+       nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
+       commitment_signed_dance!(nodes[1], nodes[0], &payment_event.commitment_msg, false, true);
+
+       // Check that we generate the PaymentIntercepted event when an intercept forward is detected.
+       let events = nodes[1].node.get_and_clear_pending_events();
+       assert_eq!(events.len(), 1);
+       let (intercept_id, expected_outbound_amount_msat) = match events[0] {
+               crate::util::events::Event::HTLCIntercepted {
+                       intercept_id, expected_outbound_amount_msat, payment_hash: pmt_hash, inbound_amount_msat, requested_next_hop_scid: short_channel_id
+               } => {
+                       assert_eq!(pmt_hash, payment_hash);
+                       assert_eq!(inbound_amount_msat, route.get_total_amount() + route.get_total_fees());
+                       assert_eq!(short_channel_id, intercept_scid);
+                       (intercept_id, expected_outbound_amount_msat)
+               },
+               _ => panic!()
+       };
+
+       // Check for unknown channel id error.
+       let unknown_chan_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &[42; 32], nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
+       assert_eq!(unknown_chan_id_err , APIError::ChannelUnavailable  { err: format!("Channel with id {} not found", log_bytes!([42; 32])) });
+
+       if test == InterceptTest::Fail {
+               // Ensure we can fail the intercepted payment back.
+               nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap();
+               expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::UnknownNextHop { requested_forward_scid: intercept_scid }]);
+               nodes[1].node.process_pending_htlc_forwards();
+               let update_fail = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
+               check_added_monitors!(&nodes[1], 1);
+               assert!(update_fail.update_fail_htlcs.len() == 1);
+               let fail_msg = update_fail.update_fail_htlcs[0].clone();
+               nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_msg);
+               commitment_signed_dance!(nodes[0], nodes[1], update_fail.commitment_signed, false);
+
+               // Ensure the payment fails with the expected error.
+               let fail_conditions = PaymentFailedConditions::new()
+                       .blamed_scid(intercept_scid)
+                       .blamed_chan_closed(true)
+                       .expected_htlc_error_data(0x4000 | 10, &[]);
+               expect_payment_failed_conditions(&nodes[0], payment_hash, false, fail_conditions);
+       } else if test == InterceptTest::Forward {
+               // Check that we'll fail as expected when sending to a channel that isn't in `ChannelReady` yet.
+               let temp_chan_id = nodes[1].node.create_channel(nodes[2].node.get_our_node_id(), 100_000, 0, 42, None).unwrap();
+               let unusable_chan_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &temp_chan_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
+               assert_eq!(unusable_chan_err , APIError::ChannelUnavailable { err: format!("Channel with id {} not fully established", log_bytes!(temp_chan_id)) });
+               assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
+
+               // Open the just-in-time channel so the payment can then be forwarded.
+               let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
+
+               // Finally, forward the intercepted payment through and claim it.
+               nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap();
+               expect_pending_htlcs_forwardable!(nodes[1]);
+
+               let payment_event = {
+                       {
+                               let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
+                               assert_eq!(added_monitors.len(), 1);
+                               added_monitors.clear();
+                       }
+                       let mut events = nodes[1].node.get_and_clear_pending_msg_events();
+                       assert_eq!(events.len(), 1);
+                       SendEvent::from_event(events.remove(0))
+               };
+               nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
+               commitment_signed_dance!(nodes[2], nodes[1], &payment_event.commitment_msg, false, true);
+               expect_pending_htlcs_forwardable!(nodes[2]);
+
+               let payment_preimage = nodes[2].node.get_payment_preimage(payment_hash, payment_secret).unwrap();
+               expect_payment_claimable!(&nodes[2], payment_hash, payment_secret, amt_msat, Some(payment_preimage), nodes[2].node.get_our_node_id());
+               do_claim_payment_along_route(&nodes[0], &vec!(&vec!(&nodes[1], &nodes[2])[..]), false, payment_preimage);
+               let events = nodes[0].node.get_and_clear_pending_events();
+               assert_eq!(events.len(), 2);
+               match events[0] {
+                       Event::PaymentSent { payment_preimage: ref ev_preimage, payment_hash: ref ev_hash, ref fee_paid_msat, .. } => {
+                               assert_eq!(payment_preimage, *ev_preimage);
+                               assert_eq!(payment_hash, *ev_hash);
+                               assert_eq!(fee_paid_msat, &Some(1000));
+                       },
+                       _ => panic!("Unexpected event")
+               }
+               match events[1] {
+                       Event::PaymentPathSuccessful { payment_hash: hash, .. } => {
+                               assert_eq!(hash, Some(payment_hash));
+                       },
+                       _ => panic!("Unexpected event")
+               }
+       } else if test == InterceptTest::Timeout {
+               let mut block = Block {
+                       header: BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 },
+                       txdata: vec![],
+               };
+               connect_block(&nodes[0], &block);
+               connect_block(&nodes[1], &block);
+               for _ in 0..TEST_FINAL_CLTV {
+                       block.header.prev_blockhash = block.block_hash();
+                       connect_block(&nodes[0], &block);
+                       connect_block(&nodes[1], &block);
+               }
+               expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::InvalidForward { requested_forward_scid: intercept_scid }]);
+               check_added_monitors!(nodes[1], 1);
+               let htlc_timeout_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
+               assert!(htlc_timeout_updates.update_add_htlcs.is_empty());
+               assert_eq!(htlc_timeout_updates.update_fail_htlcs.len(), 1);
+               assert!(htlc_timeout_updates.update_fail_malformed_htlcs.is_empty());
+               assert!(htlc_timeout_updates.update_fee.is_none());
+
+               nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_timeout_updates.update_fail_htlcs[0]);
+               commitment_signed_dance!(nodes[0], nodes[1], htlc_timeout_updates.commitment_signed, false);
+               expect_payment_failed!(nodes[0], payment_hash, false, 0x2000 | 2, []);
+
+               // Check for unknown intercept id error.
+               let (_, channel_id) = open_zero_conf_channel(&nodes[1], &nodes[2], None);
+               let unknown_intercept_id_err = nodes[1].node.forward_intercepted_htlc(intercept_id, &channel_id, nodes[2].node.get_our_node_id(), expected_outbound_amount_msat).unwrap_err();
+               assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
+               let unknown_intercept_id_err = nodes[1].node.fail_intercepted_htlc(intercept_id).unwrap_err();
+               assert_eq!(unknown_intercept_id_err , APIError::APIMisuseError { err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.0)) });
+       }
+}
index 42835ed8035637948ae89b067d229c52e9c71efc..f2166246d3c89000ab593970a3b10d5067170519 100644 (file)
@@ -694,7 +694,7 @@ fn test_0conf_channel_with_async_monitor() {
        nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_send.msgs[0]);
        commitment_signed_dance!(nodes[2], nodes[1], bs_send.commitment_msg, false);
        expect_pending_htlcs_forwardable!(nodes[2]);
-       expect_payment_received!(nodes[2], payment_hash, payment_secret, 1_000_000);
+       expect_payment_claimable!(nodes[2], payment_hash, payment_secret, 1_000_000);
        claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
 
        confirm_transaction(&nodes[0], &tx);
index dc3d53922137a720ba78366c5341bcd250164a92..2c7e8e72bbee663fd824a7f3aad5a27b1c118792 100644 (file)
 //! Functional tests which test for correct behavior across node restarts.
 
 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
+use crate::chain::chaininterface::LowerBoundedFeeEstimator;
 use crate::chain::channelmonitor::ChannelMonitor;
+use crate::chain::keysinterface::KeysInterface;
 use crate::chain::transaction::OutPoint;
 use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, PaymentId};
 use crate::ln::msgs;
 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
 use crate::util::enforcing_trait_impls::EnforcingSigner;
 use crate::util::test_utils;
-use crate::util::events::{Event, MessageSendEvent, MessageSendEventsProvider, ClosureReason};
+use crate::util::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
 use crate::util::ser::{Writeable, ReadableArgs};
 use crate::util::config::UserConfig;
 
@@ -635,7 +637,7 @@ fn test_forwardable_regen() {
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 
        expect_pending_htlcs_forwardable!(nodes[1]);
-       expect_payment_received!(nodes[1], payment_hash, payment_secret, 100_000);
+       expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 100_000);
        check_added_monitors!(nodes[1], 1);
 
        let mut events = nodes[1].node.get_and_clear_pending_msg_events();
@@ -644,7 +646,7 @@ fn test_forwardable_regen() {
        nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
        commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
        expect_pending_htlcs_forwardable!(nodes[2]);
-       expect_payment_received!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
+       expect_payment_claimable!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
 
        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
        claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
@@ -654,7 +656,7 @@ fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
        // Test what happens if a node receives an MPP payment, claims it, but crashes before
        // persisting the ChannelManager. If `persist_both_monitors` is false, also crash after only
        // updating one of the two channels' ChannelMonitors. As a result, on startup, we'll (a) still
-       // have the PaymentReceived event, (b) have one (or two) channel(s) that goes on chain with the
+       // have the PaymentClaimable event, (b) have one (or two) channel(s) that goes on chain with the
        // HTLC preimage in them, and (c) optionally have one channel that is live off-chain but does
        // not have the preimage tied to the still-pending HTLC.
        //
@@ -691,7 +693,7 @@ fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
        nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 2);
 
-       // Send the payment through to nodes[3] *without* clearing the PaymentReceived event
+       // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
        let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(send_events.len(), 2);
        do_pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), send_events[0].clone(), true, false, None);
@@ -711,7 +713,7 @@ fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
 
        let original_manager = nodes[3].node.encode();
 
-       expect_payment_received!(nodes[3], payment_hash, payment_secret, 15_000_000);
+       expect_payment_claimable!(nodes[3], payment_hash, payment_secret, 15_000_000);
 
        nodes[3].node.claim_funds(payment_preimage);
        check_added_monitors!(nodes[3], 2);
@@ -748,11 +750,11 @@ fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
        nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id(), false);
 
        // During deserialization, we should have closed one channel and broadcast its latest
-       // commitment transaction. We should also still have the original PaymentReceived event we
+       // commitment transaction. We should also still have the original PaymentClaimable event we
        // never finished processing.
        let events = nodes[3].node.get_and_clear_pending_events();
        assert_eq!(events.len(), if persist_both_monitors { 4 } else { 3 });
-       if let Event::PaymentReceived { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
+       if let Event::PaymentClaimable { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
        if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[1] { } else { panic!(); }
        if persist_both_monitors {
                if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[2] { } else { panic!(); }
@@ -786,9 +788,9 @@ fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
                let ds_msgs = nodes[3].node.get_and_clear_pending_msg_events();
                check_added_monitors!(nodes[3], 1);
                assert_eq!(ds_msgs.len(), 2);
-               if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[1] {} else { panic!(); }
+               if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[0] {} else { panic!(); }
 
-               let cs_updates = match ds_msgs[0] {
+               let cs_updates = match ds_msgs[1] {
                        MessageSendEvent::UpdateHTLCs { ref updates, .. } => {
                                nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
                                check_added_monitors!(nodes[2], 1);
@@ -811,3 +813,192 @@ fn test_partial_claim_before_restart() {
        do_test_partial_claim_before_restart(false);
        do_test_partial_claim_before_restart(true);
 }
+
+fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_htlc: bool) {
+       if !use_cs_commitment { assert!(!claim_htlc); }
+       // If we go to forward a payment, and the ChannelMonitor persistence completes, but the
+       // ChannelManager does not, we shouldn't try to forward the payment again, nor should we fail
+       // it back until the ChannelMonitor decides the fate of the HTLC.
+       // This was never an issue, but it may be easy to regress here going forward.
+       let chanmon_cfgs = create_chanmon_cfgs(3);
+       let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
+
+       let persister;
+       let new_chain_monitor;
+       let nodes_1_deserialized;
+
+       let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
+
+       let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
+       let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
+
+       let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
+       let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
+       let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
+       check_added_monitors!(nodes[0], 1);
+
+       let payment_event = SendEvent::from_node(&nodes[0]);
+       nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
+       commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
+
+       let node_encoded = nodes[1].node.encode();
+
+       expect_pending_htlcs_forwardable!(nodes[1]);
+
+       let payment_event = SendEvent::from_node(&nodes[1]);
+       nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
+       nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &payment_event.commitment_msg);
+       check_added_monitors!(nodes[2], 1);
+
+       if claim_htlc {
+               get_monitor!(nodes[2], chan_id_2).provide_payment_preimage(&payment_hash, &payment_preimage,
+                       &nodes[2].tx_broadcaster, &LowerBoundedFeeEstimator(nodes[2].fee_estimator), &nodes[2].logger);
+       }
+       assert!(nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
+
+       let _ = nodes[2].node.get_and_clear_pending_msg_events();
+
+       nodes[2].node.force_close_broadcasting_latest_txn(&chan_id_2, &nodes[1].node.get_our_node_id()).unwrap();
+       let cs_commitment_tx = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+       assert_eq!(cs_commitment_tx.len(), if claim_htlc { 2 } else { 1 });
+
+       check_added_monitors!(nodes[2], 1);
+       check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed);
+       check_closed_broadcast!(nodes[2], true);
+
+       let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
+       let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
+       reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
+
+       check_closed_event!(nodes[1], 1, ClosureReason::OutdatedChannelManager);
+
+       let bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+       assert_eq!(bs_commitment_tx.len(), 1);
+
+       nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), true);
+       reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+
+       if use_cs_commitment {
+               // If we confirm a commitment transaction that has the HTLC on-chain, nodes[1] should wait
+               // for an HTLC-spending transaction before it does anything with the HTLC upstream.
+               confirm_transaction(&nodes[1], &cs_commitment_tx[0]);
+               assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
+               assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
+
+               if claim_htlc {
+                       confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
+               } else {
+                       connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1);
+                       let bs_htlc_timeout_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+                       assert_eq!(bs_htlc_timeout_tx.len(), 1);
+                       confirm_transaction(&nodes[1], &bs_htlc_timeout_tx[0]);
+               }
+       } else {
+               confirm_transaction(&nodes[1], &bs_commitment_tx[0]);
+       }
+
+       if !claim_htlc {
+               expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
+       } else {
+               expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, true);
+       }
+       check_added_monitors!(nodes[1], 1);
+
+       let events = nodes[1].node.get_and_clear_pending_msg_events();
+       assert_eq!(events.len(), 1);
+       match &events[0] {
+               MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fulfill_htlcs, update_fail_htlcs, commitment_signed, .. }, .. } => {
+                       if claim_htlc {
+                               nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
+                       } else {
+                               nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
+                       }
+                       commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
+               },
+               _ => panic!("Unexpected event"),
+       }
+
+       if claim_htlc {
+               expect_payment_sent!(nodes[0], payment_preimage);
+       } else {
+               expect_payment_failed!(nodes[0], payment_hash, false);
+       }
+}
+
+#[test]
+fn forwarded_payment_no_manager_persistence() {
+       do_forwarded_payment_no_manager_persistence(true, true);
+       do_forwarded_payment_no_manager_persistence(true, false);
+       do_forwarded_payment_no_manager_persistence(false, false);
+}
+
+#[test]
+fn removed_payment_no_manager_persistence() {
+       // If an HTLC is failed to us on a channel, and the ChannelMonitor persistence completes, but
+       // the corresponding ChannelManager persistence does not, we need to ensure that the HTLC is
+       // still failed back to the previous hop even though the ChannelMonitor now no longer is aware
+       // of the HTLC. This was previously broken as no attempt was made to figure out which HTLCs
+       // were left dangling when a channel was force-closed due to a stale ChannelManager.
+       let chanmon_cfgs = create_chanmon_cfgs(3);
+       let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
+
+       let persister;
+       let new_chain_monitor;
+       let nodes_1_deserialized;
+
+       let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
+
+       let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
+       let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
+
+       let (_, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
+
+       let node_encoded = nodes[1].node.encode();
+
+       nodes[2].node.fail_htlc_backwards(&payment_hash);
+       expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
+       check_added_monitors!(nodes[2], 1);
+       let events = nodes[2].node.get_and_clear_pending_msg_events();
+       assert_eq!(events.len(), 1);
+       match &events[0] {
+               MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
+                       nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_htlcs[0]);
+                       commitment_signed_dance!(nodes[1], nodes[2], commitment_signed, false);
+               },
+               _ => panic!("Unexpected event"),
+       }
+
+       let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
+       let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
+       reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
+
+       match nodes[1].node.pop_pending_event().unwrap() {
+               Event::ChannelClosed { ref reason, .. } => {
+                       assert_eq!(*reason, ClosureReason::OutdatedChannelManager);
+               },
+               _ => panic!("Unexpected event"),
+       }
+
+       // Now that the ChannelManager has force-closed the channel which had the HTLC removed, it is
+       // now forgotten everywhere. The ChannelManager should have, as a side-effect of reload,
+       // learned that the HTLC is gone from the ChannelMonitor and added it to the to-fail-back set.
+       nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), true);
+       reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+
+       expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
+       check_added_monitors!(nodes[1], 1);
+       let events = nodes[1].node.get_and_clear_pending_msg_events();
+       assert_eq!(events.len(), 1);
+       match &events[0] {
+               MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
+                       nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
+                       commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
+               },
+               _ => panic!("Unexpected event"),
+       }
+
+       expect_payment_failed!(nodes[0], payment_hash, false);
+}
index f6754e8bf2dc8d6e11f0a56370117436bddd57cb..91d5b3070610579474b39737b518c94932e9200d 100644 (file)
@@ -277,6 +277,9 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, use_funding_
        assert_eq!(nodes[0].node.short_to_chan_info.read().unwrap().len(), 2);
        mem::drop(channel_state);
 
+       assert_eq!(nodes[0].node.list_channels()[0].confirmations, Some(10));
+       assert_eq!(nodes[1].node.list_channels()[0].confirmations, Some(10));
+
        if !reorg_after_reload {
                if use_funding_unconfirmed {
                        let relevant_txids = nodes[0].node.get_relevant_txids();
@@ -287,12 +290,16 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, use_funding_
                        let txid = relevant_txids[0].0;
                        assert_eq!(txid, chan.3.txid());
                        nodes[0].node.transaction_unconfirmed(&txid);
+                       assert_eq!(nodes[0].node.list_usable_channels().len(), 0);
                } else if connect_style == ConnectStyle::FullBlockViaListen {
                        disconnect_blocks(&nodes[0], CHAN_CONFIRM_DEPTH - 1);
                        assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
+                       assert_eq!(nodes[0].node.list_channels()[0].confirmations, Some(1));
                        disconnect_blocks(&nodes[0], 1);
+                       assert_eq!(nodes[0].node.list_usable_channels().len(), 0);
                } else {
                        disconnect_all_blocks(&nodes[0]);
+                       assert_eq!(nodes[0].node.list_usable_channels().len(), 0);
                }
 
                let relevant_txids = nodes[0].node.get_relevant_txids();
@@ -334,12 +341,16 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, use_funding_
                        let txid = relevant_txids[0].0;
                        assert_eq!(txid, chan.3.txid());
                        nodes[0].node.transaction_unconfirmed(&txid);
+                       assert_eq!(nodes[0].node.list_channels().len(), 0);
                } else if connect_style == ConnectStyle::FullBlockViaListen {
                        disconnect_blocks(&nodes[0], CHAN_CONFIRM_DEPTH - 1);
                        assert_eq!(nodes[0].node.list_channels().len(), 1);
+                       assert_eq!(nodes[0].node.list_channels()[0].confirmations, Some(1));
                        disconnect_blocks(&nodes[0], 1);
+                       assert_eq!(nodes[0].node.list_usable_channels().len(), 0);
                } else {
                        disconnect_all_blocks(&nodes[0]);
+                       assert_eq!(nodes[0].node.list_usable_channels().len(), 0);
                }
 
                let relevant_txids = nodes[0].node.get_relevant_txids();
index cffd9ddb0a581ea2e26a53ee3f305a8c7f81bbf7..09cee2a2e84e8b8acb2d63842c2ed054d387e9ee 100644 (file)
 //! interrogate it to get routes for your own payments.
 
 use bitcoin::secp256k1::PublicKey;
+use bitcoin::hashes::Hash;
+use bitcoin::hashes::sha256::Hash as Sha256;
 
-use crate::ln::channelmanager::ChannelDetails;
+use crate::ln::PaymentHash;
+use crate::ln::channelmanager::{ChannelDetails, PaymentId};
 use crate::ln::features::{ChannelFeatures, InvoiceFeatures, NodeFeatures};
 use crate::ln::msgs::{DecodeError, ErrorAction, LightningError, MAX_VALUE_MSAT};
 use crate::routing::gossip::{DirectedChannelInfo, EffectiveCapacity, ReadOnlyNetworkGraph, NetworkGraph, NodeId, RoutingFees};
-use crate::routing::scoring::{ChannelUsage, Score};
+use crate::routing::scoring::{ChannelUsage, LockableScore, Score};
 use crate::util::ser::{Writeable, Readable, Writer};
 use crate::util::logger::{Level, Logger};
 use crate::util::chacha20::ChaCha20;
 
 use crate::io;
 use crate::prelude::*;
+use crate::sync::Mutex;
 use alloc::collections::BinaryHeap;
 use core::cmp;
 use core::ops::Deref;
 
+/// A [`Router`] implemented using [`find_route`].
+pub struct DefaultRouter<G: Deref<Target = NetworkGraph<L>>, L: Deref, S: Deref> where
+       L::Target: Logger,
+       S::Target: for <'a> LockableScore<'a>,
+{
+       network_graph: G,
+       logger: L,
+       random_seed_bytes: Mutex<[u8; 32]>,
+       scorer: S
+}
+
+impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, S: Deref> DefaultRouter<G, L, S> where
+       L::Target: Logger,
+       S::Target: for <'a> LockableScore<'a>,
+{
+       /// Creates a new router.
+       pub fn new(network_graph: G, logger: L, random_seed_bytes: [u8; 32], scorer: S) -> Self {
+               let random_seed_bytes = Mutex::new(random_seed_bytes);
+               Self { network_graph, logger, random_seed_bytes, scorer }
+       }
+}
+
+impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, S: Deref> Router for DefaultRouter<G, L, S> where
+       L::Target: Logger,
+       S::Target: for <'a> LockableScore<'a>,
+{
+       fn find_route(
+               &self, payer: &PublicKey, params: &RouteParameters, first_hops: Option<&[&ChannelDetails]>,
+               inflight_htlcs: InFlightHtlcs
+       ) -> Result<Route, LightningError> {
+               let random_seed_bytes = {
+                       let mut locked_random_seed_bytes = self.random_seed_bytes.lock().unwrap();
+                       *locked_random_seed_bytes = Sha256::hash(&*locked_random_seed_bytes).into_inner();
+                       *locked_random_seed_bytes
+               };
+
+               find_route(
+                       payer, params, &self.network_graph, first_hops, &*self.logger,
+                       &ScorerAccountingForInFlightHtlcs::new(&mut self.scorer.lock(), inflight_htlcs),
+                       &random_seed_bytes
+               )
+       }
+
+       fn notify_payment_path_failed(&self, path: &[&RouteHop], short_channel_id: u64) {
+               self.scorer.lock().payment_path_failed(path, short_channel_id);
+       }
+
+       fn notify_payment_path_successful(&self, path: &[&RouteHop]) {
+               self.scorer.lock().payment_path_successful(path);
+       }
+
+       fn notify_payment_probe_successful(&self, path: &[&RouteHop]) {
+               self.scorer.lock().probe_successful(path);
+       }
+
+       fn notify_payment_probe_failed(&self, path: &[&RouteHop], short_channel_id: u64) {
+               self.scorer.lock().probe_failed(path, short_channel_id);
+       }
+}
+
 /// A trait defining behavior for routing a payment.
 pub trait Router {
        /// Finds a [`Route`] between `payer` and `payee` for a payment with the given values.
@@ -36,22 +100,122 @@ pub trait Router {
                &self, payer: &PublicKey, route_params: &RouteParameters,
                first_hops: Option<&[&ChannelDetails]>, inflight_htlcs: InFlightHtlcs
        ) -> Result<Route, LightningError>;
+       /// Finds a [`Route`] between `payer` and `payee` for a payment with the given values. Includes
+       /// `PaymentHash` and `PaymentId` to be able to correlate the request with a specific payment.
+       fn find_route_with_id(
+               &self, payer: &PublicKey, route_params: &RouteParameters,
+               first_hops: Option<&[&ChannelDetails]>, inflight_htlcs: InFlightHtlcs,
+               _payment_hash: PaymentHash, _payment_id: PaymentId
+       ) -> Result<Route, LightningError> {
+               self.find_route(payer, route_params, first_hops, inflight_htlcs)
+       }
+       /// Lets the router know that payment through a specific path has failed.
+       fn notify_payment_path_failed(&self, path: &[&RouteHop], short_channel_id: u64);
+       /// Lets the router know that payment through a specific path was successful.
+       fn notify_payment_path_successful(&self, path: &[&RouteHop]);
+       /// Lets the router know that a payment probe was successful.
+       fn notify_payment_probe_successful(&self, path: &[&RouteHop]);
+       /// Lets the router know that a payment probe failed.
+       fn notify_payment_probe_failed(&self, path: &[&RouteHop], short_channel_id: u64);
 }
 
-/// A map with liquidity value (in msat) keyed by a short channel id and the direction the HTLC
-/// is traveling in. The direction boolean is determined by checking if the HTLC source's public
-/// key is less than its destination. See [`InFlightHtlcs::used_liquidity_msat`] for more
-/// details.
-#[cfg(not(any(test, feature = "_test_utils")))]
-pub struct InFlightHtlcs(HashMap<(u64, bool), u64>);
-#[cfg(any(test, feature = "_test_utils"))]
-pub struct InFlightHtlcs(pub HashMap<(u64, bool), u64>);
+/// [`Score`] implementation that factors in in-flight HTLC liquidity.
+///
+/// Useful for custom [`Router`] implementations to wrap their [`Score`] on-the-fly when calling
+/// [`find_route`].
+///
+/// [`Score`]: crate::routing::scoring::Score
+pub struct ScorerAccountingForInFlightHtlcs<'a, S: Score> {
+       scorer: &'a mut S,
+       // Maps a channel's short channel id and its direction to the liquidity used up.
+       inflight_htlcs: InFlightHtlcs,
+}
+
+impl<'a, S: Score> ScorerAccountingForInFlightHtlcs<'a, S> {
+       /// Initialize a new `ScorerAccountingForInFlightHtlcs`.
+       pub fn new(scorer: &'a mut S, inflight_htlcs: InFlightHtlcs) -> Self {
+               ScorerAccountingForInFlightHtlcs {
+                       scorer,
+                       inflight_htlcs
+               }
+       }
+}
+
+#[cfg(c_bindings)]
+impl<'a, S:Score> Writeable for ScorerAccountingForInFlightHtlcs<'a, S> {
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> { self.scorer.write(writer) }
+}
+
+impl<'a, S: Score> Score for ScorerAccountingForInFlightHtlcs<'a, S> {
+       fn channel_penalty_msat(&self, short_channel_id: u64, source: &NodeId, target: &NodeId, usage: ChannelUsage) -> u64 {
+               if let Some(used_liquidity) = self.inflight_htlcs.used_liquidity_msat(
+                       source, target, short_channel_id
+               ) {
+                       let usage = ChannelUsage {
+                               inflight_htlc_msat: usage.inflight_htlc_msat + used_liquidity,
+                               ..usage
+                       };
+
+                       self.scorer.channel_penalty_msat(short_channel_id, source, target, usage)
+               } else {
+                       self.scorer.channel_penalty_msat(short_channel_id, source, target, usage)
+               }
+       }
+
+       fn payment_path_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) {
+               self.scorer.payment_path_failed(path, short_channel_id)
+       }
+
+       fn payment_path_successful(&mut self, path: &[&RouteHop]) {
+               self.scorer.payment_path_successful(path)
+       }
+
+       fn probe_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) {
+               self.scorer.probe_failed(path, short_channel_id)
+       }
+
+       fn probe_successful(&mut self, path: &[&RouteHop]) {
+               self.scorer.probe_successful(path)
+       }
+}
+
+/// A data structure for tracking in-flight HTLCs. May be used during pathfinding to account for
+/// in-use channel liquidity.
+pub struct InFlightHtlcs(
+       // A map with liquidity value (in msat) keyed by a short channel id and the direction the HTLC
+       // is traveling in. The direction boolean is determined by checking if the HTLC source's public
+       // key is less than its destination. See `InFlightHtlcs::used_liquidity_msat` for more
+       // details.
+       HashMap<(u64, bool), u64>
+);
 
 impl InFlightHtlcs {
-       /// Create a new `InFlightHtlcs` via a mapping from:
-       /// (short_channel_id, source_pubkey < target_pubkey) -> used_liquidity_msat
-       pub fn new(inflight_map: HashMap<(u64, bool), u64>) -> Self {
-               InFlightHtlcs(inflight_map)
+       /// Constructs an empty `InFlightHtlcs`.
+       pub fn new() -> Self { InFlightHtlcs(HashMap::new()) }
+
+       /// Takes in a path with payer's node id and adds the path's details to `InFlightHtlcs`.
+       pub fn process_path(&mut self, path: &[RouteHop], payer_node_id: PublicKey) {
+               if path.is_empty() { return };
+               // total_inflight_map needs to be direction-sensitive when keeping track of the HTLC value
+               // that is held up. However, the `hops` array, which is a path returned by `find_route` in
+               // the router excludes the payer node. In the following lines, the payer's information is
+               // hardcoded with an inflight value of 0 so that we can correctly represent the first hop
+               // in our sliding window of two.
+               let reversed_hops_with_payer = path.iter().rev().skip(1)
+                       .map(|hop| hop.pubkey)
+                       .chain(core::iter::once(payer_node_id));
+               let mut cumulative_msat = 0;
+
+               // Taking the reversed vector from above, we zip it with just the reversed hops list to
+               // work "backwards" of the given path, since the last hop's `fee_msat` actually represents
+               // the total amount sent.
+               for (next_hop, prev_hop) in path.iter().rev().zip(reversed_hops_with_payer) {
+                       cumulative_msat += next_hop.fee_msat;
+                       self.0
+                               .entry((next_hop.short_channel_id, NodeId::from_pubkey(&prev_hop) < NodeId::from_pubkey(&next_hop.pubkey)))
+                               .and_modify(|used_liquidity_msat| *used_liquidity_msat += cumulative_msat)
+                               .or_insert(cumulative_msat);
+               }
        }
 
        /// Returns liquidity in msat given the public key of the HTLC source, target, and short channel
@@ -2025,6 +2189,7 @@ mod tests {
                        inbound_capacity_msat: 42,
                        unspendable_punishment_reserve: None,
                        confirmations_required: None,
+                       confirmations: None,
                        force_close_spend_delay: None,
                        is_outbound: true, is_channel_ready: true,
                        is_usable: true, is_public: true,
@@ -5539,6 +5704,7 @@ mod benches {
                        inbound_capacity_msat: 0,
                        unspendable_punishment_reserve: None,
                        confirmations_required: None,
+                       confirmations: None,
                        force_close_spend_delay: None,
                        is_outbound: true,
                        is_channel_ready: true,
index 1ab6384e3b8ebe997b3e4c511ddcaf721946e935..419a6b73a697621bbb825865f440d2a04b26641a 100644 (file)
@@ -17,34 +17,6 @@ pub fn slice_to_be48(v: &[u8]) -> u64 {
        ((v[5] as u64) << 8*0)
 }
 #[inline]
-pub fn slice_to_be64(v: &[u8]) -> u64 {
-       ((v[0] as u64) << 8*7) |
-       ((v[1] as u64) << 8*6) |
-       ((v[2] as u64) << 8*5) |
-       ((v[3] as u64) << 8*4) |
-       ((v[4] as u64) << 8*3) |
-       ((v[5] as u64) << 8*2) |
-       ((v[6] as u64) << 8*1) |
-       ((v[7] as u64) << 8*0)
-}
-
-#[inline]
-pub fn be16_to_array(u: u16) -> [u8; 2] {
-       let mut v = [0; 2];
-       v[0] = ((u >> 8*1) & 0xff) as u8;
-       v[1] = ((u >> 8*0) & 0xff) as u8;
-       v
-}
-#[inline]
-pub fn be32_to_array(u: u32) -> [u8; 4] {
-       let mut v = [0; 4];
-       v[0] = ((u >> 8*3) & 0xff) as u8;
-       v[1] = ((u >> 8*2) & 0xff) as u8;
-       v[2] = ((u >> 8*1) & 0xff) as u8;
-       v[3] = ((u >> 8*0) & 0xff) as u8;
-       v
-}
-#[inline]
 pub fn be48_to_array(u: u64) -> [u8; 6] {
        assert!(u & 0xffff_0000_0000_0000 == 0);
        let mut v = [0; 6];
@@ -56,31 +28,14 @@ pub fn be48_to_array(u: u64) -> [u8; 6] {
        v[5] = ((u >> 8*0) & 0xff) as u8;
        v
 }
-#[inline]
-pub fn be64_to_array(u: u64) -> [u8; 8] {
-       let mut v = [0; 8];
-       v[0] = ((u >> 8*7) & 0xff) as u8;
-       v[1] = ((u >> 8*6) & 0xff) as u8;
-       v[2] = ((u >> 8*5) & 0xff) as u8;
-       v[3] = ((u >> 8*4) & 0xff) as u8;
-       v[4] = ((u >> 8*3) & 0xff) as u8;
-       v[5] = ((u >> 8*2) & 0xff) as u8;
-       v[6] = ((u >> 8*1) & 0xff) as u8;
-       v[7] = ((u >> 8*0) & 0xff) as u8;
-       v
-}
 
 #[cfg(test)]
 mod tests {
        use super::*;
-       
+
        #[test]
        fn test_all() {
                assert_eq!(slice_to_be48(&[0xde, 0xad, 0xbe, 0xef, 0x1b, 0xad]), 0xdeadbeef1bad);
-               assert_eq!(slice_to_be64(&[0xde, 0xad, 0xbe, 0xef, 0x1b, 0xad, 0x1d, 0xea]), 0xdeadbeef1bad1dea);
-               assert_eq!(be16_to_array(0xdead), [0xde, 0xad]);
-               assert_eq!(be32_to_array(0xdeadbeef), [0xde, 0xad, 0xbe, 0xef]);
                assert_eq!(be48_to_array(0xdeadbeef1bad), [0xde, 0xad, 0xbe, 0xef, 0x1b, 0xad]);
-               assert_eq!(be64_to_array(0xdeadbeef1bad1dea), [0xde, 0xad, 0xbe, 0xef, 0x1b, 0xad, 0x1d, 0xea]);
        }
 }
index 26c8da7543615777ec814a88adff9163b147ddd0..c9c76f4e9fa5480e2c8a6be0a642b2192db903ca 100644 (file)
@@ -505,6 +505,17 @@ pub struct UserConfig {
        /// [`msgs::OpenChannel`]: crate::ln::msgs::OpenChannel
        /// [`msgs::AcceptChannel`]: crate::ln::msgs::AcceptChannel
        pub manually_accept_inbound_channels: bool,
+       ///  If this is set to true, LDK will intercept HTLCs that are attempting to be forwarded over
+       ///  fake short channel ids generated via [`ChannelManager::get_intercept_scid`]. Upon HTLC
+       ///  intercept, LDK will generate an [`Event::HTLCIntercepted`] which MUST be handled by the user.
+       ///
+       ///  Setting this to true may break backwards compatibility with LDK versions < 0.0.113.
+       ///
+       ///  Default value: false.
+       ///
+       /// [`ChannelManager::get_intercept_scid`]: crate::ln::channelmanager::ChannelManager::get_intercept_scid
+       /// [`Event::HTLCIntercepted`]: crate::util::events::Event::HTLCIntercepted
+       pub accept_intercept_htlcs: bool,
 }
 
 impl Default for UserConfig {
@@ -516,6 +527,7 @@ impl Default for UserConfig {
                        accept_forwards_to_priv_channels: false,
                        accept_inbound_channels: true,
                        manually_accept_inbound_channels: false,
+                       accept_intercept_htlcs: false,
                }
        }
 }
index 6317b43d5ed3b406f13f2f0f932bf59c6b07947c..1a9038a2f5dc1c3c2122e71f8f6a5b0d4e945782 100644 (file)
@@ -23,6 +23,8 @@ use bitcoin::util::sighash;
 use bitcoin::secp256k1;
 use bitcoin::secp256k1::{SecretKey, PublicKey};
 use bitcoin::secp256k1::{Secp256k1, ecdsa::Signature};
+#[cfg(anchors)]
+use crate::util::events::HTLCDescriptor;
 use crate::util::ser::{Writeable, Writer};
 use crate::io::Error;
 
@@ -157,7 +159,7 @@ impl BaseSign for EnforcingSigner {
                for (this_htlc, sig) in trusted_tx.htlcs().iter().zip(&commitment_tx.counterparty_htlc_sigs) {
                        assert!(this_htlc.transaction_output_index.is_some());
                        let keys = trusted_tx.keys();
-                       let htlc_tx = chan_utils::build_htlc_transaction(&commitment_txid, trusted_tx.feerate_per_kw(), holder_csv, &this_htlc, self.opt_anchors(), &keys.broadcaster_delayed_payment_key, &keys.revocation_key);
+                       let htlc_tx = chan_utils::build_htlc_transaction(&commitment_txid, trusted_tx.feerate_per_kw(), holder_csv, &this_htlc, self.opt_anchors(), false, &keys.broadcaster_delayed_payment_key, &keys.revocation_key);
 
                        let htlc_redeemscript = chan_utils::get_htlc_redeemscript(&this_htlc, self.opt_anchors(), &keys);
 
@@ -190,6 +192,17 @@ impl BaseSign for EnforcingSigner {
                Ok(self.inner.sign_justice_revoked_htlc(justice_tx, input, amount, per_commitment_key, htlc, secp_ctx).unwrap())
        }
 
+       #[cfg(anchors)]
+       fn sign_holder_htlc_transaction(
+               &self, htlc_tx: &Transaction, input: usize, htlc_descriptor: &HTLCDescriptor,
+               secp_ctx: &Secp256k1<secp256k1::All>
+       ) -> Result<Signature, ()> {
+               let per_commitment_point = self.get_per_commitment_point(htlc_descriptor.per_commitment_number, secp_ctx);
+               assert_eq!(htlc_tx.input[input], htlc_descriptor.unsigned_tx_input());
+               assert_eq!(htlc_tx.output[input], htlc_descriptor.tx_output(&per_commitment_point, secp_ctx));
+               Ok(self.inner.sign_holder_htlc_transaction(htlc_tx, input, htlc_descriptor, secp_ctx).unwrap())
+       }
+
        fn sign_counterparty_htlc_transaction(&self, htlc_tx: &Transaction, input: usize, amount: u64, per_commitment_point: &PublicKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
                Ok(self.inner.sign_counterparty_htlc_transaction(htlc_tx, input, amount, per_commitment_point, htlc, secp_ctx).unwrap())
        }
@@ -215,8 +228,8 @@ impl BaseSign for EnforcingSigner {
                self.inner.sign_channel_announcement(msg, secp_ctx)
        }
 
-       fn ready_channel(&mut self, channel_parameters: &ChannelTransactionParameters) {
-               self.inner.ready_channel(channel_parameters)
+       fn provide_channel_parameters(&mut self, channel_parameters: &ChannelTransactionParameters) {
+               self.inner.provide_channel_parameters(channel_parameters)
        }
 }
 
index 092f104b1a7a0b8897a094d21ec9c0a29f5d2cd7..17e55a9799bb503c987782a2937c52a00634abdf 100644 (file)
@@ -35,7 +35,7 @@ pub enum APIError {
        },
        /// A malformed Route was provided (eg overflowed value, node id mismatch, overly-looped route,
        /// too-many-hops, etc).
-       RouteError {
+       InvalidRoute {
                /// A human-readable error message
                err: &'static str
        },
@@ -74,7 +74,7 @@ impl fmt::Debug for APIError {
                match *self {
                        APIError::APIMisuseError {ref err} => write!(f, "Misuse error: {}", err),
                        APIError::FeeRateTooHigh {ref err, ref feerate} => write!(f, "{} feerate: {}", err, feerate),
-                       APIError::RouteError {ref err} => write!(f, "Route error: {}", err),
+                       APIError::InvalidRoute {ref err} => write!(f, "Invalid route provided: {}", err),
                        APIError::ChannelUnavailable {ref err} => write!(f, "Channel unavailable: {}", err),
                        APIError::MonitorUpdateInProgress => f.write_str("Client indicated a channel monitor update is in progress but not yet complete"),
                        APIError::IncompatibleShutdownScript { ref script } => {
index 3593cbf97e6f69ad54fb92010903b56993beec09..2c7e5413dafff9d9f027eab116d0967faae6ed2e 100644 (file)
@@ -16,8 +16,8 @@
 
 use crate::chain::keysinterface::SpendableOutputDescriptor;
 #[cfg(anchors)]
-use crate::ln::chan_utils::HTLCOutputInCommitment;
-use crate::ln::channelmanager::PaymentId;
+use crate::ln::chan_utils::{self, ChannelTransactionParameters, HTLCOutputInCommitment};
+use crate::ln::channelmanager::{InterceptId, PaymentId};
 use crate::ln::channel::FUNDING_CONF_DEADLINE_BLOCKS;
 use crate::ln::features::ChannelTypeFeatures;
 use crate::ln::msgs;
@@ -29,11 +29,15 @@ use crate::routing::router::{RouteHop, RouteParameters};
 
 use bitcoin::{PackedLockTime, Transaction};
 #[cfg(anchors)]
-use bitcoin::OutPoint;
+use bitcoin::{OutPoint, Txid, TxIn, TxOut, Witness};
 use bitcoin::blockdata::script::Script;
 use bitcoin::hashes::Hash;
 use bitcoin::hashes::sha256::Hash as Sha256;
 use bitcoin::secp256k1::PublicKey;
+#[cfg(anchors)]
+use bitcoin::secp256k1::{self, Secp256k1};
+#[cfg(anchors)]
+use bitcoin::secp256k1::ecdsa::Signature;
 use crate::io;
 use crate::prelude::*;
 use core::time::Duration;
@@ -182,6 +186,12 @@ pub enum HTLCDestination {
                /// Short channel id we are requesting to forward an HTLC to.
                requested_forward_scid: u64,
        },
+       /// We couldn't forward to the outgoing scid. An example would be attempting to send a duplicate
+       /// intercept HTLC.
+       InvalidForward {
+               /// Short channel id we are requesting to forward an HTLC to.
+               requested_forward_scid: u64
+       },
        /// Failure scenario where an HTLC may have been forwarded to be intended for us,
        /// but is invalid for some reason, so we reject it.
        ///
@@ -200,12 +210,15 @@ impl_writeable_tlv_based_enum_upgradable!(HTLCDestination,
                (0, node_id, required),
                (2, channel_id, required),
        },
+       (1, InvalidForward) => {
+               (0, requested_forward_scid, required),
+       },
        (2, UnknownNextHop) => {
                (0, requested_forward_scid, required),
        },
        (4, FailedPayment) => {
                (0, payment_hash, required),
-       }
+       },
 );
 
 #[cfg(anchors)]
@@ -228,6 +241,99 @@ pub struct AnchorDescriptor {
        pub outpoint: OutPoint,
 }
 
+#[cfg(anchors)]
+/// A descriptor used to sign for a commitment transaction's HTLC output.
+#[derive(Clone, Debug)]
+pub struct HTLCDescriptor {
+       /// A unique identifier used along with `channel_value_satoshis` to re-derive the
+       /// [`InMemorySigner`] required to sign `input`.
+       ///
+       /// [`InMemorySigner`]: crate::chain::keysinterface::InMemorySigner
+       pub channel_keys_id: [u8; 32],
+       /// The value in satoshis of the channel we're attempting to spend the anchor output of. This is
+       /// used along with `channel_keys_id` to re-derive the [`InMemorySigner`] required to sign
+       /// `input`.
+       ///
+       /// [`InMemorySigner`]: crate::chain::keysinterface::InMemorySigner
+       pub channel_value_satoshis: u64,
+       /// The necessary channel parameters that need to be provided to the re-derived
+       /// [`InMemorySigner`] through [`BaseSign::ready_channel`].
+       ///
+       /// [`InMemorySigner`]: crate::chain::keysinterface::InMemorySigner
+       /// [`BaseSign::ready_channel`]: crate::chain::keysinterface::BaseSign::ready_channel
+       pub channel_parameters: ChannelTransactionParameters,
+       /// The txid of the commitment transaction in which the HTLC output lives.
+       pub commitment_txid: Txid,
+       /// The number of the commitment transaction in which the HTLC output lives.
+       pub per_commitment_number: u64,
+       /// The details of the HTLC as it appears in the commitment transaction.
+       pub htlc: HTLCOutputInCommitment,
+       /// The preimage, if `Some`, to claim the HTLC output with. If `None`, the timeout path must be
+       /// taken.
+       pub preimage: Option<PaymentPreimage>,
+       /// The counterparty's signature required to spend the HTLC output.
+       pub counterparty_sig: Signature
+}
+
+#[cfg(anchors)]
+impl HTLCDescriptor {
+       /// Returns the unsigned transaction input spending the HTLC output in the commitment
+       /// transaction.
+       pub fn unsigned_tx_input(&self) -> TxIn {
+               chan_utils::build_htlc_input(&self.commitment_txid, &self.htlc, true /* opt_anchors */)
+       }
+
+       /// Returns the delayed output created as a result of spending the HTLC output in the commitment
+       /// transaction.
+       pub fn tx_output<C: secp256k1::Signing + secp256k1::Verification>(
+               &self, per_commitment_point: &PublicKey, secp: &Secp256k1<C>
+       ) -> TxOut {
+               let channel_params = self.channel_parameters.as_holder_broadcastable();
+               let broadcaster_keys = channel_params.broadcaster_pubkeys();
+               let counterparty_keys = channel_params.countersignatory_pubkeys();
+               let broadcaster_delayed_key = chan_utils::derive_public_key(
+                       secp, per_commitment_point, &broadcaster_keys.delayed_payment_basepoint
+               );
+               let counterparty_revocation_key = chan_utils::derive_public_revocation_key(
+                       secp, per_commitment_point, &counterparty_keys.revocation_basepoint
+               );
+               chan_utils::build_htlc_output(
+                       0 /* feerate_per_kw */, channel_params.contest_delay(), &self.htlc, true /* opt_anchors */,
+                       false /* use_non_zero_fee_anchors */, &broadcaster_delayed_key, &counterparty_revocation_key
+               )
+       }
+
+       /// Returns the witness script of the HTLC output in the commitment transaction.
+       pub fn witness_script<C: secp256k1::Signing + secp256k1::Verification>(
+               &self, per_commitment_point: &PublicKey, secp: &Secp256k1<C>
+       ) -> Script {
+               let channel_params = self.channel_parameters.as_holder_broadcastable();
+               let broadcaster_keys = channel_params.broadcaster_pubkeys();
+               let counterparty_keys = channel_params.countersignatory_pubkeys();
+               let broadcaster_htlc_key = chan_utils::derive_public_key(
+                       secp, per_commitment_point, &broadcaster_keys.htlc_basepoint
+               );
+               let counterparty_htlc_key = chan_utils::derive_public_key(
+                       secp, per_commitment_point, &counterparty_keys.htlc_basepoint
+               );
+               let counterparty_revocation_key = chan_utils::derive_public_revocation_key(
+                       secp, per_commitment_point, &counterparty_keys.revocation_basepoint
+               );
+               chan_utils::get_htlc_redeemscript_with_explicit_keys(
+                       &self.htlc, true /* opt_anchors */, &broadcaster_htlc_key, &counterparty_htlc_key,
+                       &counterparty_revocation_key,
+               )
+       }
+
+       /// Returns the fully signed witness required to spend the HTLC output in the commitment
+       /// transaction.
+       pub fn tx_input_witness(&self, signature: &Signature, witness_script: &Script) -> Witness {
+               chan_utils::build_htlc_input_witness(
+                       signature, &self.counterparty_sig, &self.preimage, witness_script, true /* opt_anchors */
+               )
+       }
+}
+
 #[cfg(anchors)]
 /// Represents the different types of transactions, originating from LDK, to be bumped.
 #[derive(Clone, Debug)]
@@ -240,14 +346,17 @@ pub enum BumpTransactionEvent {
        /// with additional inputs to meet the target feerate. Failure to meet the target feerate
        /// decreases the confirmation odds of the transaction package (which includes the commitment
        /// and child anchor transactions), possibly resulting in a loss of funds. Once the transaction
-       /// is constructed, it must be fully signed for and broadcasted by the consumer of the event
+       /// is constructed, it must be fully signed for and broadcast by the consumer of the event
        /// along with the `commitment_tx` enclosed. Note that the `commitment_tx` must always be
        /// broadcast first, as the child anchor transaction depends on it.
        ///
        /// The consumer should be able to sign for any of the additional inputs included within the
        /// child anchor transaction. To sign its anchor input, an [`InMemorySigner`] should be
        /// re-derived through [`KeysManager::derive_channel_keys`] with the help of
-       /// [`AnchorDescriptor::channel_keys_id`] and [`AnchorDescriptor::channel_value_satoshis`].
+       /// [`AnchorDescriptor::channel_keys_id`] and [`AnchorDescriptor::channel_value_satoshis`]. The
+       /// anchor input signature can be computed with [`BaseSign::sign_holder_anchor_input`],
+       /// which can then be provided to [`build_anchor_input_witness`] along with the `funding_pubkey`
+       /// to obtain the full witness required to spend.
        ///
        /// It is possible to receive more than one instance of this event if a valid child anchor
        /// transaction is never broadcast or is but not with a sufficient fee to be mined. Care should
@@ -268,6 +377,8 @@ pub enum BumpTransactionEvent {
        ///
        /// [`InMemorySigner`]: crate::chain::keysinterface::InMemorySigner
        /// [`KeysManager::derive_channel_keys`]: crate::chain::keysinterface::KeysManager::derive_channel_keys
+       /// [`BaseSign::sign_holder_anchor_input`]: crate::chain::keysinterface::BaseSign::sign_holder_anchor_input
+       /// [`build_anchor_input_witness`]: crate::ln::chan_utils::build_anchor_input_witness
        ChannelClose {
                /// The target feerate that the transaction package, which consists of the commitment
                /// transaction and the to-be-crafted child anchor transaction, must meet.
@@ -286,8 +397,59 @@ pub enum BumpTransactionEvent {
                /// commitment transaction confirms.
                pending_htlcs: Vec<HTLCOutputInCommitment>,
        },
+       /// Indicates that a channel featuring anchor outputs has unilaterally closed on-chain by a
+       /// holder commitment transaction and its HTLC(s) need to be resolved on-chain. With the
+       /// zero-HTLC-transaction-fee variant of anchor outputs, the pre-signed HTLC
+       /// transactions have a zero fee, thus requiring additional inputs and/or outputs to be attached
+       /// for a timely confirmation within the chain. These additional inputs and/or outputs must be
+       /// appended to the resulting HTLC transaction to meet the target feerate. Failure to meet the
+       /// target feerate decreases the confirmation odds of the transaction, possibly resulting in a
+       /// loss of funds. Once the transaction meets the target feerate, it must be signed for and
+       /// broadcast by the consumer of the event.
+       ///
+       /// The consumer should be able to sign for any of the non-HTLC inputs added to the resulting
+       /// HTLC transaction. To sign HTLC inputs, an [`InMemorySigner`] should be re-derived through
+       /// [`KeysManager::derive_channel_keys`] with the help of `channel_keys_id` and
+       /// `channel_value_satoshis`. Each HTLC input's signature can be computed with
+       /// [`BaseSign::sign_holder_htlc_transaction`], which can then be provided to
+       /// [`HTLCDescriptor::tx_input_witness`] to obtain the fully signed witness required to spend.
+       ///
+       /// It is possible to receive more than one instance of this event if a valid HTLC transaction
+       /// is never broadcast or is but not with a sufficient fee to be mined. Care should be taken by
+       /// the consumer of the event to ensure any future iterations of the HTLC transaction adhere to
+       /// the [Replace-By-Fee
+       /// rules](https://github.com/bitcoin/bitcoin/blob/master/doc/policy/mempool-replacements.md)
+       /// for fee bumps to be accepted into the mempool, and eventually the chain. As the frequency of
+       /// these events is not user-controlled, users may ignore/drop the event if either they are no
+       /// longer able to commit external confirmed funds to the HTLC transaction or the fee committed
+       /// to the HTLC transaction is greater in value than the HTLCs being claimed.
+       ///
+       /// [`InMemorySigner`]: crate::chain::keysinterface::InMemorySigner
+       /// [`KeysManager::derive_channel_keys`]: crate::chain::keysinterface::KeysManager::derive_channel_keys
+       /// [`BaseSign::sign_holder_htlc_transaction`]: crate::chain::keysinterface::BaseSign::sign_holder_htlc_transaction
+       /// [`HTLCDescriptor::tx_input_witness`]: HTLCDescriptor::tx_input_witness
+       HTLCResolution {
+               target_feerate_sat_per_1000_weight: u32,
+               htlc_descriptors: Vec<HTLCDescriptor>,
+       },
+}
+
+/// Will be used in [`Event::HTLCIntercepted`] to identify the next hop in the HTLC's path.
+/// Currently only used in serialization for the sake of maintaining compatibility. More variants
+/// will be added for general-purpose HTLC forward intercepts as well as trampoline forward
+/// intercepts in upcoming work.
+enum InterceptNextHop {
+       FakeScid {
+               requested_next_hop_scid: u64,
+       },
 }
 
+impl_writeable_tlv_based_enum!(InterceptNextHop,
+       (0, FakeScid) => {
+               (0, requested_next_hop_scid, required),
+       };
+);
+
 /// An Event which you should probably take some action in response to.
 ///
 /// Note that while Writeable and Readable are implemented for Event, you probably shouldn't use
@@ -325,8 +487,8 @@ pub enum Event {
                /// [`ChannelManager::create_channel`]: crate::ln::channelmanager::ChannelManager::create_channel
                user_channel_id: u128,
        },
-       /// Indicates we've received (an offer of) money! Just gotta dig out that payment preimage and
-       /// feed it to [`ChannelManager::claim_funds`] to get it....
+       /// Indicates that we've been offered a payment and it needs to be claimed via calling
+       /// [`ChannelManager::claim_funds`] with the preimage given in [`PaymentPurpose`].
        ///
        /// Note that if the preimage is not known, you should call
        /// [`ChannelManager::fail_htlc_backwards`] to free up resources for this HTLC and avoid
@@ -337,11 +499,21 @@ pub enum Event {
        ///
        /// # Note
        /// LDK will not stop an inbound payment from being paid multiple times, so multiple
-       /// `PaymentReceived` events may be generated for the same payment.
+       /// `PaymentClaimable` events may be generated for the same payment.
+       ///
+       /// # Note
+       /// This event used to be called `PaymentReceived` in LDK versions 0.0.112 and earlier.
        ///
        /// [`ChannelManager::claim_funds`]: crate::ln::channelmanager::ChannelManager::claim_funds
        /// [`ChannelManager::fail_htlc_backwards`]: crate::ln::channelmanager::ChannelManager::fail_htlc_backwards
-       PaymentReceived {
+       PaymentClaimable {
+               /// The node that will receive the payment after it has been claimed.
+               /// This is useful to identify payments received via [phantom nodes].
+               /// This field will always be filled in when the event was generated by LDK versions
+               /// 0.0.113 and above.
+               ///
+               /// [phantom nodes]: crate::chain::keysinterface::PhantomKeysManager
+               receiver_node_id: Option<PublicKey>,
                /// The hash for which the preimage should be handed to the ChannelManager. Note that LDK will
                /// not stop you from registering duplicate payment hashes for inbound payments.
                payment_hash: PaymentHash,
@@ -350,28 +522,39 @@ pub enum Event {
                /// Information for claiming this received payment, based on whether the purpose of the
                /// payment is to pay an invoice or to send a spontaneous payment.
                purpose: PaymentPurpose,
+               /// The `channel_id` indicating over which channel we received the payment.
+               via_channel_id: Option<[u8; 32]>,
+               /// The `user_channel_id` indicating over which channel we received the payment.
+               via_user_channel_id: Option<u128>,
        },
        /// Indicates a payment has been claimed and we've received money!
        ///
        /// This most likely occurs when [`ChannelManager::claim_funds`] has been called in response
-       /// to an [`Event::PaymentReceived`]. However, if we previously crashed during a
+       /// to an [`Event::PaymentClaimable`]. However, if we previously crashed during a
        /// [`ChannelManager::claim_funds`] call you may see this event without a corresponding
-       /// [`Event::PaymentReceived`] event.
+       /// [`Event::PaymentClaimable`] event.
        ///
        /// # Note
        /// LDK will not stop an inbound payment from being paid multiple times, so multiple
-       /// `PaymentReceived` events may be generated for the same payment. If you then call
-       /// [`ChannelManager::claim_funds`] twice for the same [`Event::PaymentReceived`] you may get
+       /// `PaymentClaimable` events may be generated for the same payment. If you then call
+       /// [`ChannelManager::claim_funds`] twice for the same [`Event::PaymentClaimable`] you may get
        /// multiple `PaymentClaimed` events.
        ///
        /// [`ChannelManager::claim_funds`]: crate::ln::channelmanager::ChannelManager::claim_funds
        PaymentClaimed {
+               /// The node that received the payment.
+               /// This is useful to identify payments which were received via [phantom nodes].
+               /// This field will always be filled in when the event was generated by LDK versions
+               /// 0.0.113 and above.
+               ///
+               /// [phantom nodes]: crate::chain::keysinterface::PhantomKeysManager
+               receiver_node_id: Option<PublicKey>,
                /// The payment hash of the claimed payment. Note that LDK will not stop you from
                /// registering duplicate payment hashes for inbound payments.
                payment_hash: PaymentHash,
                /// The value, in thousandths of a satoshi, that this payment is for.
                amount_msat: u64,
-               /// The purpose of this claimed payment, i.e. whether the payment was for an invoice or a
+               /// The purpose of the claimed payment, i.e. whether the payment was for an invoice or a
                /// spontaneous payment.
                purpose: PaymentPurpose,
        },
@@ -567,6 +750,38 @@ pub enum Event {
                /// now + 5*time_forwardable).
                time_forwardable: Duration,
        },
+       /// Used to indicate that we've intercepted an HTLC forward. This event will only be generated if
+       /// you've encoded an intercept scid in the receiver's invoice route hints using
+       /// [`ChannelManager::get_intercept_scid`] and have set [`UserConfig::accept_intercept_htlcs`].
+       ///
+       /// [`ChannelManager::forward_intercepted_htlc`] or
+       /// [`ChannelManager::fail_intercepted_htlc`] MUST be called in response to this event. See
+       /// their docs for more information.
+       ///
+       /// [`ChannelManager::get_intercept_scid`]: crate::ln::channelmanager::ChannelManager::get_intercept_scid
+       /// [`UserConfig::accept_intercept_htlcs`]: crate::util::config::UserConfig::accept_intercept_htlcs
+       /// [`ChannelManager::forward_intercepted_htlc`]: crate::ln::channelmanager::ChannelManager::forward_intercepted_htlc
+       /// [`ChannelManager::fail_intercepted_htlc`]: crate::ln::channelmanager::ChannelManager::fail_intercepted_htlc
+       HTLCIntercepted {
+               /// An id to help LDK identify which HTLC is being forwarded or failed.
+               intercept_id: InterceptId,
+               /// The fake scid that was programmed as the next hop's scid, generated using
+               /// [`ChannelManager::get_intercept_scid`].
+               ///
+               /// [`ChannelManager::get_intercept_scid`]: crate::ln::channelmanager::ChannelManager::get_intercept_scid
+               requested_next_hop_scid: u64,
+               /// The payment hash used for this HTLC.
+               payment_hash: PaymentHash,
+               /// How many msats were received on the inbound edge of this HTLC.
+               inbound_amount_msat: u64,
+               /// How many msats the payer intended to route to the next node. Depending on the reason you are
+               /// intercepting this payment, you might take a fee by forwarding less than this amount.
+               ///
+               /// Note that LDK will NOT check that expected fees were factored into this value. You MUST
+               /// check that whatever fee you want has been included here or subtract it as required. Further,
+               /// LDK will not stop you from forwarding more than you received.
+               expected_outbound_amount_msat: u64,
+       },
        /// Used to indicate that an output which you should know how to spend was confirmed on chain
        /// and is now spendable.
        /// Such an output will *not* ever be spent by rust-lightning, and are not at risk of your
@@ -739,7 +954,7 @@ impl Writeable for Event {
                                // We never write out FundingGenerationReady events as, upon disconnection, peers
                                // drop any channels which have not yet exchanged funding_signed.
                        },
-                       &Event::PaymentReceived { ref payment_hash, ref amount_msat, ref purpose } => {
+                       &Event::PaymentClaimable { ref payment_hash, ref amount_msat, ref purpose, ref receiver_node_id, ref via_channel_id, ref via_user_channel_id } => {
                                1u8.write(writer)?;
                                let mut payment_secret = None;
                                let payment_preimage;
@@ -754,8 +969,11 @@ impl Writeable for Event {
                                }
                                write_tlv_fields!(writer, {
                                        (0, payment_hash, required),
+                                       (1, receiver_node_id, option),
                                        (2, payment_secret, option),
+                                       (3, via_channel_id, option),
                                        (4, amount_msat, required),
+                                       (5, via_user_channel_id, option),
                                        (6, 0u64, required), // user_payment_id required for compatibility with 0.0.103 and earlier
                                        (8, payment_preimage, option),
                                });
@@ -804,6 +1022,17 @@ impl Writeable for Event {
                                        (0, WithoutLength(outputs), required),
                                });
                        },
+                       &Event::HTLCIntercepted { requested_next_hop_scid, payment_hash, inbound_amount_msat, expected_outbound_amount_msat, intercept_id } => {
+                               6u8.write(writer)?;
+                               let intercept_scid = InterceptNextHop::FakeScid { requested_next_hop_scid };
+                               write_tlv_fields!(writer, {
+                                       (0, intercept_id, required),
+                                       (2, intercept_scid, required),
+                                       (4, payment_hash, required),
+                                       (6, inbound_amount_msat, required),
+                                       (8, expected_outbound_amount_msat, required),
+                               });
+                       }
                        &Event::PaymentForwarded { fee_earned_msat, prev_channel_id, claim_from_onchain_tx, next_channel_id } => {
                                7u8.write(writer)?;
                                write_tlv_fields!(writer, {
@@ -854,10 +1083,11 @@ impl Writeable for Event {
                                // We never write the OpenChannelRequest events as, upon disconnection, peers
                                // drop any channels which have not yet exchanged funding_signed.
                        },
-                       &Event::PaymentClaimed { ref payment_hash, ref amount_msat, ref purpose } => {
+                       &Event::PaymentClaimed { ref payment_hash, ref amount_msat, ref purpose, ref receiver_node_id } => {
                                19u8.write(writer)?;
                                write_tlv_fields!(writer, {
                                        (0, payment_hash, required),
+                                       (1, receiver_node_id, option),
                                        (2, purpose, required),
                                        (4, amount_msat, required),
                                });
@@ -890,9 +1120,10 @@ impl Writeable for Event {
                        &Event::BumpTransaction(ref event)=> {
                                27u8.write(writer)?;
                                match event {
-                                       // We never write the ChannelClose events as they'll be replayed upon restarting
-                                       // anyway if the commitment transaction remains unconfirmed.
+                                       // We never write the ChannelClose|HTLCResolution events as they'll be replayed
+                                       // upon restarting anyway if they remain unresolved.
                                        BumpTransactionEvent::ChannelClose { .. } => {}
+                                       BumpTransactionEvent::HTLCResolution { .. } => {}
                                }
                        }
                        &Event::ChannelReady { ref channel_id, ref user_channel_id, ref counterparty_node_id, ref channel_type } => {
@@ -923,11 +1154,17 @@ impl MaybeReadable for Event {
                                        let mut payment_preimage = None;
                                        let mut payment_secret = None;
                                        let mut amount_msat = 0;
+                                       let mut receiver_node_id = None;
                                        let mut _user_payment_id = None::<u64>; // For compatibility with 0.0.103 and earlier
+                                       let mut via_channel_id = None;
+                                       let mut via_user_channel_id = None;
                                        read_tlv_fields!(reader, {
                                                (0, payment_hash, required),
+                                               (1, receiver_node_id, option),
                                                (2, payment_secret, option),
+                                               (3, via_channel_id, option),
                                                (4, amount_msat, required),
+                                               (5, via_user_channel_id, option),
                                                (6, _user_payment_id, option),
                                                (8, payment_preimage, option),
                                        });
@@ -939,10 +1176,13 @@ impl MaybeReadable for Event {
                                                None if payment_preimage.is_some() => PaymentPurpose::SpontaneousPayment(payment_preimage.unwrap()),
                                                None => return Err(msgs::DecodeError::InvalidValue),
                                        };
-                                       Ok(Some(Event::PaymentReceived {
+                                       Ok(Some(Event::PaymentClaimable {
+                                               receiver_node_id,
                                                payment_hash,
                                                amount_msat,
                                                purpose,
+                                               via_channel_id,
+                                               via_user_channel_id,
                                        }))
                                };
                                f()
@@ -1023,6 +1263,30 @@ impl MaybeReadable for Event {
                                };
                                f()
                        },
+                       6u8 => {
+                               let mut payment_hash = PaymentHash([0; 32]);
+                               let mut intercept_id = InterceptId([0; 32]);
+                               let mut requested_next_hop_scid = InterceptNextHop::FakeScid { requested_next_hop_scid: 0 };
+                               let mut inbound_amount_msat = 0;
+                               let mut expected_outbound_amount_msat = 0;
+                               read_tlv_fields!(reader, {
+                                       (0, intercept_id, required),
+                                       (2, requested_next_hop_scid, required),
+                                       (4, payment_hash, required),
+                                       (6, inbound_amount_msat, required),
+                                       (8, expected_outbound_amount_msat, required),
+                               });
+                               let next_scid = match requested_next_hop_scid {
+                                       InterceptNextHop::FakeScid { requested_next_hop_scid: scid } => scid
+                               };
+                               Ok(Some(Event::HTLCIntercepted {
+                                       payment_hash,
+                                       requested_next_hop_scid: next_scid,
+                                       inbound_amount_msat,
+                                       expected_outbound_amount_msat,
+                                       intercept_id,
+                               }))
+                       },
                        7u8 => {
                                let f = || {
                                        let mut fee_earned_msat = None;
@@ -1117,13 +1381,16 @@ impl MaybeReadable for Event {
                                        let mut payment_hash = PaymentHash([0; 32]);
                                        let mut purpose = None;
                                        let mut amount_msat = 0;
+                                       let mut receiver_node_id = None;
                                        read_tlv_fields!(reader, {
                                                (0, payment_hash, required),
+                                               (1, receiver_node_id, option),
                                                (2, purpose, ignorable),
                                                (4, amount_msat, required),
                                        });
                                        if purpose.is_none() { return Ok(None); }
                                        Ok(Some(Event::PaymentClaimed {
+                                               receiver_node_id,
                                                payment_hash,
                                                purpose: purpose.unwrap(),
                                                amount_msat,
index 40c042a27ff702c457a1c856db2097720cd4caf3..31e53239410cc69ece83795bb3456971a855aaf3 100644 (file)
 
 use core::ops::Deref;
 use bitcoin::hashes::hex::ToHex;
-use crate::io::{self};
+use crate::io;
 use crate::routing::scoring::WriteableScore;
 
-use crate::{chain::{keysinterface::{Sign, KeysInterface}, self, transaction::{OutPoint}, chaininterface::{BroadcasterInterface, FeeEstimator}, chainmonitor::{Persist, MonitorUpdateId}, channelmonitor::{ChannelMonitor, ChannelMonitorUpdate}}, ln::channelmanager::ChannelManager, routing::gossip::NetworkGraph};
+use crate::chain;
+use crate::chain::chaininterface::{BroadcasterInterface, FeeEstimator};
+use crate::chain::chainmonitor::{Persist, MonitorUpdateId};
+use crate::chain::keysinterface::{Sign, KeysInterface};
+use crate::chain::transaction::OutPoint;
+use crate::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate};
+use crate::ln::channelmanager::ChannelManager;
+use crate::routing::gossip::NetworkGraph;
 use super::{logger::Logger, ser::Writeable};
 
 /// Trait for a key-value store for persisting some writeable object at some key
index 651b36ef32c3ce5a7d4382be73562733a5fb8a4b..1d951b8f936e49acc2cf51a028725a19c360eef9 100644 (file)
@@ -63,6 +63,8 @@ pub fn scid_from_parts(block: u64, tx_index: u64, vout_index: u64) -> Result<u64
 /// LDK has multiple reasons to generate fake short channel ids:
 /// 1) outbound SCID aliases we use for private channels
 /// 2) phantom node payments, to get an scid for the phantom node's phantom channel
+/// 3) payments intended to be intercepted will route using a fake scid (this is typically used so
+///    the forwarding node can open a JIT channel to the next hop)
 pub(crate) mod fake_scid {
        use bitcoin::hash_types::BlockHash;
        use bitcoin::hashes::hex::FromHex;
@@ -91,6 +93,7 @@ pub(crate) mod fake_scid {
        pub(crate) enum Namespace {
                Phantom,
                OutboundAlias,
+               Intercept
        }
 
        impl Namespace {
@@ -150,7 +153,7 @@ pub(crate) mod fake_scid {
                }
        }
 
-       /// Returns whether the given fake scid falls into the given namespace.
+       /// Returns whether the given fake scid falls into the phantom namespace.
        pub fn is_valid_phantom(fake_scid_rand_bytes: &[u8; 32], scid: u64, genesis_hash: &BlockHash) -> bool {
                let block_height = scid_utils::block_from_scid(&scid);
                let tx_index = scid_utils::tx_index_from_scid(&scid);
@@ -160,11 +163,21 @@ pub(crate) mod fake_scid {
                        && valid_vout == scid_utils::vout_from_scid(&scid) as u8
        }
 
+       /// Returns whether the given fake scid falls into the intercept namespace.
+       pub fn is_valid_intercept(fake_scid_rand_bytes: &[u8; 32], scid: u64, genesis_hash: &BlockHash) -> bool {
+               let block_height = scid_utils::block_from_scid(&scid);
+               let tx_index = scid_utils::tx_index_from_scid(&scid);
+               let namespace = Namespace::Intercept;
+               let valid_vout = namespace.get_encrypted_vout(block_height, tx_index, fake_scid_rand_bytes);
+               block_height >= segwit_activation_height(genesis_hash)
+                       && valid_vout == scid_utils::vout_from_scid(&scid) as u8
+       }
+
        #[cfg(test)]
        mod tests {
                use bitcoin::blockdata::constants::genesis_block;
                use bitcoin::network::constants::Network;
-               use crate::util::scid_utils::fake_scid::{is_valid_phantom, MAINNET_SEGWIT_ACTIVATION_HEIGHT, MAX_TX_INDEX, MAX_NAMESPACES, Namespace, NAMESPACE_ID_BITMASK, segwit_activation_height, TEST_SEGWIT_ACTIVATION_HEIGHT};
+               use crate::util::scid_utils::fake_scid::{is_valid_intercept, is_valid_phantom, MAINNET_SEGWIT_ACTIVATION_HEIGHT, MAX_TX_INDEX, MAX_NAMESPACES, Namespace, NAMESPACE_ID_BITMASK, segwit_activation_height, TEST_SEGWIT_ACTIVATION_HEIGHT};
                use crate::util::scid_utils;
                use crate::util::test_utils;
                use crate::sync::Arc;
@@ -174,6 +187,10 @@ pub(crate) mod fake_scid {
                        let phantom_namespace = Namespace::Phantom;
                        assert!((phantom_namespace as u8) < MAX_NAMESPACES);
                        assert!((phantom_namespace as u8) <= NAMESPACE_ID_BITMASK);
+
+                       let intercept_namespace = Namespace::Intercept;
+                       assert!((intercept_namespace as u8) < MAX_NAMESPACES);
+                       assert!((intercept_namespace as u8) <= NAMESPACE_ID_BITMASK);
                }
 
                #[test]
@@ -203,6 +220,18 @@ pub(crate) mod fake_scid {
                        assert!(!is_valid_phantom(&fake_scid_rand_bytes, invalid_fake_scid, &testnet_genesis));
                }
 
+               #[test]
+               fn test_is_valid_intercept() {
+                       let namespace = Namespace::Intercept;
+                       let fake_scid_rand_bytes = [0; 32];
+                       let testnet_genesis = genesis_block(Network::Testnet).header.block_hash();
+                       let valid_encrypted_vout = namespace.get_encrypted_vout(0, 0, &fake_scid_rand_bytes);
+                       let valid_fake_scid = scid_utils::scid_from_parts(1, 0, valid_encrypted_vout as u64).unwrap();
+                       assert!(is_valid_intercept(&fake_scid_rand_bytes, valid_fake_scid, &testnet_genesis));
+                       let invalid_fake_scid = scid_utils::scid_from_parts(1, 0, 12).unwrap();
+                       assert!(!is_valid_intercept(&fake_scid_rand_bytes, invalid_fake_scid, &testnet_genesis));
+               }
+
                #[test]
                fn test_get_fake_scid() {
                        let mainnet_genesis = genesis_block(Network::Bitcoin).header.block_hash();
index 231320ac1591dd392e8e415073c10b9a30bd2bc6..73c4015726191f5ae5ec8b95eadcf82101bc3af8 100644 (file)
@@ -679,7 +679,7 @@ macro_rules! impl_writeable_tlv_based_enum {
                                                Ok($st::$tuple_variant_name(Readable::read(reader)?))
                                        }),*
                                        _ => {
-                                               Err(DecodeError::UnknownRequiredFeature)
+                                               Err($crate::ln::msgs::DecodeError::UnknownRequiredFeature)
                                        },
                                }
                        }
index 89bf27de2805ef8c152279cc02855d114d7a4e9b..17bdb06c73f9047368e746edfd07afbbf1d37036 100644 (file)
@@ -80,7 +80,8 @@ impl keysinterface::KeysInterface for OnlyReadsKeysInterface {
        fn get_inbound_payment_key_material(&self) -> KeyMaterial { unreachable!(); }
        fn get_destination_script(&self) -> Script { unreachable!(); }
        fn get_shutdown_scriptpubkey(&self) -> ShutdownScript { unreachable!(); }
-       fn get_channel_signer(&self, _inbound: bool, _channel_value_satoshis: u64) -> EnforcingSigner { unreachable!(); }
+       fn generate_channel_keys_id(&self, _inbound: bool, _channel_value_satoshis: u64, _user_channel_id: u128) -> [u8; 32] { unreachable!(); }
+       fn derive_channel_signer(&self, _channel_value_satoshis: u64, _channel_keys_id: [u8; 32]) -> Self::Signer { unreachable!(); }
        fn get_secure_random_bytes(&self) -> [u8; 32] { [0; 32] }
 
        fn read_chan_signer(&self, mut reader: &[u8]) -> Result<Self::Signer, msgs::DecodeError> {
@@ -629,8 +630,12 @@ impl keysinterface::KeysInterface for TestKeysInterface {
                }
        }
 
-       fn get_channel_signer(&self, inbound: bool, channel_value_satoshis: u64) -> EnforcingSigner {
-               let keys = self.backing.get_channel_signer(inbound, channel_value_satoshis);
+       fn generate_channel_keys_id(&self, inbound: bool, channel_value_satoshis: u64, user_channel_id: u128) -> [u8; 32] {
+               self.backing.generate_channel_keys_id(inbound, channel_value_satoshis, user_channel_id)
+       }
+
+       fn derive_channel_signer(&self, channel_value_satoshis: u64, channel_keys_id: [u8; 32]) -> EnforcingSigner {
+               let keys = self.backing.derive_channel_signer(channel_value_satoshis, channel_keys_id);
                let state = self.make_enforcement_state_cell(keys.commitment_seed);
                EnforcingSigner::new_with_revoked(keys, state, self.disable_revocation_policy_check)
        }
diff --git a/pending_changelog/1856.txt b/pending_changelog/1856.txt
new file mode 100644 (file)
index 0000000..d45cff6
--- /dev/null
@@ -0,0 +1,9 @@
+## API Updates
+- `PaymentReceived` events now have `via_channel_id` and `via_user_channel_id`
+  fields exposing the hop over which we received an inbound payment. Also,
+  `ChannelDetails` now expose the currently observed number of `confirmations`
+  on the funding transaction.
+
+## Backwards Compatibilty
+- Inbound payments with HTLCs pending on update to 0.0.113 will result
+  in a `PaymentReceived` event with `user_channel_id` 0.
diff --git a/pending_changelog/elias-event-renaming.txt b/pending_changelog/elias-event-renaming.txt
new file mode 100644 (file)
index 0000000..c647296
--- /dev/null
@@ -0,0 +1,5 @@
+## API Updates
+
+## Backwards Compatibilty
+- The event formerly known as `PaymentReceived` is now called
+  `PaymentClaimable`.