Merge pull request #1772 from ViktorTigerstrom/2022-10-move-claimable-htlcs-to-sepera...
authorMatt Corallo <649246+TheBlueMatt@users.noreply.github.com>
Tue, 22 Nov 2022 01:06:29 +0000 (01:06 +0000)
committerGitHub <noreply@github.com>
Tue, 22 Nov 2022 01:06:29 +0000 (01:06 +0000)
Move `claimable_htlcs` to separate lock

1  2 
lightning/src/ln/channelmanager.rs

index 3526726478887093f077a7c8bd79f465a2a940e5,e475cf13854ad7eca231811847c3fbb7dab40511..04adc124d65a0fa6c0379d85c88f7e488d2c209d
@@@ -46,14 -46,14 +46,14 @@@ use crate::ln::channel::{Channel, Chann
  use crate::ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, NodeFeatures};
  #[cfg(any(feature = "_test_utils", test))]
  use crate::ln::features::InvoiceFeatures;
 -use crate::routing::router::{PaymentParameters, Route, RouteHop, RoutePath, RouteParameters};
 +use crate::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, RoutePath, RouteParameters};
  use crate::ln::msgs;
  use crate::ln::onion_utils;
  use crate::ln::msgs::{ChannelMessageHandler, DecodeError, LightningError, MAX_VALUE_MSAT};
  use crate::ln::wire::Encode;
  use crate::chain::keysinterface::{Sign, KeysInterface, KeysManager, Recipient};
  use crate::util::config::{UserConfig, ChannelConfig};
 -use crate::util::events::{EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination};
 +use crate::util::events::{Event, EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination};
  use crate::util::{byte_utils, events};
  use crate::util::wakers::{Future, Notifier};
  use crate::util::scid_utils::fake_scid;
@@@ -112,8 -112,7 +112,8 @@@ pub(super) struct PendingHTLCInfo 
        pub(super) routing: PendingHTLCRouting,
        pub(super) incoming_shared_secret: [u8; 32],
        payment_hash: PaymentHash,
 -      pub(super) amt_to_forward: u64,
 +      pub(super) incoming_amt_msat: Option<u64>, // Added in 0.0.113
 +      pub(super) outgoing_amt_msat: u64,
        pub(super) outgoing_cltv_value: u32,
  }
  
@@@ -130,22 -129,20 +130,22 @@@ pub(super) enum PendingHTLCStatus 
        Fail(HTLCFailureMsg),
  }
  
 -pub(super) enum HTLCForwardInfo {
 -      AddHTLC {
 -              forward_info: PendingHTLCInfo,
 +pub(super) struct PendingAddHTLCInfo {
 +      pub(super) forward_info: PendingHTLCInfo,
  
 -              // These fields are produced in `forward_htlcs()` and consumed in
 -              // `process_pending_htlc_forwards()` for constructing the
 -              // `HTLCSource::PreviousHopData` for failed and forwarded
 -              // HTLCs.
 -              //
 -              // Note that this may be an outbound SCID alias for the associated channel.
 -              prev_short_channel_id: u64,
 -              prev_htlc_id: u64,
 -              prev_funding_outpoint: OutPoint,
 -      },
 +      // These fields are produced in `forward_htlcs()` and consumed in
 +      // `process_pending_htlc_forwards()` for constructing the
 +      // `HTLCSource::PreviousHopData` for failed and forwarded
 +      // HTLCs.
 +      //
 +      // Note that this may be an outbound SCID alias for the associated channel.
 +      prev_short_channel_id: u64,
 +      prev_htlc_id: u64,
 +      prev_funding_outpoint: OutPoint,
 +}
 +
 +pub(super) enum HTLCForwardInfo {
 +      AddHTLC(PendingAddHTLCInfo),
        FailHTLC {
                htlc_id: u64,
                err_packet: msgs::OnionErrorPacket,
@@@ -292,7 -289,7 +292,7 @@@ type ShutdownResult = (Option<(OutPoint
  
  struct MsgHandleErrInternal {
        err: msgs::LightningError,
 -      chan_id: Option<([u8; 32], u64)>, // If Some a channel of ours has been closed
 +      chan_id: Option<([u8; 32], u128)>, // If Some a channel of ours has been closed
        shutdown_finish: Option<(ShutdownResult, Option<msgs::ChannelUpdate>)>,
  }
  impl MsgHandleErrInternal {
                Self { err, chan_id: None, shutdown_finish: None }
        }
        #[inline]
 -      fn from_finish_shutdown(err: String, channel_id: [u8; 32], user_channel_id: u64, shutdown_res: ShutdownResult, channel_update: Option<msgs::ChannelUpdate>) -> Self {
 +      fn from_finish_shutdown(err: String, channel_id: [u8; 32], user_channel_id: u128, shutdown_res: ShutdownResult, channel_update: Option<msgs::ChannelUpdate>) -> Self {
                Self {
                        err: LightningError {
                                err: err.clone(),
@@@ -398,13 -395,6 +398,6 @@@ pub(super) enum RAACommitmentOrder 
  // Note this is only exposed in cfg(test):
  pub(super) struct ChannelHolder<Signer: Sign> {
        pub(super) by_id: HashMap<[u8; 32], Channel<Signer>>,
-       /// Map from payment hash to the payment data and any HTLCs which are to us and can be
-       /// failed/claimed by the user.
-       ///
-       /// Note that while this is held in the same mutex as the channels themselves, no consistency
-       /// guarantees are made about the channels given here actually existing anymore by the time you
-       /// go to read them!
-       claimable_htlcs: HashMap<PaymentHash, (events::PaymentPurpose, Vec<ClaimableHTLC>)>,
        /// Messages to send to peers - pushed to in the same lock that they are generated in (except
        /// for broadcast messages, where ordering isn't as strict).
        pub(super) pending_msg_events: Vec<MessageSendEvent>,
@@@ -673,19 -663,21 +666,21 @@@ pub type SimpleRefChannelManager<'a, 'b
  //  |
  //  |__`forward_htlcs`
  //  |
- //  |__`channel_state`
+ //  |__`pending_inbound_payments`
  //  |   |
- //  |   |__`id_to_peer`
+ //  |   |__`claimable_htlcs`
  //  |   |
- //  |   |__`short_to_chan_info`
- //  |   |
- //  |   |__`per_peer_state`
- //  |       |
- //  |       |__`outbound_scid_aliases`
+ //  |   |__`pending_outbound_payments`
  //  |       |
- //  |       |__`pending_inbound_payments`
+ //  |       |__`channel_state`
+ //  |           |
+ //  |           |__`id_to_peer`
  //  |           |
- //  |           |__`pending_outbound_payments`
+ //  |           |__`short_to_chan_info`
+ //  |           |
+ //  |           |__`per_peer_state`
+ //  |               |
+ //  |               |__`outbound_scid_aliases`
  //  |               |
  //  |               |__`best_block`
  //  |               |
@@@ -756,6 -748,15 +751,15 @@@ pub struct ChannelManager<M: Deref, T: 
        #[cfg(not(test))]
        forward_htlcs: Mutex<HashMap<u64, Vec<HTLCForwardInfo>>>,
  
+       /// Map from payment hash to the payment data and any HTLCs which are to us and can be
+       /// failed/claimed by the user.
+       ///
+       /// Note that, no consistency guarantees are made about the channels given here actually
+       /// existing anymore by the time you go to read them!
+       ///
+       /// See `ChannelManager` struct-level documentation for lock order requirements.
+       claimable_htlcs: Mutex<HashMap<PaymentHash, (events::PaymentPurpose, Vec<ClaimableHTLC>)>>,
        /// The set of outbound SCID aliases across all our channels, including unconfirmed channels
        /// and some closed channels which reached a usable state prior to being closed. This is used
        /// only to avoid duplicates, and is not persisted explicitly to disk, but rebuilt from the
@@@ -1086,10 -1087,8 +1090,10 @@@ pub struct ChannelDetails 
        ///
        /// [`outbound_capacity_msat`]: ChannelDetails::outbound_capacity_msat
        pub unspendable_punishment_reserve: Option<u64>,
 -      /// The `user_channel_id` passed in to create_channel, or 0 if the channel was inbound.
 -      pub user_channel_id: u64,
 +      /// The `user_channel_id` passed in to create_channel, or a random value if the channel was
 +      /// inbound. This may be zero for inbound channels serialized with LDK versions prior to
 +      /// 0.0.113.
 +      pub user_channel_id: u128,
        /// Our total balance.  This is the amount we would get if we close the channel.
        /// This value is not exact. Due to various in-flight changes and feerate changes, exactly this
        /// amount is not likely to be recoverable on close.
@@@ -1205,40 -1204,24 +1209,40 @@@ impl ChannelDetails 
  #[derive(Clone, Debug)]
  pub enum PaymentSendFailure {
        /// A parameter which was passed to send_payment was invalid, preventing us from attempting to
 -      /// send the payment at all. No channel state has been changed or messages sent to peers, and
 -      /// once you've changed the parameter at error, you can freely retry the payment in full.
 +      /// send the payment at all.
 +      ///
 +      /// You can freely resend the payment in full (with the parameter error fixed).
 +      ///
 +      /// Because the payment failed outright, no payment tracking is done, you do not need to call
 +      /// [`ChannelManager::abandon_payment`] and [`ChannelManager::retry_payment`] will *not* work
 +      /// for this payment.
        ParameterError(APIError),
        /// A parameter in a single path which was passed to send_payment was invalid, preventing us
 -      /// from attempting to send the payment at all. No channel state has been changed or messages
 -      /// sent to peers, and once you've changed the parameter at error, you can freely retry the
 -      /// payment in full.
 +      /// from attempting to send the payment at all.
 +      ///
 +      /// You can freely resend the payment in full (with the parameter error fixed).
        ///
        /// The results here are ordered the same as the paths in the route object which was passed to
        /// send_payment.
 +      ///
 +      /// Because the payment failed outright, no payment tracking is done, you do not need to call
 +      /// [`ChannelManager::abandon_payment`] and [`ChannelManager::retry_payment`] will *not* work
 +      /// for this payment.
        PathParameterError(Vec<Result<(), APIError>>),
        /// All paths which were attempted failed to send, with no channel state change taking place.
 -      /// You can freely retry the payment in full (though you probably want to do so over different
 +      /// You can freely resend the payment in full (though you probably want to do so over different
        /// paths than the ones selected).
        ///
 -      /// [`ChannelManager::abandon_payment`] does *not* need to be called for this payment and
 -      /// [`ChannelManager::retry_payment`] will *not* work for this payment.
 -      AllFailedRetrySafe(Vec<APIError>),
 +      /// Because the payment failed outright, no payment tracking is done, you do not need to call
 +      /// [`ChannelManager::abandon_payment`] and [`ChannelManager::retry_payment`] will *not* work
 +      /// for this payment.
 +      AllFailedResendSafe(Vec<APIError>),
 +      /// Indicates that a payment for the provided [`PaymentId`] is already in-flight and has not
 +      /// yet completed (i.e. generated an [`Event::PaymentSent`]) or been abandoned (via
 +      /// [`ChannelManager::abandon_payment`]).
 +      ///
 +      /// [`Event::PaymentSent`]: events::Event::PaymentSent
 +      DuplicatePayment,
        /// Some paths which were attempted failed to send, though possibly not all. At least some
        /// paths have irrevocably committed to the HTLC and retrying the payment in full would result
        /// in over-/re-payment.
@@@ -1678,13 -1661,13 +1682,13 @@@ impl<M: Deref, T: Deref, K: Deref, F: D
  
                        channel_state: Mutex::new(ChannelHolder{
                                by_id: HashMap::new(),
-                               claimable_htlcs: HashMap::new(),
                                pending_msg_events: Vec::new(),
                        }),
                        outbound_scid_aliases: Mutex::new(HashSet::new()),
                        pending_inbound_payments: Mutex::new(HashMap::new()),
                        pending_outbound_payments: Mutex::new(HashMap::new()),
                        forward_htlcs: Mutex::new(HashMap::new()),
+                       claimable_htlcs: Mutex::new(HashMap::new()),
                        id_to_peer: Mutex::new(HashMap::new()),
                        short_to_chan_info: FairRwLock::new(HashMap::new()),
  
        ///
        /// `user_channel_id` will be provided back as in
        /// [`Event::FundingGenerationReady::user_channel_id`] to allow tracking of which events
 -      /// correspond with which `create_channel` call. Note that the `user_channel_id` defaults to 0
 -      /// for inbound channels, so you may wish to avoid using 0 for `user_channel_id` here.
 -      /// `user_channel_id` has no meaning inside of LDK, it is simply copied to events and otherwise
 -      /// ignored.
 +      /// correspond with which `create_channel` call. Note that the `user_channel_id` defaults to a
 +      /// randomized value for inbound channels. `user_channel_id` has no meaning inside of LDK, it
 +      /// is simply copied to events and otherwise ignored.
        ///
        /// Raises [`APIError::APIMisuseError`] when `channel_value_satoshis` > 2**24 or `push_msat` is
        /// greater than `channel_value_satoshis * 1k` or `channel_value_satoshis < 1000`.
        /// [`Event::FundingGenerationReady::user_channel_id`]: events::Event::FundingGenerationReady::user_channel_id
        /// [`Event::FundingGenerationReady::temporary_channel_id`]: events::Event::FundingGenerationReady::temporary_channel_id
        /// [`Event::ChannelClosed::channel_id`]: events::Event::ChannelClosed::channel_id
 -      pub fn create_channel(&self, their_network_key: PublicKey, channel_value_satoshis: u64, push_msat: u64, user_channel_id: u64, override_config: Option<UserConfig>) -> Result<[u8; 32], APIError> {
 +      pub fn create_channel(&self, their_network_key: PublicKey, channel_value_satoshis: u64, push_msat: u64, user_channel_id: u128, override_config: Option<UserConfig>) -> Result<[u8; 32], APIError> {
                if channel_value_satoshis < 1000 {
                        return Err(APIError::APIMisuseError { err: format!("Channel value must be at least 1000 satoshis. It was {}", channel_value_satoshis) });
                }
                                        if *counterparty_node_id != chan_entry.get().get_counterparty_node_id(){
                                                return Err(APIError::APIMisuseError { err: "The passed counterparty_node_id doesn't match the channel's counterparty node_id".to_owned() });
                                        }
-                                       let per_peer_state = self.per_peer_state.read().unwrap();
-                                       let (shutdown_msg, monitor_update, htlcs) = match per_peer_state.get(&counterparty_node_id) {
-                                               Some(peer_state) => {
-                                                       let peer_state = peer_state.lock().unwrap();
-                                                       let their_features = &peer_state.latest_features;
-                                                       chan_entry.get_mut().get_shutdown(&self.keys_manager, their_features, target_feerate_sats_per_1000_weight)?
-                                               },
-                                               None => return Err(APIError::ChannelUnavailable { err: format!("Not connected to node: {}", counterparty_node_id) }),
+                                       let (shutdown_msg, monitor_update, htlcs) = {
+                                               let per_peer_state = self.per_peer_state.read().unwrap();
+                                               match per_peer_state.get(&counterparty_node_id) {
+                                                       Some(peer_state) => {
+                                                               let peer_state = peer_state.lock().unwrap();
+                                                               let their_features = &peer_state.latest_features;
+                                                               chan_entry.get_mut().get_shutdown(&self.keys_manager, their_features, target_feerate_sats_per_1000_weight)?
+                                                       },
+                                                       None => return Err(APIError::ChannelUnavailable { err: format!("Not connected to node: {}", counterparty_node_id) }),
+                                               }
                                        };
                                        failed_htlcs = htlcs;
  
                }
  
                let routing = match hop_data.format {
 -                      msgs::OnionHopDataFormat::Legacy { .. } => {
 -                              return Err(ReceiveError {
 -                                      err_code: 0x4000|0x2000|3,
 -                                      err_data: Vec::new(),
 -                                      msg: "We require payment_secrets",
 -                              });
 -                      },
                        msgs::OnionHopDataFormat::NonFinalNode { .. } => {
                                return Err(ReceiveError {
                                        err_code: 0x4000|22,
                        routing,
                        payment_hash,
                        incoming_shared_secret: shared_secret,
 -                      amt_to_forward: amt_msat,
 +                      incoming_amt_msat: Some(amt_msat),
 +                      outgoing_amt_msat: amt_msat,
                        outgoing_cltv_value: hop_data.outgoing_cltv_value,
                })
        }
                                };
  
                                let short_channel_id = match next_hop_data.format {
 -                                      msgs::OnionHopDataFormat::Legacy { short_channel_id } => short_channel_id,
                                        msgs::OnionHopDataFormat::NonFinalNode { short_channel_id } => short_channel_id,
                                        msgs::OnionHopDataFormat::FinalNode { .. } => {
                                                return_err!("Final Node OnionHopData provided for us as an intermediary node", 0x4000 | 22, &[0;0]);
                                        },
                                        payment_hash: msg.payment_hash.clone(),
                                        incoming_shared_secret: shared_secret,
 -                                      amt_to_forward: next_hop_data.amt_to_forward,
 +                                      incoming_amt_msat: Some(msg.amount_msat),
 +                                      outgoing_amt_msat: next_hop_data.amt_to_forward,
                                        outgoing_cltv_value: next_hop_data.outgoing_cltv_value,
                                })
                        }
                };
  
 -              if let &PendingHTLCStatus::Forward(PendingHTLCInfo { ref routing, ref amt_to_forward, ref outgoing_cltv_value, .. }) = &pending_forward_info {
 +              if let &PendingHTLCStatus::Forward(PendingHTLCInfo { ref routing, ref outgoing_amt_msat, ref outgoing_cltv_value, .. }) = &pending_forward_info {
                        // If short_channel_id is 0 here, we'll reject the HTLC as there cannot be a channel
                        // with a short_channel_id of 0. This is important as various things later assume
                        // short_channel_id is non-0 in any ::Forward.
                                                None => { // unknown_next_peer
                                                        // Note that this is likely a timing oracle for detecting whether an scid is a
                                                        // phantom.
 -                                                      if fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, *short_channel_id) {
 +                                                      if fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, *short_channel_id, &self.genesis_hash) {
                                                                None
                                                        } else {
                                                                break Some(("Don't have available channel for forwarding as requested.", 0x4000 | 10, None));
                                                if !chan.is_live() { // channel_disabled
                                                        break Some(("Forwarding channel is not in a ready state.", 0x1000 | 20, chan_update_opt));
                                                }
 -                                              if *amt_to_forward < chan.get_counterparty_htlc_minimum_msat() { // amount_below_minimum
 +                                              if *outgoing_amt_msat < chan.get_counterparty_htlc_minimum_msat() { // amount_below_minimum
                                                        break Some(("HTLC amount was below the htlc_minimum_msat", 0x1000 | 11, chan_update_opt));
                                                }
 -                                              if let Err((err, code)) = chan.htlc_satisfies_config(&msg, *amt_to_forward, *outgoing_cltv_value) {
 +                                              if let Err((err, code)) = chan.htlc_satisfies_config(&msg, *outgoing_amt_msat, *outgoing_cltv_value) {
                                                        break Some((err, code, chan_update_opt));
                                                }
                                                chan_update_opt
  
                let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap();
                match pending_outbounds.entry(payment_id) {
 -                      hash_map::Entry::Occupied(_) => Err(PaymentSendFailure::ParameterError(APIError::RouteError {
 -                              err: "Payment already in progress"
 -                      })),
 +                      hash_map::Entry::Occupied(_) => Err(PaymentSendFailure::DuplicatePayment),
                        hash_map::Entry::Vacant(entry) => {
                                let payment = entry.insert(PendingOutboundPayment::Retryable {
                                        session_privs: HashSet::new(),
                        // `pending_outbound_payments` map, as the user isn't expected to `abandon_payment`.
                        let removed = self.pending_outbound_payments.lock().unwrap().remove(&payment_id).is_some();
                        debug_assert!(removed, "We should always have a pending payment to remove here");
 -                      Err(PaymentSendFailure::AllFailedRetrySafe(results.drain(..).map(|r| r.unwrap_err()).collect()))
 +                      Err(PaymentSendFailure::AllFailedResendSafe(results.drain(..).map(|r| r.unwrap_err()).collect()))
                } else {
                        Ok(())
                }
                        mem::swap(&mut forward_htlcs, &mut self.forward_htlcs.lock().unwrap());
  
                        for (short_chan_id, mut pending_forwards) in forward_htlcs {
-                               let mut channel_state_lock = self.channel_state.lock().unwrap();
-                               let channel_state = &mut *channel_state_lock;
                                if short_chan_id != 0 {
                                        macro_rules! forwarding_channel_not_found {
                                                () => {
                                                        for forward_info in pending_forwards.drain(..) {
                                                                match forward_info {
 -                                                                      HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo {
 -                                                                              routing, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value },
 -                                                                              prev_funding_outpoint } => {
 -                                                                                      macro_rules! failure_handler {
 -                                                                                              ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr, $next_hop_unknown: expr) => {
 -                                                                                                      log_info!(self.logger, "Failed to accept/forward incoming HTLC: {}", $msg);
 -
 -                                                                                                      let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
 -                                                                                                              short_channel_id: prev_short_channel_id,
 -                                                                                                              outpoint: prev_funding_outpoint,
 -                                                                                                              htlc_id: prev_htlc_id,
 -                                                                                                              incoming_packet_shared_secret: incoming_shared_secret,
 -                                                                                                              phantom_shared_secret: $phantom_ss,
 -                                                                                                      });
 -
 -                                                                                                      let reason = if $next_hop_unknown {
 -                                                                                                              HTLCDestination::UnknownNextHop { requested_forward_scid: short_chan_id }
 -                                                                                                      } else {
 -                                                                                                              HTLCDestination::FailedPayment{ payment_hash }
 -                                                                                                      };
 -
 -                                                                                                      failed_forwards.push((htlc_source, payment_hash,
 -                                                                                                              HTLCFailReason::Reason { failure_code: $err_code, data: $err_data },
 -                                                                                                              reason
 -                                                                                                      ));
 -                                                                                                      continue;
 -                                                                                              }
 +                                                                      HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
 +                                                                              prev_short_channel_id, prev_htlc_id, prev_funding_outpoint,
 +                                                                              forward_info: PendingHTLCInfo {
 +                                                                                      routing, incoming_shared_secret, payment_hash, outgoing_amt_msat,
 +                                                                                      outgoing_cltv_value, incoming_amt_msat: _
 +                                                                              }
 +                                                                      }) => {
 +                                                                              macro_rules! failure_handler {
 +                                                                                      ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr, $next_hop_unknown: expr) => {
 +                                                                                              log_info!(self.logger, "Failed to accept/forward incoming HTLC: {}", $msg);
 +
 +                                                                                              let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
 +                                                                                                      short_channel_id: prev_short_channel_id,
 +                                                                                                      outpoint: prev_funding_outpoint,
 +                                                                                                      htlc_id: prev_htlc_id,
 +                                                                                                      incoming_packet_shared_secret: incoming_shared_secret,
 +                                                                                                      phantom_shared_secret: $phantom_ss,
 +                                                                                              });
 +
 +                                                                                              let reason = if $next_hop_unknown {
 +                                                                                                      HTLCDestination::UnknownNextHop { requested_forward_scid: short_chan_id }
 +                                                                                              } else {
 +                                                                                                      HTLCDestination::FailedPayment{ payment_hash }
 +                                                                                              };
 +
 +                                                                                              failed_forwards.push((htlc_source, payment_hash,
 +                                                                                                      HTLCFailReason::Reason { failure_code: $err_code, data: $err_data },
 +                                                                                                      reason
 +                                                                                              ));
 +                                                                                              continue;
                                                                                        }
 -                                                                                      macro_rules! fail_forward {
 -                                                                                              ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
 -                                                                                                      {
 -                                                                                                              failure_handler!($msg, $err_code, $err_data, $phantom_ss, true);
 -                                                                                                      }
 +                                                                              }
 +                                                                              macro_rules! fail_forward {
 +                                                                                      ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
 +                                                                                              {
 +                                                                                                      failure_handler!($msg, $err_code, $err_data, $phantom_ss, true);
                                                                                                }
                                                                                        }
 -                                                                                      macro_rules! failed_payment {
 -                                                                                              ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
 -                                                                                                      {
 -                                                                                                              failure_handler!($msg, $err_code, $err_data, $phantom_ss, false);
 -                                                                                                      }
 +                                                                              }
 +                                                                              macro_rules! failed_payment {
 +                                                                                      ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
 +                                                                                              {
 +                                                                                                      failure_handler!($msg, $err_code, $err_data, $phantom_ss, false);
                                                                                                }
                                                                                        }
 -                                                                                      if let PendingHTLCRouting::Forward { onion_packet, .. } = routing {
 -                                                                                              let phantom_secret_res = self.keys_manager.get_node_secret(Recipient::PhantomNode);
 -                                                                                              if phantom_secret_res.is_ok() && fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, short_chan_id) {
 -                                                                                                      let phantom_shared_secret = SharedSecret::new(&onion_packet.public_key.unwrap(), &phantom_secret_res.unwrap()).secret_bytes();
 -                                                                                                      let next_hop = match onion_utils::decode_next_payment_hop(phantom_shared_secret, &onion_packet.hop_data, onion_packet.hmac, payment_hash) {
 -                                                                                                              Ok(res) => res,
 -                                                                                                              Err(onion_utils::OnionDecodeErr::Malformed { err_msg, err_code }) => {
 -                                                                                                                      let sha256_of_onion = Sha256::hash(&onion_packet.hop_data).into_inner();
 -                                                                                                                      // In this scenario, the phantom would have sent us an
 -                                                                                                                      // `update_fail_malformed_htlc`, meaning here we encrypt the error as
 -                                                                                                                      // if it came from us (the second-to-last hop) but contains the sha256
 -                                                                                                                      // of the onion.
 -                                                                                                                      failed_payment!(err_msg, err_code, sha256_of_onion.to_vec(), None);
 -                                                                                                              },
 -                                                                                                              Err(onion_utils::OnionDecodeErr::Relay { err_msg, err_code }) => {
 -                                                                                                                      failed_payment!(err_msg, err_code, Vec::new(), Some(phantom_shared_secret));
 -                                                                                                              },
 -                                                                                                      };
 -                                                                                                      match next_hop {
 -                                                                                                              onion_utils::Hop::Receive(hop_data) => {
 -                                                                                                                      match self.construct_recv_pending_htlc_info(hop_data, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value, Some(phantom_shared_secret)) {
 -                                                                                                                              Ok(info) => phantom_receives.push((prev_short_channel_id, prev_funding_outpoint, vec![(info, prev_htlc_id)])),
 -                                                                                                                              Err(ReceiveError { err_code, err_data, msg }) => failed_payment!(msg, err_code, err_data, Some(phantom_shared_secret))
 -                                                                                                                      }
 -                                                                                                              },
 -                                                                                                              _ => panic!(),
 -                                                                                                      }
 -                                                                                              } else {
 -                                                                                                      fail_forward!(format!("Unknown short channel id {} for forward HTLC", short_chan_id), 0x4000 | 10, Vec::new(), None);
 +                                                                              }
 +                                                                              if let PendingHTLCRouting::Forward { onion_packet, .. } = routing {
 +                                                                                      let phantom_secret_res = self.keys_manager.get_node_secret(Recipient::PhantomNode);
 +                                                                                      if phantom_secret_res.is_ok() && fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, short_chan_id, &self.genesis_hash) {
 +                                                                                              let phantom_shared_secret = SharedSecret::new(&onion_packet.public_key.unwrap(), &phantom_secret_res.unwrap()).secret_bytes();
 +                                                                                              let next_hop = match onion_utils::decode_next_payment_hop(phantom_shared_secret, &onion_packet.hop_data, onion_packet.hmac, payment_hash) {
 +                                                                                                      Ok(res) => res,
 +                                                                                                      Err(onion_utils::OnionDecodeErr::Malformed { err_msg, err_code }) => {
 +                                                                                                              let sha256_of_onion = Sha256::hash(&onion_packet.hop_data).into_inner();
 +                                                                                                              // In this scenario, the phantom would have sent us an
 +                                                                                                              // `update_fail_malformed_htlc`, meaning here we encrypt the error as
 +                                                                                                              // if it came from us (the second-to-last hop) but contains the sha256
 +                                                                                                              // of the onion.
 +                                                                                                              failed_payment!(err_msg, err_code, sha256_of_onion.to_vec(), None);
 +                                                                                                      },
 +                                                                                                      Err(onion_utils::OnionDecodeErr::Relay { err_msg, err_code }) => {
 +                                                                                                              failed_payment!(err_msg, err_code, Vec::new(), Some(phantom_shared_secret));
 +                                                                                                      },
 +                                                                                              };
 +                                                                                              match next_hop {
 +                                                                                                      onion_utils::Hop::Receive(hop_data) => {
 +                                                                                                              match self.construct_recv_pending_htlc_info(hop_data, incoming_shared_secret, payment_hash, outgoing_amt_msat, outgoing_cltv_value, Some(phantom_shared_secret)) {
 +                                                                                                                      Ok(info) => phantom_receives.push((prev_short_channel_id, prev_funding_outpoint, vec![(info, prev_htlc_id)])),
 +                                                                                                                      Err(ReceiveError { err_code, err_data, msg }) => failed_payment!(msg, err_code, err_data, Some(phantom_shared_secret))
 +                                                                                                              }
 +                                                                                                      },
 +                                                                                                      _ => panic!(),
                                                                                                }
                                                                                        } else {
                                                                                                fail_forward!(format!("Unknown short channel id {} for forward HTLC", short_chan_id), 0x4000 | 10, Vec::new(), None);
                                                                                        }
 -                                                                              },
 +                                                                              } else {
 +                                                                                      fail_forward!(format!("Unknown short channel id {} for forward HTLC", short_chan_id), 0x4000 | 10, Vec::new(), None);
 +                                                                              }
 +                                                                      },
                                                                        HTLCForwardInfo::FailHTLC { .. } => {
                                                                                // Channel went away before we could fail it. This implies
                                                                                // the channel is now on chain and our counterparty is
                                                        continue;
                                                }
                                        };
+                                       let mut channel_state_lock = self.channel_state.lock().unwrap();
+                                       let channel_state = &mut *channel_state_lock;
                                        match channel_state.by_id.entry(forward_chan_id) {
                                                hash_map::Entry::Vacant(_) => {
                                                        forwarding_channel_not_found!();
                                                        let mut fail_htlc_msgs = Vec::new();
                                                        for forward_info in pending_forwards.drain(..) {
                                                                match forward_info {
 -                                                                      HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo {
 -                                                                                      routing: PendingHTLCRouting::Forward {
 -                                                                                              onion_packet, ..
 -                                                                                      }, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value },
 -                                                                                      prev_funding_outpoint } => {
 +                                                                      HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
 +                                                                              prev_short_channel_id, prev_htlc_id, prev_funding_outpoint ,
 +                                                                              forward_info: PendingHTLCInfo {
 +                                                                                      incoming_shared_secret, payment_hash, outgoing_amt_msat, outgoing_cltv_value,
 +                                                                                      routing: PendingHTLCRouting::Forward { onion_packet, .. }, incoming_amt_msat: _,
 +                                                                              },
 +                                                                      }) => {
                                                                                log_trace!(self.logger, "Adding HTLC from short id {} with payment_hash {} to channel with short id {} after delay", prev_short_channel_id, log_bytes!(payment_hash.0), short_chan_id);
                                                                                let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
                                                                                        short_channel_id: prev_short_channel_id,
                                                                                        // Phantom payments are only PendingHTLCRouting::Receive.
                                                                                        phantom_shared_secret: None,
                                                                                });
 -                                                                              match chan.get_mut().send_htlc(amt_to_forward, payment_hash, outgoing_cltv_value, htlc_source.clone(), onion_packet, &self.logger) {
 +                                                                              match chan.get_mut().send_htlc(outgoing_amt_msat, payment_hash, outgoing_cltv_value, htlc_source.clone(), onion_packet, &self.logger) {
                                                                                        Err(e) => {
                                                                                                if let ChannelError::Ignore(msg) = e {
                                                                                                        log_trace!(self.logger, "Failed to forward HTLC with payment_hash {}: {}", log_bytes!(payment_hash.0), msg);
                                } else {
                                        for forward_info in pending_forwards.drain(..) {
                                                match forward_info {
 -                                                      HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo {
 -                                                                      routing, incoming_shared_secret, payment_hash, amt_to_forward, .. },
 -                                                                      prev_funding_outpoint } => {
 +                                                      HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
 +                                                              prev_short_channel_id, prev_htlc_id, prev_funding_outpoint,
 +                                                              forward_info: PendingHTLCInfo {
 +                                                                      routing, incoming_shared_secret, payment_hash, outgoing_amt_msat, ..
 +                                                              }
 +                                                      }) => {
                                                                let (cltv_expiry, onion_payload, payment_data, phantom_shared_secret) = match routing {
                                                                        PendingHTLCRouting::Receive { payment_data, incoming_cltv_expiry, phantom_shared_secret } => {
                                                                                let _legacy_hop_data = Some(payment_data.clone());
                                                                                incoming_packet_shared_secret: incoming_shared_secret,
                                                                                phantom_shared_secret,
                                                                        },
 -                                                                      value: amt_to_forward,
 +                                                                      value: outgoing_amt_msat,
                                                                        timer_ticks: 0,
 -                                                                      total_msat: if let Some(data) = &payment_data { data.total_msat } else { amt_to_forward },
 +                                                                      total_msat: if let Some(data) = &payment_data { data.total_msat } else { outgoing_amt_msat },
                                                                        cltv_expiry,
                                                                        onion_payload,
                                                                };
                                                                                                payment_secret: $payment_data.payment_secret,
                                                                                        }
                                                                                };
-                                                                               let (_, htlcs) = channel_state.claimable_htlcs.entry(payment_hash)
+                                                                               let mut claimable_htlcs = self.claimable_htlcs.lock().unwrap();
+                                                                               let (_, htlcs) = claimable_htlcs.entry(payment_hash)
                                                                                        .or_insert_with(|| (purpose(), Vec::new()));
                                                                                if htlcs.len() == 1 {
                                                                                        if let OnionPayload::Spontaneous(_) = htlcs[0].onion_payload {
                                                                                                check_total_value!(payment_data, payment_preimage);
                                                                                        },
                                                                                        OnionPayload::Spontaneous(preimage) => {
-                                                                                               match channel_state.claimable_htlcs.entry(payment_hash) {
+                                                                                               match self.claimable_htlcs.lock().unwrap().entry(payment_hash) {
                                                                                                        hash_map::Entry::Vacant(e) => {
                                                                                                                let purpose = events::PaymentPurpose::SpontaneousPayment(preimage);
                                                                                                                e.insert((purpose.clone(), vec![claimable_htlc]));
                                                                                                                new_events.push(events::Event::PaymentReceived {
                                                                                                                        payment_hash,
 -                                                                                                                      amount_msat: amt_to_forward,
 +                                                                                                                      amount_msat: outgoing_amt_msat,
                                                                                                                        purpose,
                                                                                                                });
                                                                                                        },
  
                                        true
                                });
+                       }
  
-                               channel_state.claimable_htlcs.retain(|payment_hash, (_, htlcs)| {
-                                       if htlcs.is_empty() {
-                                               // This should be unreachable
-                                               debug_assert!(false);
+                       self.claimable_htlcs.lock().unwrap().retain(|payment_hash, (_, htlcs)| {
+                               if htlcs.is_empty() {
+                                       // This should be unreachable
+                                       debug_assert!(false);
+                                       return false;
+                               }
+                               if let OnionPayload::Invoice { .. } = htlcs[0].onion_payload {
+                                       // Check if we've received all the parts we need for an MPP (the value of the parts adds to total_msat).
+                                       // In this case we're not going to handle any timeouts of the parts here.
+                                       if htlcs[0].total_msat == htlcs.iter().fold(0, |total, htlc| total + htlc.value) {
+                                               return true;
+                                       } else if htlcs.into_iter().any(|htlc| {
+                                               htlc.timer_ticks += 1;
+                                               return htlc.timer_ticks >= MPP_TIMEOUT_TICKS
+                                       }) {
+                                               timed_out_mpp_htlcs.extend(htlcs.into_iter().map(|htlc| (htlc.prev_hop.clone(), payment_hash.clone())));
                                                return false;
                                        }
-                                       if let OnionPayload::Invoice { .. } = htlcs[0].onion_payload {
-                                               // Check if we've received all the parts we need for an MPP (the value of the parts adds to total_msat).
-                                               // In this case we're not going to handle any timeouts of the parts here.
-                                               if htlcs[0].total_msat == htlcs.iter().fold(0, |total, htlc| total + htlc.value) {
-                                                       return true;
-                                               } else if htlcs.into_iter().any(|htlc| {
-                                                       htlc.timer_ticks += 1;
-                                                       return htlc.timer_ticks >= MPP_TIMEOUT_TICKS
-                                               }) {
-                                                       timed_out_mpp_htlcs.extend(htlcs.into_iter().map(|htlc| (htlc.prev_hop.clone(), payment_hash.clone())));
-                                                       return false;
-                                               }
-                                       }
-                                       true
-                               });
-                       }
+                               }
+                               true
+                       });
  
                        for htlc_source in timed_out_mpp_htlcs.drain(..) {
                                let receiver = HTLCDestination::FailedPayment { payment_hash: htlc_source.1 };
        pub fn fail_htlc_backwards(&self, payment_hash: &PaymentHash) {
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
  
-               let removed_source = {
-                       let mut channel_state = self.channel_state.lock().unwrap();
-                       channel_state.claimable_htlcs.remove(payment_hash)
-               };
+               let removed_source = self.claimable_htlcs.lock().unwrap().remove(payment_hash);
                if let Some((_, mut sources)) = removed_source {
                        for htlc in sources.drain(..) {
                                let mut htlc_msat_height_data = byte_utils::be64_to_array(htlc.value).to_vec();
  
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
  
-               let removed_source = self.channel_state.lock().unwrap().claimable_htlcs.remove(&payment_hash);
+               let removed_source = self.claimable_htlcs.lock().unwrap().remove(&payment_hash);
                if let Some((payment_purpose, mut sources)) = removed_source {
                        assert!(!sources.is_empty());
  
        ///
        /// [`Event::OpenChannelRequest`]: events::Event::OpenChannelRequest
        /// [`Event::ChannelClosed::user_channel_id`]: events::Event::ChannelClosed::user_channel_id
 -      pub fn accept_inbound_channel(&self, temporary_channel_id: &[u8; 32], counterparty_node_id: &PublicKey, user_channel_id: u64) -> Result<(), APIError> {
 +      pub fn accept_inbound_channel(&self, temporary_channel_id: &[u8; 32], counterparty_node_id: &PublicKey, user_channel_id: u128) -> Result<(), APIError> {
                self.do_accept_inbound_channel(temporary_channel_id, counterparty_node_id, false, user_channel_id)
        }
  
        ///
        /// [`Event::OpenChannelRequest`]: events::Event::OpenChannelRequest
        /// [`Event::ChannelClosed::user_channel_id`]: events::Event::ChannelClosed::user_channel_id
 -      pub fn accept_inbound_channel_from_trusted_peer_0conf(&self, temporary_channel_id: &[u8; 32], counterparty_node_id: &PublicKey, user_channel_id: u64) -> Result<(), APIError> {
 +      pub fn accept_inbound_channel_from_trusted_peer_0conf(&self, temporary_channel_id: &[u8; 32], counterparty_node_id: &PublicKey, user_channel_id: u128) -> Result<(), APIError> {
                self.do_accept_inbound_channel(temporary_channel_id, counterparty_node_id, true, user_channel_id)
        }
  
 -      fn do_accept_inbound_channel(&self, temporary_channel_id: &[u8; 32], counterparty_node_id: &PublicKey, accept_0conf: bool, user_channel_id: u64) -> Result<(), APIError> {
 +      fn do_accept_inbound_channel(&self, temporary_channel_id: &[u8; 32], counterparty_node_id: &PublicKey, accept_0conf: bool, user_channel_id: u128) -> Result<(), APIError> {
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
  
                let mut channel_state_lock = self.channel_state.lock().unwrap();
                        return Err(MsgHandleErrInternal::send_err_msg_no_close("No inbound channels accepted".to_owned(), msg.temporary_channel_id.clone()));
                }
  
 +              let mut random_bytes = [0u8; 16];
 +              random_bytes.copy_from_slice(&self.keys_manager.get_secure_random_bytes()[..16]);
 +              let user_channel_id = u128::from_be_bytes(random_bytes);
 +
                let outbound_scid_alias = self.create_and_insert_outbound_scid_alias();
                let mut channel = match Channel::new_from_req(&self.fee_estimator, &self.keys_manager,
 -                      counterparty_node_id.clone(), &their_features, msg, 0, &self.default_configuration,
 +                      counterparty_node_id.clone(), &their_features, msg, user_channel_id, &self.default_configuration,
                        self.best_block.read().unwrap().height(), &self.logger, outbound_scid_alias)
                {
                        Err(e) => {
                                        }
                                        channel_state.pending_msg_events.push(events::MessageSendEvent::SendAcceptChannel {
                                                node_id: counterparty_node_id.clone(),
 -                                              msg: channel.accept_inbound_channel(0),
 +                                              msg: channel.accept_inbound_channel(user_channel_id),
                                        });
                                } else {
                                        let mut pending_events = self.pending_events.lock().unwrap();
                                                        PendingHTLCRouting::ReceiveKeysend { .. } => 0,
                                        }) {
                                                hash_map::Entry::Occupied(mut entry) => {
 -                                                      entry.get_mut().push(HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_funding_outpoint,
 -                                                                                                      prev_htlc_id, forward_info });
 +                                                      entry.get_mut().push(HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
 +                                                              prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, forward_info }));
                                                },
                                                hash_map::Entry::Vacant(entry) => {
 -                                                      entry.insert(vec!(HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_funding_outpoint,
 -                                                                                                   prev_htlc_id, forward_info }));
 +                                                      entry.insert(vec!(HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
 +                                                              prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, forward_info })));
                                                }
                                        }
                                }
                                if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
 -                              try_chan_entry!(self, chan.get_mut().update_fee(&self.fee_estimator, &msg), chan);
 +                              try_chan_entry!(self, chan.get_mut().update_fee(&self.fee_estimator, &msg, &self.logger), chan);
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                }
                }
        }
  
 +      /// Gets inflight HTLC information by processing pending outbound payments that are in
 +      /// our channels. May be used during pathfinding to account for in-use channel liquidity.
 +      pub fn compute_inflight_htlcs(&self) -> InFlightHtlcs {
 +              let mut inflight_htlcs = InFlightHtlcs::new();
 +
 +              for chan in self.channel_state.lock().unwrap().by_id.values() {
 +                      for htlc_source in chan.inflight_htlc_sources() {
 +                              if let HTLCSource::OutboundRoute { path, .. } = htlc_source {
 +                                      inflight_htlcs.process_path(path, self.get_our_node_id());
 +                              }
 +                      }
 +              }
 +
 +              inflight_htlcs
 +      }
 +
        #[cfg(any(test, fuzzing, feature = "_test_utils"))]
        pub fn get_and_clear_pending_events(&self) -> Vec<events::Event> {
                let events = core::cell::RefCell::new(Vec::new());
 -              let event_handler = |event: &events::Event| events.borrow_mut().push(event.clone());
 +              let event_handler = |event: events::Event| events.borrow_mut().push(event);
                self.process_pending_events(&event_handler);
                events.into_inner()
        }
        pub fn clear_pending_payments(&self) {
                self.pending_outbound_payments.lock().unwrap().clear()
        }
 +
 +      /// Processes any events asynchronously in the order they were generated since the last call
 +      /// using the given event handler.
 +      ///
 +      /// See the trait-level documentation of [`EventsProvider`] for requirements.
 +      pub async fn process_pending_events_async<Future: core::future::Future, H: Fn(Event) -> Future>(
 +              &self, handler: H
 +      ) {
 +              // We'll acquire our total consistency lock until the returned future completes so that
 +              // we can be sure no other persists happen while processing events.
 +              let _read_guard = self.total_consistency_lock.read().unwrap();
 +
 +              let mut result = NotifyOption::SkipPersist;
 +
 +              // TODO: This behavior should be documented. It's unintuitive that we query
 +              // ChannelMonitors when clearing other events.
 +              if self.process_pending_monitor_events() {
 +                      result = NotifyOption::DoPersist;
 +              }
 +
 +              let pending_events = mem::replace(&mut *self.pending_events.lock().unwrap(), vec![]);
 +              if !pending_events.is_empty() {
 +                      result = NotifyOption::DoPersist;
 +              }
 +
 +              for event in pending_events {
 +                      handler(event).await;
 +              }
 +
 +              if result == NotifyOption::DoPersist {
 +                      self.persistence_notifier.notify();
 +              }
 +      }
  }
  
  impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> MessageSendEventsProvider for ChannelManager<M, T, K, F, L>
@@@ -5851,13 -5781,13 +5855,13 @@@ wher
                                result = NotifyOption::DoPersist;
                        }
  
 -                      let mut pending_events = mem::replace(&mut *self.pending_events.lock().unwrap(), vec![]);
 +                      let pending_events = mem::replace(&mut *self.pending_events.lock().unwrap(), vec![]);
                        if !pending_events.is_empty() {
                                result = NotifyOption::DoPersist;
                        }
  
 -                      for event in pending_events.drain(..) {
 -                              handler.handle_event(&event);
 +                      for event in pending_events {
 +                              handler.handle_event(event);
                        }
  
                        result
@@@ -5966,12 -5896,12 +5970,12 @@@ wher
                });
        }
  
 -      fn get_relevant_txids(&self) -> Vec<Txid> {
 +      fn get_relevant_txids(&self) -> Vec<(Txid, Option<BlockHash>)> {
                let channel_state = self.channel_state.lock().unwrap();
                let mut res = Vec::with_capacity(channel_state.by_id.len());
                for chan in channel_state.by_id.values() {
 -                      if let Some(funding_txo) = chan.get_funding_txo() {
 -                              res.push(funding_txo.txid);
 +                      if let (Some(funding_txo), block_hash) = (chan.get_funding_txo(), chan.get_funding_tx_confirmed_in()) {
 +                              res.push((funding_txo.txid, block_hash));
                        }
                }
                res
@@@ -6093,28 -6023,28 +6097,28 @@@ wher
                                }
                                true
                        });
+               }
  
-                       if let Some(height) = height_opt {
-                               channel_state.claimable_htlcs.retain(|payment_hash, (_, htlcs)| {
-                                       htlcs.retain(|htlc| {
-                                               // If height is approaching the number of blocks we think it takes us to get
-                                               // our commitment transaction confirmed before the HTLC expires, plus the
-                                               // number of blocks we generally consider it to take to do a commitment update,
-                                               // just give up on it and fail the HTLC.
-                                               if height >= htlc.cltv_expiry - HTLC_FAIL_BACK_BUFFER {
-                                                       let mut htlc_msat_height_data = byte_utils::be64_to_array(htlc.value).to_vec();
-                                                       htlc_msat_height_data.extend_from_slice(&byte_utils::be32_to_array(height));
-                                                       timed_out_htlcs.push((HTLCSource::PreviousHopData(htlc.prev_hop.clone()), payment_hash.clone(), HTLCFailReason::Reason {
-                                                               failure_code: 0x4000 | 15,
-                                                               data: htlc_msat_height_data
-                                                       }, HTLCDestination::FailedPayment { payment_hash: payment_hash.clone() }));
-                                                       false
-                                               } else { true }
-                                       });
-                                       !htlcs.is_empty() // Only retain this entry if htlcs has at least one entry.
+               if let Some(height) = height_opt {
+                       self.claimable_htlcs.lock().unwrap().retain(|payment_hash, (_, htlcs)| {
+                               htlcs.retain(|htlc| {
+                                       // If height is approaching the number of blocks we think it takes us to get
+                                       // our commitment transaction confirmed before the HTLC expires, plus the
+                                       // number of blocks we generally consider it to take to do a commitment update,
+                                       // just give up on it and fail the HTLC.
+                                       if height >= htlc.cltv_expiry - HTLC_FAIL_BACK_BUFFER {
+                                               let mut htlc_msat_height_data = byte_utils::be64_to_array(htlc.value).to_vec();
+                                               htlc_msat_height_data.extend_from_slice(&byte_utils::be32_to_array(height));
+                                               timed_out_htlcs.push((HTLCSource::PreviousHopData(htlc.prev_hop.clone()), payment_hash.clone(), HTLCFailReason::Reason {
+                                                       failure_code: 0x4000 | 15,
+                                                       data: htlc_msat_height_data
+                                               }, HTLCDestination::FailedPayment { payment_hash: payment_hash.clone() }));
+                                               false
+                                       } else { true }
                                });
-                       }
+                               !htlcs.is_empty() // Only retain this entry if htlcs has at least one entry.
+                       });
                }
  
                self.handle_init_event_channel_failures(failed_channels);
  
        /// Blocks until ChannelManager needs to be persisted or a timeout is reached. It returns a bool
        /// indicating whether persistence is necessary. Only one listener on
 -      /// `await_persistable_update` or `await_persistable_update_timeout` is guaranteed to be woken
 -      /// up.
 +      /// [`await_persistable_update`], [`await_persistable_update_timeout`], or a future returned by
 +      /// [`get_persistable_update_future`] is guaranteed to be woken up.
        ///
        /// Note that this method is not available with the `no-std` feature.
 +      ///
 +      /// [`await_persistable_update`]: Self::await_persistable_update
 +      /// [`await_persistable_update_timeout`]: Self::await_persistable_update_timeout
 +      /// [`get_persistable_update_future`]: Self::get_persistable_update_future
        #[cfg(any(test, feature = "std"))]
        pub fn await_persistable_update_timeout(&self, max_wait: Duration) -> bool {
                self.persistence_notifier.wait_timeout(max_wait)
        }
  
        /// Blocks until ChannelManager needs to be persisted. Only one listener on
 -      /// `await_persistable_update` or `await_persistable_update_timeout` is guaranteed to be woken
 -      /// up.
 +      /// [`await_persistable_update`], `await_persistable_update_timeout`, or a future returned by
 +      /// [`get_persistable_update_future`] is guaranteed to be woken up.
 +      ///
 +      /// [`await_persistable_update`]: Self::await_persistable_update
 +      /// [`get_persistable_update_future`]: Self::get_persistable_update_future
        pub fn await_persistable_update(&self) {
                self.persistence_notifier.wait()
        }
@@@ -6485,108 -6408,33 +6489,108 @@@ impl_writeable_tlv_based!(ChannelCounte
        (11, outbound_htlc_maximum_msat, option),
  });
  
 -impl_writeable_tlv_based!(ChannelDetails, {
 -      (1, inbound_scid_alias, option),
 -      (2, channel_id, required),
 -      (3, channel_type, option),
 -      (4, counterparty, required),
 -      (5, outbound_scid_alias, option),
 -      (6, funding_txo, option),
 -      (7, config, option),
 -      (8, short_channel_id, option),
 -      (10, channel_value_satoshis, required),
 -      (12, unspendable_punishment_reserve, option),
 -      (14, user_channel_id, required),
 -      (16, balance_msat, required),
 -      (18, outbound_capacity_msat, required),
 -      // Note that by the time we get past the required read above, outbound_capacity_msat will be
 -      // filled in, so we can safely unwrap it here.
 -      (19, next_outbound_htlc_limit_msat, (default_value, outbound_capacity_msat.0.unwrap() as u64)),
 -      (20, inbound_capacity_msat, required),
 -      (22, confirmations_required, option),
 -      (24, force_close_spend_delay, option),
 -      (26, is_outbound, required),
 -      (28, is_channel_ready, required),
 -      (30, is_usable, required),
 -      (32, is_public, required),
 -      (33, inbound_htlc_minimum_msat, option),
 -      (35, inbound_htlc_maximum_msat, option),
 -});
 +impl Writeable for ChannelDetails {
 +      fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
 +              // `user_channel_id` used to be a single u64 value. In order to remain backwards compatible with
 +              // versions prior to 0.0.113, the u128 is serialized as two separate u64 values.
 +              let user_channel_id_low = self.user_channel_id as u64;
 +              let user_channel_id_high_opt = Some((self.user_channel_id >> 64) as u64);
 +              write_tlv_fields!(writer, {
 +                      (1, self.inbound_scid_alias, option),
 +                      (2, self.channel_id, required),
 +                      (3, self.channel_type, option),
 +                      (4, self.counterparty, required),
 +                      (5, self.outbound_scid_alias, option),
 +                      (6, self.funding_txo, option),
 +                      (7, self.config, option),
 +                      (8, self.short_channel_id, option),
 +                      (10, self.channel_value_satoshis, required),
 +                      (12, self.unspendable_punishment_reserve, option),
 +                      (14, user_channel_id_low, required),
 +                      (16, self.balance_msat, required),
 +                      (18, self.outbound_capacity_msat, required),
 +                      // Note that by the time we get past the required read above, outbound_capacity_msat will be
 +                      // filled in, so we can safely unwrap it here.
 +                      (19, self.next_outbound_htlc_limit_msat, (default_value, outbound_capacity_msat.0.unwrap() as u64)),
 +                      (20, self.inbound_capacity_msat, required),
 +                      (22, self.confirmations_required, option),
 +                      (24, self.force_close_spend_delay, option),
 +                      (26, self.is_outbound, required),
 +                      (28, self.is_channel_ready, required),
 +                      (30, self.is_usable, required),
 +                      (32, self.is_public, required),
 +                      (33, self.inbound_htlc_minimum_msat, option),
 +                      (35, self.inbound_htlc_maximum_msat, option),
 +                      (37, user_channel_id_high_opt, option),
 +              });
 +              Ok(())
 +      }
 +}
 +
 +impl Readable for ChannelDetails {
 +      fn read<R: Read>(reader: &mut R) -> Result<Self, DecodeError> {
 +              init_and_read_tlv_fields!(reader, {
 +                      (1, inbound_scid_alias, option),
 +                      (2, channel_id, required),
 +                      (3, channel_type, option),
 +                      (4, counterparty, required),
 +                      (5, outbound_scid_alias, option),
 +                      (6, funding_txo, option),
 +                      (7, config, option),
 +                      (8, short_channel_id, option),
 +                      (10, channel_value_satoshis, required),
 +                      (12, unspendable_punishment_reserve, option),
 +                      (14, user_channel_id_low, required),
 +                      (16, balance_msat, required),
 +                      (18, outbound_capacity_msat, required),
 +                      // Note that by the time we get past the required read above, outbound_capacity_msat will be
 +                      // filled in, so we can safely unwrap it here.
 +                      (19, next_outbound_htlc_limit_msat, (default_value, outbound_capacity_msat.0.unwrap() as u64)),
 +                      (20, inbound_capacity_msat, required),
 +                      (22, confirmations_required, option),
 +                      (24, force_close_spend_delay, option),
 +                      (26, is_outbound, required),
 +                      (28, is_channel_ready, required),
 +                      (30, is_usable, required),
 +                      (32, is_public, required),
 +                      (33, inbound_htlc_minimum_msat, option),
 +                      (35, inbound_htlc_maximum_msat, option),
 +                      (37, user_channel_id_high_opt, option),
 +              });
 +
 +              // `user_channel_id` used to be a single u64 value. In order to remain backwards compatible with
 +              // versions prior to 0.0.113, the u128 is serialized as two separate u64 values.
 +              let user_channel_id_low: u64 = user_channel_id_low.0.unwrap();
 +              let user_channel_id = user_channel_id_low as u128 +
 +                      ((user_channel_id_high_opt.unwrap_or(0 as u64) as u128) << 64);
 +
 +              Ok(Self {
 +                      inbound_scid_alias,
 +                      channel_id: channel_id.0.unwrap(),
 +                      channel_type,
 +                      counterparty: counterparty.0.unwrap(),
 +                      outbound_scid_alias,
 +                      funding_txo,
 +                      config,
 +                      short_channel_id,
 +                      channel_value_satoshis: channel_value_satoshis.0.unwrap(),
 +                      unspendable_punishment_reserve,
 +                      user_channel_id,
 +                      balance_msat: balance_msat.0.unwrap(),
 +                      outbound_capacity_msat: outbound_capacity_msat.0.unwrap(),
 +                      next_outbound_htlc_limit_msat: next_outbound_htlc_limit_msat.0.unwrap(),
 +                      inbound_capacity_msat: inbound_capacity_msat.0.unwrap(),
 +                      confirmations_required,
 +                      force_close_spend_delay,
 +                      is_outbound: is_outbound.0.unwrap(),
 +                      is_channel_ready: is_channel_ready.0.unwrap(),
 +                      is_usable: is_usable.0.unwrap(),
 +                      is_public: is_public.0.unwrap(),
 +                      inbound_htlc_minimum_msat,
 +                      inbound_htlc_maximum_msat,
 +              })
 +      }
 +}
  
  impl_writeable_tlv_based!(PhantomRouteHints, {
        (2, channels, vec_type),
@@@ -6614,9 -6462,8 +6618,9 @@@ impl_writeable_tlv_based!(PendingHTLCIn
        (0, routing, required),
        (2, incoming_shared_secret, required),
        (4, payment_hash, required),
 -      (6, amt_to_forward, required),
 -      (8, outgoing_cltv_value, required)
 +      (6, outgoing_amt_msat, required),
 +      (8, outgoing_cltv_value, required),
 +      (9, incoming_amt_msat, option),
  });
  
  
@@@ -6815,7 -6662,7 +6819,7 @@@ impl Writeable for HTLCSource 
                                        (1, payment_id_opt, option),
                                        (2, first_hop_htlc_msat, required),
                                        (3, payment_secret, option),
 -                                      (4, path, vec_type),
 +                                      (4, *path, vec_type),
                                        (5, payment_params, option),
                                 });
                        }
@@@ -6838,20 -6685,18 +6842,20 @@@ impl_writeable_tlv_based_enum!(HTLCFail
        },
  ;);
  
 +impl_writeable_tlv_based!(PendingAddHTLCInfo, {
 +      (0, forward_info, required),
 +      (2, prev_short_channel_id, required),
 +      (4, prev_htlc_id, required),
 +      (6, prev_funding_outpoint, required),
 +});
 +
  impl_writeable_tlv_based_enum!(HTLCForwardInfo,
 -      (0, AddHTLC) => {
 -              (0, forward_info, required),
 -              (2, prev_short_channel_id, required),
 -              (4, prev_htlc_id, required),
 -              (6, prev_funding_outpoint, required),
 -      },
        (1, FailHTLC) => {
                (0, htlc_id, required),
                (2, err_packet, required),
 -      },
 -;);
 +      };
 +      (0, AddHTLC)
 +);
  
  impl_writeable_tlv_based!(PendingInboundPayment, {
        (0, payment_secret, required),
@@@ -6934,10 -6779,13 +6938,13 @@@ impl<M: Deref, T: Deref, K: Deref, F: D
                        }
                }
  
-               let channel_state = self.channel_state.lock().unwrap();
+               let pending_inbound_payments = self.pending_inbound_payments.lock().unwrap();
+               let claimable_htlcs = self.claimable_htlcs.lock().unwrap();
+               let pending_outbound_payments = self.pending_outbound_payments.lock().unwrap();
                let mut htlc_purposes: Vec<&events::PaymentPurpose> = Vec::new();
-               (channel_state.claimable_htlcs.len() as u64).write(writer)?;
-               for (payment_hash, (purpose, previous_hops)) in channel_state.claimable_htlcs.iter() {
+               (claimable_htlcs.len() as u64).write(writer)?;
+               for (payment_hash, (purpose, previous_hops)) in claimable_htlcs.iter() {
                        payment_hash.write(writer)?;
                        (previous_hops.len() as u64).write(writer)?;
                        for htlc in previous_hops.iter() {
                        peer_state.latest_features.write(writer)?;
                }
  
-               let pending_inbound_payments = self.pending_inbound_payments.lock().unwrap();
-               let pending_outbound_payments = self.pending_outbound_payments.lock().unwrap();
                let events = self.pending_events.lock().unwrap();
                (events.len() as u64).write(writer)?;
                for event in events.iter() {
  /// which you've already broadcasted the transaction.
  ///
  /// [`ChainMonitor`]: crate::chain::chainmonitor::ChainMonitor
 -pub struct ChannelManagerReadArgs<'a, Signer: 'a + Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 -      where M::Target: chain::Watch<Signer>,
 +pub struct ChannelManagerReadArgs<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 +      where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
          T::Target: BroadcasterInterface,
 -        K::Target: KeysInterface<Signer = Signer>,
 +        K::Target: KeysInterface,
          F::Target: FeeEstimator,
          L::Target: Logger,
  {
        /// this struct.
        ///
        /// (C-not exported) because we have no HashMap bindings
 -      pub channel_monitors: HashMap<OutPoint, &'a mut ChannelMonitor<Signer>>,
 +      pub channel_monitors: HashMap<OutPoint, &'a mut ChannelMonitor<<K::Target as KeysInterface>::Signer>>,
  }
  
 -impl<'a, Signer: 'a + Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 -              ChannelManagerReadArgs<'a, Signer, M, T, K, F, L>
 -      where M::Target: chain::Watch<Signer>,
 +impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 +              ChannelManagerReadArgs<'a, M, T, K, F, L>
 +      where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
                T::Target: BroadcasterInterface,
 -              K::Target: KeysInterface<Signer = Signer>,
 +              K::Target: KeysInterface,
                F::Target: FeeEstimator,
                L::Target: Logger,
        {
        /// HashMap for you. This is primarily useful for C bindings where it is not practical to
        /// populate a HashMap directly from C.
        pub fn new(keys_manager: K, fee_estimator: F, chain_monitor: M, tx_broadcaster: T, logger: L, default_config: UserConfig,
 -                      mut channel_monitors: Vec<&'a mut ChannelMonitor<Signer>>) -> Self {
 +                      mut channel_monitors: Vec<&'a mut ChannelMonitor<<K::Target as KeysInterface>::Signer>>) -> Self {
                Self {
                        keys_manager, fee_estimator, chain_monitor, tx_broadcaster, logger, default_config,
                        channel_monitors: channel_monitors.drain(..).map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect()
  // Implement ReadableArgs for an Arc'd ChannelManager to make it a bit easier to work with the
  // SipmleArcChannelManager type:
  impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 -      ReadableArgs<ChannelManagerReadArgs<'a, <K::Target as KeysInterface>::Signer, M, T, K, F, L>> for (BlockHash, Arc<ChannelManager<M, T, K, F, L>>)
 +      ReadableArgs<ChannelManagerReadArgs<'a, M, T, K, F, L>> for (BlockHash, Arc<ChannelManager<M, T, K, F, L>>)
        where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
          T::Target: BroadcasterInterface,
          K::Target: KeysInterface,
          F::Target: FeeEstimator,
          L::Target: Logger,
  {
 -      fn read<R: io::Read>(reader: &mut R, args: ChannelManagerReadArgs<'a, <K::Target as KeysInterface>::Signer, M, T, K, F, L>) -> Result<Self, DecodeError> {
 +      fn read<R: io::Read>(reader: &mut R, args: ChannelManagerReadArgs<'a, M, T, K, F, L>) -> Result<Self, DecodeError> {
                let (blockhash, chan_manager) = <(BlockHash, ChannelManager<M, T, K, F, L>)>::read(reader, args)?;
                Ok((blockhash, Arc::new(chan_manager)))
        }
  }
  
  impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 -      ReadableArgs<ChannelManagerReadArgs<'a, <K::Target as KeysInterface>::Signer, M, T, K, F, L>> for (BlockHash, ChannelManager<M, T, K, F, L>)
 +      ReadableArgs<ChannelManagerReadArgs<'a, M, T, K, F, L>> for (BlockHash, ChannelManager<M, T, K, F, L>)
        where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
          T::Target: BroadcasterInterface,
          K::Target: KeysInterface,
          F::Target: FeeEstimator,
          L::Target: Logger,
  {
 -      fn read<R: io::Read>(reader: &mut R, mut args: ChannelManagerReadArgs<'a, <K::Target as KeysInterface>::Signer, M, T, K, F, L>) -> Result<Self, DecodeError> {
 +      fn read<R: io::Read>(reader: &mut R, mut args: ChannelManagerReadArgs<'a, M, T, K, F, L>) -> Result<Self, DecodeError> {
                let _ver = read_ver_prefix!(reader, SERIALIZATION_VERSION);
  
                let genesis_hash: BlockHash = Readable::read(reader)?;
  
                        channel_state: Mutex::new(ChannelHolder {
                                by_id,
-                               claimable_htlcs,
                                pending_msg_events: Vec::new(),
                        }),
                        inbound_payment_key: expanded_inbound_key,
                        pending_outbound_payments: Mutex::new(pending_outbound_payments.unwrap()),
  
                        forward_htlcs: Mutex::new(forward_htlcs),
+                       claimable_htlcs: Mutex::new(claimable_htlcs),
                        outbound_scid_aliases: Mutex::new(outbound_scid_aliases),
                        id_to_peer: Mutex::new(id_to_peer),
                        short_to_chan_info: FairRwLock::new(short_to_chan_info),