Merge pull request #1796 from tnull/2022-10-track-confirmation-block-hash
authorMatt Corallo <649246+TheBlueMatt@users.noreply.github.com>
Wed, 9 Nov 2022 20:24:10 +0000 (20:24 +0000)
committerGitHub <noreply@github.com>
Wed, 9 Nov 2022 20:24:10 +0000 (20:24 +0000)
Track confirmation block hash and return via `Confirm::get_relevant_txids`

22 files changed:
CONTRIBUTING.md
lightning-rapid-gossip-sync/src/processing.rs
lightning/src/chain/chaininterface.rs
lightning/src/chain/chainmonitor.rs
lightning/src/chain/channelmonitor.rs
lightning/src/lib.rs
lightning/src/ln/channelmanager.rs
lightning/src/ln/features.rs
lightning/src/ln/onion_route_tests.rs
lightning/src/ln/reorg_tests.rs
lightning/src/offers/mod.rs [new file with mode: 0644]
lightning/src/offers/offer.rs [new file with mode: 0644]
lightning/src/onion_message/blinded_route.rs
lightning/src/onion_message/mod.rs
lightning/src/onion_message/packet.rs
lightning/src/routing/gossip.rs
lightning/src/util/events.rs
lightning/src/util/mod.rs
lightning/src/util/scid_utils.rs
lightning/src/util/ser.rs
lightning/src/util/ser_macros.rs
lightning/src/util/string.rs [new file with mode: 0644]

index f1e4ce5171e71626f124701741e4f485a96f9ad5..6f12c21e9f9d6c9035d3225316d3e5ed42b5d3eb 100644 (file)
@@ -17,8 +17,8 @@ Communication Channels
 
 Communication about the development of LDK and `rust-lightning` happens
 primarily on the [LDK Discord](https://discord.gg/5AcknnMfBw) in the `#ldk-dev`
-channel. Additionally, live LDK devevelopment meetings are held every other
-Monday 19:00 UTC in the [LDK Dev Jitsi Meeting
+channel. Additionally, live LDK development meetings are held every other
+Monday 17:00 UTC in the [LDK Dev Jitsi Meeting
 Room](https://meet.jit.si/ldkdevmeeting). Upcoming events can be found in the
 [LDK calendar](https://calendar.google.com/calendar/embed?src=c_e6fv6vlshbpoob2mmbvblkkoj4%40group.calendar.google.com).
 
index 1a46ef587afad0ef28d90d95ead951e1d1f8f69e..3e001ec45c4be1aa7becbaa6da9f9158f3d217fa 100644 (file)
@@ -37,12 +37,9 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
                let mut prefix = [0u8; 4];
                read_cursor.read_exact(&mut prefix)?;
 
-               match prefix {
-                       GOSSIP_PREFIX => {}
-                       _ => {
-                               return Err(DecodeError::UnknownVersion.into());
-                       }
-               };
+               if prefix != GOSSIP_PREFIX {
+                       return Err(DecodeError::UnknownVersion.into());
+               }
 
                let chain_hash: BlockHash = Readable::read(read_cursor)?;
                let latest_seen_timestamp: u32 = Readable::read(read_cursor)?;
@@ -75,6 +72,7 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
 
                        let node_id_1_index: BigSize = Readable::read(read_cursor)?;
                        let node_id_2_index: BigSize = Readable::read(read_cursor)?;
+
                        if max(node_id_1_index.0, node_id_2_index.0) >= node_id_count as u64 {
                                return Err(DecodeError::InvalidValue.into());
                        };
@@ -123,49 +121,43 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
                        // flags are always sent in full, and hence always need updating
                        let standard_channel_flags = channel_flags & 0b_0000_0011;
 
-                       let mut synthetic_update = if channel_flags & 0b_1000_0000 == 0 {
-                               // full update, field flags will indicate deviations from the default
-                               UnsignedChannelUpdate {
-                                       chain_hash,
-                                       short_channel_id,
-                                       timestamp: backdated_timestamp,
-                                       flags: standard_channel_flags,
-                                       cltv_expiry_delta: default_cltv_expiry_delta,
-                                       htlc_minimum_msat: default_htlc_minimum_msat,
-                                       htlc_maximum_msat: default_htlc_maximum_msat,
-                                       fee_base_msat: default_fee_base_msat,
-                                       fee_proportional_millionths: default_fee_proportional_millionths,
-                                       excess_data: Vec::new(),
-                               }
-                       } else {
+                       let mut synthetic_update = UnsignedChannelUpdate {
+                               chain_hash,
+                               short_channel_id,
+                               timestamp: backdated_timestamp,
+                               flags: standard_channel_flags,
+                               cltv_expiry_delta: default_cltv_expiry_delta,
+                               htlc_minimum_msat: default_htlc_minimum_msat,
+                               htlc_maximum_msat: default_htlc_maximum_msat,
+                               fee_base_msat: default_fee_base_msat,
+                               fee_proportional_millionths: default_fee_proportional_millionths,
+                               excess_data: Vec::new(),
+                       };
+
+                       let mut skip_update_for_unknown_channel = false;
+
+                       if (channel_flags & 0b_1000_0000) != 0 {
                                // incremental update, field flags will indicate mutated values
                                let read_only_network_graph = network_graph.read_only();
-                               let channel = read_only_network_graph
+                               if let Some(channel) = read_only_network_graph
                                        .channels()
-                                       .get(&short_channel_id)
-                                       .ok_or(LightningError {
-                                               err: "Couldn't find channel for update".to_owned(),
-                                               action: ErrorAction::IgnoreError,
-                                       })?;
-
-                               let directional_info = channel
-                                       .get_directional_info(channel_flags)
-                                       .ok_or(LightningError {
-                                               err: "Couldn't find previous directional data for update".to_owned(),
-                                               action: ErrorAction::IgnoreError,
-                                       })?;
-
-                               UnsignedChannelUpdate {
-                                       chain_hash,
-                                       short_channel_id,
-                                       timestamp: backdated_timestamp,
-                                       flags: standard_channel_flags,
-                                       cltv_expiry_delta: directional_info.cltv_expiry_delta,
-                                       htlc_minimum_msat: directional_info.htlc_minimum_msat,
-                                       htlc_maximum_msat: directional_info.htlc_maximum_msat,
-                                       fee_base_msat: directional_info.fees.base_msat,
-                                       fee_proportional_millionths: directional_info.fees.proportional_millionths,
-                                       excess_data: Vec::new(),
+                                       .get(&short_channel_id) {
+
+                                       let directional_info = channel
+                                               .get_directional_info(channel_flags)
+                                               .ok_or(LightningError {
+                                                       err: "Couldn't find previous directional data for update".to_owned(),
+                                                       action: ErrorAction::IgnoreError,
+                                               })?;
+
+                                       synthetic_update.cltv_expiry_delta = directional_info.cltv_expiry_delta;
+                                       synthetic_update.htlc_minimum_msat = directional_info.htlc_minimum_msat;
+                                       synthetic_update.htlc_maximum_msat = directional_info.htlc_maximum_msat;
+                                       synthetic_update.fee_base_msat = directional_info.fees.base_msat;
+                                       synthetic_update.fee_proportional_millionths = directional_info.fees.proportional_millionths;
+
+                               } else {
+                                       skip_update_for_unknown_channel = true;
                                }
                        };
 
@@ -194,6 +186,10 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
                                synthetic_update.htlc_maximum_msat = htlc_maximum_msat;
                        }
 
+                       if skip_update_for_unknown_channel {
+                               continue;
+                       }
+
                        match network_graph.update_channel_unsigned(&synthetic_update) {
                                Ok(_) => {},
                                Err(LightningError { action: ErrorAction::IgnoreDuplicateGossip, .. }) => {},
@@ -254,7 +250,7 @@ mod tests {
        }
 
        #[test]
-       fn incremental_only_update_fails_without_prior_announcements() {
+       fn incremental_only_update_ignores_missing_channel() {
                let incremental_update_input = vec![
                        76, 68, 75, 1, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
                        79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0, 0, 0, 0, 97, 229, 183, 167,
@@ -271,12 +267,7 @@ mod tests {
 
                let rapid_sync = RapidGossipSync::new(&network_graph);
                let update_result = rapid_sync.update_network_graph(&incremental_update_input[..]);
-               assert!(update_result.is_err());
-               if let Err(GraphSyncError::LightningError(lightning_error)) = update_result {
-                       assert_eq!(lightning_error.err, "Couldn't find channel for update");
-               } else {
-                       panic!("Unexpected update result: {:?}", update_result)
-               }
+               assert!(update_result.is_ok());
        }
 
        #[test]
@@ -534,4 +525,39 @@ mod tests {
                assert!(after.contains("619737530008010752"));
                assert!(after.contains("783241506229452801"));
        }
+
+       #[test]
+       pub fn update_fails_with_unknown_version() {
+               let unknown_version_input = vec![
+                       76, 68, 75, 2, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
+                       79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0, 0, 0, 0, 97, 227, 98, 218,
+                       0, 0, 0, 4, 2, 22, 7, 207, 206, 25, 164, 197, 231, 230, 231, 56, 102, 61, 250, 251,
+                       187, 172, 38, 46, 79, 247, 108, 44, 155, 48, 219, 238, 252, 53, 192, 6, 67, 2, 36, 125,
+                       157, 176, 223, 175, 234, 116, 94, 248, 201, 225, 97, 235, 50, 47, 115, 172, 63, 136,
+                       88, 216, 115, 11, 111, 217, 114, 84, 116, 124, 231, 107, 2, 158, 1, 242, 121, 152, 106,
+                       204, 131, 186, 35, 93, 70, 216, 10, 237, 224, 183, 89, 95, 65, 3, 83, 185, 58, 138,
+                       181, 64, 187, 103, 127, 68, 50, 2, 201, 19, 17, 138, 136, 149, 185, 226, 156, 137, 175,
+                       110, 32, 237, 0, 217, 90, 31, 100, 228, 149, 46, 219, 175, 168, 77, 4, 143, 38, 128,
+                       76, 97, 0, 0, 0, 2, 0, 0, 255, 8, 153, 192, 0, 2, 27, 0, 0, 0, 1, 0, 0, 255, 2, 68,
+                       226, 0, 6, 11, 0, 1, 2, 3, 0, 0, 0, 4, 0, 40, 0, 0, 0, 0, 0, 0, 3, 232, 0, 0, 3, 232,
+                       0, 0, 0, 1, 0, 0, 0, 0, 29, 129, 25, 192, 255, 8, 153, 192, 0, 2, 27, 0, 0, 60, 0, 0,
+                       0, 0, 0, 0, 0, 1, 0, 0, 0, 100, 0, 0, 2, 224, 0, 0, 0, 0, 58, 85, 116, 216, 0, 29, 0,
+                       0, 0, 1, 0, 0, 0, 125, 0, 0, 0, 0, 58, 85, 116, 216, 255, 2, 68, 226, 0, 6, 11, 0, 1,
+                       0, 0, 1,
+               ];
+
+               let block_hash = genesis_block(Network::Bitcoin).block_hash();
+               let logger = TestLogger::new();
+               let network_graph = NetworkGraph::new(block_hash, &logger);
+               let rapid_sync = RapidGossipSync::new(&network_graph);
+               let update_result = rapid_sync.update_network_graph(&unknown_version_input[..]);
+
+               assert!(update_result.is_err());
+
+               if let Err(GraphSyncError::DecodeError(DecodeError::UnknownVersion)) = update_result {
+                       // this is the expected error type
+               } else {
+                       panic!("Unexpected update result: {:?}", update_result)
+               }
+       }
 }
index cbf609485ce1984800fa991e22d952e33ab5db56..1a446249105c8772ead4b9f1c1167d3ea160219a 100644 (file)
@@ -25,7 +25,7 @@ pub trait BroadcasterInterface {
 
 /// An enum that represents the speed at which we want a transaction to confirm used for feerate
 /// estimation.
-#[derive(Clone, Copy, PartialEq, Eq)]
+#[derive(Clone, Copy, Hash, PartialEq, Eq)]
 pub enum ConfirmationTarget {
        /// We are happy with this transaction confirming slowly when feerate drops some.
        Background,
index 00e25ae9db1dc9a66095b280b67323ff6c4f3de1..fc4caef8afd90f6fa68a2d0e43238732d4c04e6c 100644 (file)
@@ -395,6 +395,23 @@ where C::Target: chain::Filter,
                self.monitors.read().unwrap().keys().map(|outpoint| *outpoint).collect()
        }
 
+       #[cfg(not(c_bindings))]
+       /// Lists the pending updates for each [`ChannelMonitor`] (by `OutPoint` being monitored).
+       pub fn list_pending_monitor_updates(&self) -> HashMap<OutPoint, Vec<MonitorUpdateId>> {
+               self.monitors.read().unwrap().iter().map(|(outpoint, holder)| {
+                       (*outpoint, holder.pending_monitor_updates.lock().unwrap().clone())
+               }).collect()
+       }
+
+       #[cfg(c_bindings)]
+       /// Lists the pending updates for each [`ChannelMonitor`] (by `OutPoint` being monitored).
+       pub fn list_pending_monitor_updates(&self) -> Vec<(OutPoint, Vec<MonitorUpdateId>)> {
+               self.monitors.read().unwrap().iter().map(|(outpoint, holder)| {
+                       (*outpoint, holder.pending_monitor_updates.lock().unwrap().clone())
+               }).collect()
+       }
+
+
        #[cfg(test)]
        pub fn remove_monitor(&self, funding_txo: &OutPoint) -> ChannelMonitor<ChannelSigner> {
                self.monitors.write().unwrap().remove(funding_txo).unwrap().monitor
@@ -798,7 +815,22 @@ mod tests {
                // Note that updates is a HashMap so the ordering here is actually random. This shouldn't
                // fail either way but if it fails intermittently it's depending on the ordering of updates.
                let mut update_iter = updates.iter();
-               nodes[1].chain_monitor.chain_monitor.channel_monitor_updated(*funding_txo, update_iter.next().unwrap().clone()).unwrap();
+               let next_update = update_iter.next().unwrap().clone();
+               // Should contain next_update when pending updates listed.
+               #[cfg(not(c_bindings))]
+               assert!(nodes[1].chain_monitor.chain_monitor.list_pending_monitor_updates().get(funding_txo)
+                       .unwrap().contains(&next_update));
+               #[cfg(c_bindings)]
+               assert!(nodes[1].chain_monitor.chain_monitor.list_pending_monitor_updates().iter()
+                       .find(|(txo, _)| txo == funding_txo).unwrap().1.contains(&next_update));
+               nodes[1].chain_monitor.chain_monitor.channel_monitor_updated(*funding_txo, next_update.clone()).unwrap();
+               // Should not contain the previously pending next_update when pending updates listed.
+               #[cfg(not(c_bindings))]
+               assert!(!nodes[1].chain_monitor.chain_monitor.list_pending_monitor_updates().get(funding_txo)
+                       .unwrap().contains(&next_update));
+               #[cfg(c_bindings)]
+               assert!(!nodes[1].chain_monitor.chain_monitor.list_pending_monitor_updates().iter()
+                       .find(|(txo, _)| txo == funding_txo).unwrap().1.contains(&next_update));
                assert!(nodes[1].chain_monitor.release_pending_monitor_events().is_empty());
                assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
                nodes[1].chain_monitor.chain_monitor.channel_monitor_updated(*funding_txo, update_iter.next().unwrap().clone()).unwrap();
index e225f1f095820f49f46d7d8250ca2502a2d40d41..0bfd944456ab6851749505350c781e8ec267d27e 100644 (file)
@@ -1309,9 +1309,11 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
        /// Gets the list of pending events which were generated by previous actions, clearing the list
        /// in the process.
        ///
-       /// This is called by ChainMonitor::get_and_clear_pending_events() and is equivalent to
-       /// EventsProvider::get_and_clear_pending_events() except that it requires &mut self as we do
-       /// no internal locking in ChannelMonitors.
+       /// This is called by the [`EventsProvider::process_pending_events`] implementation for
+       /// [`ChainMonitor`].
+       ///
+       /// [`EventsProvider::process_pending_events`]: crate::util::events::EventsProvider::process_pending_events
+       /// [`ChainMonitor`]: crate::chain::chainmonitor::ChainMonitor
        pub fn get_and_clear_pending_events(&self) -> Vec<Event> {
                self.inner.lock().unwrap().get_and_clear_pending_events()
        }
index 25eba1d74f8f87ddb60fd80d610b187f853f81ce..1f3ab47b1ae3f7d2ca8802e46c267d5428e3822c 100644 (file)
@@ -78,6 +78,8 @@ extern crate core;
 pub mod util;
 pub mod chain;
 pub mod ln;
+#[allow(unused)]
+mod offers;
 pub mod routing;
 pub mod onion_message;
 
index a24e623e3c47c4d206fa73b8d45adaa928b41531..91a853132064049abca699a211de956e6e5cdc22 100644 (file)
@@ -66,7 +66,7 @@ use crate::prelude::*;
 use core::{cmp, mem};
 use core::cell::RefCell;
 use crate::io::Read;
-use crate::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard};
+use crate::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, FairRwLock};
 use core::sync::atomic::{AtomicUsize, Ordering};
 use core::time::Duration;
 use core::ops::Deref;
@@ -113,6 +113,7 @@ pub(super) struct PendingHTLCInfo {
        pub(super) incoming_shared_secret: [u8; 32],
        payment_hash: PaymentHash,
        pub(super) amt_to_forward: u64,
+       pub(super) amt_incoming: Option<u64>, // Added in 0.0.113
        pub(super) outgoing_cltv_value: u32,
 }
 
@@ -129,20 +130,22 @@ pub(super) enum PendingHTLCStatus {
        Fail(HTLCFailureMsg),
 }
 
-pub(super) enum HTLCForwardInfo {
-       AddHTLC {
-               forward_info: PendingHTLCInfo,
+pub(super) struct PendingAddHTLCInfo {
+       pub(super) forward_info: PendingHTLCInfo,
 
-               // These fields are produced in `forward_htlcs()` and consumed in
-               // `process_pending_htlc_forwards()` for constructing the
-               // `HTLCSource::PreviousHopData` for failed and forwarded
-               // HTLCs.
-               //
-               // Note that this may be an outbound SCID alias for the associated channel.
-               prev_short_channel_id: u64,
-               prev_htlc_id: u64,
-               prev_funding_outpoint: OutPoint,
-       },
+       // These fields are produced in `forward_htlcs()` and consumed in
+       // `process_pending_htlc_forwards()` for constructing the
+       // `HTLCSource::PreviousHopData` for failed and forwarded
+       // HTLCs.
+       //
+       // Note that this may be an outbound SCID alias for the associated channel.
+       prev_short_channel_id: u64,
+       prev_htlc_id: u64,
+       prev_funding_outpoint: OutPoint,
+}
+
+pub(super) enum HTLCForwardInfo {
+       AddHTLC(PendingAddHTLCInfo),
        FailHTLC {
                htlc_id: u64,
                err_packet: msgs::OnionErrorPacket,
@@ -395,12 +398,6 @@ pub(super) enum RAACommitmentOrder {
 // Note this is only exposed in cfg(test):
 pub(super) struct ChannelHolder<Signer: Sign> {
        pub(super) by_id: HashMap<[u8; 32], Channel<Signer>>,
-       /// SCIDs (and outbound SCID aliases) -> `counterparty_node_id`s and `channel_id`s.
-       ///
-       /// Outbound SCID aliases are added here once the channel is available for normal use, with
-       /// SCIDs being added once the funding transaction is confirmed at the channel's required
-       /// confirmation depth.
-       pub(super) short_to_chan_info: HashMap<u64, (PublicKey, [u8; 32])>,
        /// Map from payment hash to the payment data and any HTLCs which are to us and can be
        /// failed/claimed by the user.
        ///
@@ -680,6 +677,8 @@ pub type SimpleRefChannelManager<'a, 'b, 'c, 'd, 'e, M, T, F, L> = ChannelManage
 //  |   |
 //  |   |__`id_to_peer`
 //  |   |
+//  |   |__`short_to_chan_info`
+//  |   |
 //  |   |__`per_peer_state`
 //  |       |
 //  |       |__`outbound_scid_aliases`
@@ -786,6 +785,22 @@ pub struct ChannelManager<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
        /// See `ChannelManager` struct-level documentation for lock order requirements.
        id_to_peer: Mutex<HashMap<[u8; 32], PublicKey>>,
 
+       /// SCIDs (and outbound SCID aliases) -> `counterparty_node_id`s and `channel_id`s.
+       ///
+       /// Outbound SCID aliases are added here once the channel is available for normal use, with
+       /// SCIDs being added once the funding transaction is confirmed at the channel's required
+       /// confirmation depth.
+       ///
+       /// Note that while this holds `counterparty_node_id`s and `channel_id`s, no consistency
+       /// guarantees are made about the existence of a peer with the `counterparty_node_id` nor a
+       /// channel with the `channel_id` in our other maps.
+       ///
+       /// See `ChannelManager` struct-level documentation for lock order requirements.
+       #[cfg(test)]
+       pub(super) short_to_chan_info: FairRwLock<HashMap<u64, (PublicKey, [u8; 32])>>,
+       #[cfg(not(test))]
+       short_to_chan_info: FairRwLock<HashMap<u64, (PublicKey, [u8; 32])>>,
+
        our_network_key: SecretKey,
        our_network_pubkey: PublicKey,
 
@@ -1295,9 +1310,11 @@ macro_rules! handle_error {
 }
 
 macro_rules! update_maps_on_chan_removal {
-       ($self: expr, $short_to_chan_info: expr, $channel: expr) => {
+       ($self: expr, $channel: expr) => {{
+               $self.id_to_peer.lock().unwrap().remove(&$channel.channel_id());
+               let mut short_to_chan_info = $self.short_to_chan_info.write().unwrap();
                if let Some(short_id) = $channel.get_short_channel_id() {
-                       $short_to_chan_info.remove(&short_id);
+                       short_to_chan_info.remove(&short_id);
                } else {
                        // If the channel was never confirmed on-chain prior to its closure, remove the
                        // outbound SCID alias we used for it from the collision-prevention set. While we
@@ -1308,14 +1325,13 @@ macro_rules! update_maps_on_chan_removal {
                        let alias_removed = $self.outbound_scid_aliases.lock().unwrap().remove(&$channel.outbound_scid_alias());
                        debug_assert!(alias_removed);
                }
-               $self.id_to_peer.lock().unwrap().remove(&$channel.channel_id());
-               $short_to_chan_info.remove(&$channel.outbound_scid_alias());
-       }
+               short_to_chan_info.remove(&$channel.outbound_scid_alias());
+       }}
 }
 
 /// Returns (boolean indicating if we should remove the Channel object from memory, a mapped error)
 macro_rules! convert_chan_err {
-       ($self: ident, $err: expr, $short_to_chan_info: expr, $channel: expr, $channel_id: expr) => {
+       ($self: ident, $err: expr, $channel: expr, $channel_id: expr) => {
                match $err {
                        ChannelError::Warn(msg) => {
                                (false, MsgHandleErrInternal::from_chan_no_close(ChannelError::Warn(msg), $channel_id.clone()))
@@ -1325,7 +1341,7 @@ macro_rules! convert_chan_err {
                        },
                        ChannelError::Close(msg) => {
                                log_error!($self.logger, "Closing channel {} due to close-required error: {}", log_bytes!($channel_id[..]), msg);
-                               update_maps_on_chan_removal!($self, $short_to_chan_info, $channel);
+                               update_maps_on_chan_removal!($self, $channel);
                                let shutdown_res = $channel.force_shutdown(true);
                                (true, MsgHandleErrInternal::from_finish_shutdown(msg, *$channel_id, $channel.get_user_id(),
                                        shutdown_res, $self.get_channel_update_for_broadcast(&$channel).ok()))
@@ -1335,11 +1351,11 @@ macro_rules! convert_chan_err {
 }
 
 macro_rules! break_chan_entry {
-       ($self: ident, $res: expr, $channel_state: expr, $entry: expr) => {
+       ($self: ident, $res: expr, $entry: expr) => {
                match $res {
                        Ok(res) => res,
                        Err(e) => {
-                               let (drop, res) = convert_chan_err!($self, e, $channel_state.short_to_chan_info, $entry.get_mut(), $entry.key());
+                               let (drop, res) = convert_chan_err!($self, e, $entry.get_mut(), $entry.key());
                                if drop {
                                        $entry.remove_entry();
                                }
@@ -1350,11 +1366,11 @@ macro_rules! break_chan_entry {
 }
 
 macro_rules! try_chan_entry {
-       ($self: ident, $res: expr, $channel_state: expr, $entry: expr) => {
+       ($self: ident, $res: expr, $entry: expr) => {
                match $res {
                        Ok(res) => res,
                        Err(e) => {
-                               let (drop, res) = convert_chan_err!($self, e, $channel_state.short_to_chan_info, $entry.get_mut(), $entry.key());
+                               let (drop, res) = convert_chan_err!($self, e, $entry.get_mut(), $entry.key());
                                if drop {
                                        $entry.remove_entry();
                                }
@@ -1365,21 +1381,21 @@ macro_rules! try_chan_entry {
 }
 
 macro_rules! remove_channel {
-       ($self: expr, $channel_state: expr, $entry: expr) => {
+       ($self: expr, $entry: expr) => {
                {
                        let channel = $entry.remove_entry().1;
-                       update_maps_on_chan_removal!($self, $channel_state.short_to_chan_info, channel);
+                       update_maps_on_chan_removal!($self, channel);
                        channel
                }
        }
 }
 
 macro_rules! handle_monitor_update_res {
-       ($self: ident, $err: expr, $short_to_chan_info: expr, $chan: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $resend_channel_ready: expr, $failed_forwards: expr, $failed_fails: expr, $failed_finalized_fulfills: expr, $chan_id: expr) => {
+       ($self: ident, $err: expr, $chan: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $resend_channel_ready: expr, $failed_forwards: expr, $failed_fails: expr, $failed_finalized_fulfills: expr, $chan_id: expr) => {
                match $err {
                        ChannelMonitorUpdateStatus::PermanentFailure => {
                                log_error!($self.logger, "Closing channel {} due to monitor update ChannelMonitorUpdateStatus::PermanentFailure", log_bytes!($chan_id[..]));
-                               update_maps_on_chan_removal!($self, $short_to_chan_info, $chan);
+                               update_maps_on_chan_removal!($self, $chan);
                                // TODO: $failed_fails is dropped here, which will cause other channels to hit the
                                // chain in a confused state! We need to move them into the ChannelMonitor which
                                // will be responsible for failing backwards once things confirm on-chain.
@@ -1421,48 +1437,49 @@ macro_rules! handle_monitor_update_res {
                        },
                }
        };
-       ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $resend_channel_ready: expr, $failed_forwards: expr, $failed_fails: expr, $failed_finalized_fulfills: expr) => { {
-               let (res, drop) = handle_monitor_update_res!($self, $err, $channel_state.short_to_chan_info, $entry.get_mut(), $action_type, $resend_raa, $resend_commitment, $resend_channel_ready, $failed_forwards, $failed_fails, $failed_finalized_fulfills, $entry.key());
+       ($self: ident, $err: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $resend_channel_ready: expr, $failed_forwards: expr, $failed_fails: expr, $failed_finalized_fulfills: expr) => { {
+               let (res, drop) = handle_monitor_update_res!($self, $err, $entry.get_mut(), $action_type, $resend_raa, $resend_commitment, $resend_channel_ready, $failed_forwards, $failed_fails, $failed_finalized_fulfills, $entry.key());
                if drop {
                        $entry.remove_entry();
                }
                res
        } };
-       ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $chan_id: expr, COMMITMENT_UPDATE_ONLY) => { {
+       ($self: ident, $err: expr, $entry: expr, $action_type: path, $chan_id: expr, COMMITMENT_UPDATE_ONLY) => { {
                debug_assert!($action_type == RAACommitmentOrder::CommitmentFirst);
-               handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, false, true, false, Vec::new(), Vec::new(), Vec::new(), $chan_id)
+               handle_monitor_update_res!($self, $err, $entry, $action_type, false, true, false, Vec::new(), Vec::new(), Vec::new(), $chan_id)
        } };
-       ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $chan_id: expr, NO_UPDATE) => {
-               handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, false, false, false, Vec::new(), Vec::new(), Vec::new(), $chan_id)
+       ($self: ident, $err: expr, $entry: expr, $action_type: path, $chan_id: expr, NO_UPDATE) => {
+               handle_monitor_update_res!($self, $err, $entry, $action_type, false, false, false, Vec::new(), Vec::new(), Vec::new(), $chan_id)
        };
-       ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $resend_channel_ready: expr, OPTIONALLY_RESEND_FUNDING_LOCKED) => {
-               handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, false, false, $resend_channel_ready, Vec::new(), Vec::new(), Vec::new())
+       ($self: ident, $err: expr, $entry: expr, $action_type: path, $resend_channel_ready: expr, OPTIONALLY_RESEND_FUNDING_LOCKED) => {
+               handle_monitor_update_res!($self, $err, $entry, $action_type, false, false, $resend_channel_ready, Vec::new(), Vec::new(), Vec::new())
        };
-       ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr) => {
-               handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, $resend_raa, $resend_commitment, false, Vec::new(), Vec::new(), Vec::new())
+       ($self: ident, $err: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr) => {
+               handle_monitor_update_res!($self, $err, $entry, $action_type, $resend_raa, $resend_commitment, false, Vec::new(), Vec::new(), Vec::new())
        };
-       ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $failed_forwards: expr, $failed_fails: expr) => {
-               handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, $resend_raa, $resend_commitment, false, $failed_forwards, $failed_fails, Vec::new())
+       ($self: ident, $err: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $failed_forwards: expr, $failed_fails: expr) => {
+               handle_monitor_update_res!($self, $err, $entry, $action_type, $resend_raa, $resend_commitment, false, $failed_forwards, $failed_fails, Vec::new())
        };
 }
 
 macro_rules! send_channel_ready {
-       ($short_to_chan_info: expr, $pending_msg_events: expr, $channel: expr, $channel_ready_msg: expr) => {
+       ($self: ident, $pending_msg_events: expr, $channel: expr, $channel_ready_msg: expr) => {{
                $pending_msg_events.push(events::MessageSendEvent::SendChannelReady {
                        node_id: $channel.get_counterparty_node_id(),
                        msg: $channel_ready_msg,
                });
                // Note that we may send a `channel_ready` multiple times for a channel if we reconnect, so
                // we allow collisions, but we shouldn't ever be updating the channel ID pointed to.
-               let outbound_alias_insert = $short_to_chan_info.insert($channel.outbound_scid_alias(), ($channel.get_counterparty_node_id(), $channel.channel_id()));
+               let mut short_to_chan_info = $self.short_to_chan_info.write().unwrap();
+               let outbound_alias_insert = short_to_chan_info.insert($channel.outbound_scid_alias(), ($channel.get_counterparty_node_id(), $channel.channel_id()));
                assert!(outbound_alias_insert.is_none() || outbound_alias_insert.unwrap() == ($channel.get_counterparty_node_id(), $channel.channel_id()),
                        "SCIDs should never collide - ensure you weren't behind the chain tip by a full month when creating channels");
                if let Some(real_scid) = $channel.get_short_channel_id() {
-                       let scid_insert = $short_to_chan_info.insert(real_scid, ($channel.get_counterparty_node_id(), $channel.channel_id()));
+                       let scid_insert = short_to_chan_info.insert(real_scid, ($channel.get_counterparty_node_id(), $channel.channel_id()));
                        assert!(scid_insert.is_none() || scid_insert.unwrap() == ($channel.get_counterparty_node_id(), $channel.channel_id()),
                                "SCIDs should never collide - ensure you weren't behind the chain tip by a full month when creating channels");
                }
-       }
+       }}
 }
 
 macro_rules! emit_channel_ready_event {
@@ -1516,7 +1533,7 @@ macro_rules! handle_chan_restoration_locked {
                                // Similar to the above, this implies that we're letting the channel_ready fly
                                // before it should be allowed to.
                                assert!(chanmon_update.is_none());
-                               send_channel_ready!($channel_state.short_to_chan_info, $channel_state.pending_msg_events, $channel_entry.get(), msg);
+                               send_channel_ready!($self, $channel_state.pending_msg_events, $channel_entry.get(), msg);
                        }
                        if let Some(msg) = $announcement_sigs {
                                $channel_state.pending_msg_events.push(events::MessageSendEvent::SendAnnouncementSignatures {
@@ -1549,7 +1566,7 @@ macro_rules! handle_chan_restoration_locked {
                                                if $raa.is_none() {
                                                        order = RAACommitmentOrder::CommitmentFirst;
                                                }
-                                               break handle_monitor_update_res!($self, e, $channel_state, $channel_entry, order, $raa.is_some(), true);
+                                               break handle_monitor_update_res!($self, e, $channel_entry, order, $raa.is_some(), true);
                                        }
                                }
                        }
@@ -1643,7 +1660,6 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
                        channel_state: Mutex::new(ChannelHolder{
                                by_id: HashMap::new(),
-                               short_to_chan_info: HashMap::new(),
                                claimable_htlcs: HashMap::new(),
                                pending_msg_events: Vec::new(),
                        }),
@@ -1652,6 +1668,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                        pending_outbound_payments: Mutex::new(HashMap::new()),
                        forward_htlcs: Mutex::new(HashMap::new()),
                        id_to_peer: Mutex::new(HashMap::new()),
+                       short_to_chan_info: FairRwLock::new(HashMap::new()),
 
                        our_network_key: keys_manager.get_node_secret(Recipient::Node).unwrap(),
                        our_network_pubkey: PublicKey::from_secret_key(&secp_ctx, &keys_manager.get_node_secret(Recipient::Node).unwrap()),
@@ -1903,9 +1920,9 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        if let Some(monitor_update) = monitor_update {
                                                let update_res = self.chain_monitor.update_channel(chan_entry.get().get_funding_txo().unwrap(), monitor_update);
                                                let (result, is_permanent) =
-                                                       handle_monitor_update_res!(self, update_res, channel_state.short_to_chan_info, chan_entry.get_mut(), RAACommitmentOrder::CommitmentFirst, chan_entry.key(), NO_UPDATE);
+                                                       handle_monitor_update_res!(self, update_res, chan_entry.get_mut(), RAACommitmentOrder::CommitmentFirst, chan_entry.key(), NO_UPDATE);
                                                if is_permanent {
-                                                       remove_channel!(self, channel_state, chan_entry);
+                                                       remove_channel!(self, chan_entry);
                                                        break result;
                                                }
                                        }
@@ -1916,7 +1933,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        });
 
                                        if chan_entry.get().is_shutdown() {
-                                               let channel = remove_channel!(self, channel_state, chan_entry);
+                                               let channel = remove_channel!(self, chan_entry);
                                                if let Ok(channel_update) = self.get_channel_update_for_broadcast(&channel) {
                                                        channel_state.pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
                                                                msg: channel_update
@@ -2017,7 +2034,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                } else {
                                        self.issue_channel_close_events(chan.get(),ClosureReason::HolderForceClosed);
                                }
-                               remove_channel!(self, channel_state, chan)
+                               remove_channel!(self, chan)
                        } else {
                                return Err(APIError::ChannelUnavailable{err: "No such channel".to_owned()});
                        }
@@ -2180,6 +2197,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                        routing,
                        payment_hash,
                        incoming_shared_secret: shared_secret,
+                       amt_incoming: Some(amt_msat),
                        amt_to_forward: amt_msat,
                        outgoing_cltv_value: hop_data.outgoing_cltv_value,
                })
@@ -2276,6 +2294,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        },
                                        payment_hash: msg.payment_hash.clone(),
                                        incoming_shared_secret: shared_secret,
+                                       amt_incoming: Some(msg.amount_msat),
                                        amt_to_forward: next_hop_data.amt_to_forward,
                                        outgoing_cltv_value: next_hop_data.outgoing_cltv_value,
                                })
@@ -2288,13 +2307,13 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                        // short_channel_id is non-0 in any ::Forward.
                        if let &PendingHTLCRouting::Forward { ref short_channel_id, .. } = routing {
                                if let Some((err, code, chan_update)) = loop {
+                                       let id_option = self.short_to_chan_info.read().unwrap().get(&short_channel_id).cloned();
                                        let mut channel_state = self.channel_state.lock().unwrap();
-                                       let id_option = channel_state.short_to_chan_info.get(&short_channel_id).cloned();
                                        let forwarding_id_opt = match id_option {
                                                None => { // unknown_next_peer
                                                        // Note that this is likely a timing oracle for detecting whether an scid is a
                                                        // phantom.
-                                                       if fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, *short_channel_id) {
+                                                       if fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, *short_channel_id, &self.genesis_hash) {
                                                                None
                                                        } else {
                                                                break Some(("Don't have available channel for forwarding as requested.", 0x4000 | 10, None));
@@ -2303,7 +2322,14 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                Some((_cp_id, chan_id)) => Some(chan_id.clone()),
                                        };
                                        let chan_update_opt = if let Some(forwarding_id) = forwarding_id_opt {
-                                               let chan = channel_state.by_id.get_mut(&forwarding_id).unwrap();
+                                               let chan = match channel_state.by_id.get_mut(&forwarding_id) {
+                                                       None => {
+                                                               // Channel was removed. The short_to_chan_info and by_id maps have
+                                                               // no consistency guarantees.
+                                                               break Some(("Don't have available channel for forwarding as requested.", 0x4000 | 10, None));
+                                                       },
+                                                       Some(chan) => chan
+                                               };
                                                if !chan.should_announce() && !self.default_configuration.accept_forwards_to_priv_channels {
                                                        // Note that the behavior here should be identical to the above block - we
                                                        // should NOT reveal the existence or non-existence of a private channel if
@@ -2468,13 +2494,12 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
 
                let err: Result<(), _> = loop {
-                       let mut channel_lock = self.channel_state.lock().unwrap();
-
-                       let id = match channel_lock.short_to_chan_info.get(&path.first().unwrap().short_channel_id) {
+                       let id = match self.short_to_chan_info.read().unwrap().get(&path.first().unwrap().short_channel_id) {
                                None => return Err(APIError::ChannelUnavailable{err: "No channel available with first hop!".to_owned()}),
                                Some((_cp_id, chan_id)) => chan_id.clone(),
                        };
 
+                       let mut channel_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_lock;
                        if let hash_map::Entry::Occupied(mut chan) = channel_state.by_id.entry(id) {
                                match {
@@ -2493,13 +2518,13 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                        payment_secret: payment_secret.clone(),
                                                        payment_params: payment_params.clone(),
                                                }, onion_packet, &self.logger),
-                                       channel_state, chan)
+                                               chan)
                                } {
                                        Some((update_add, commitment_signed, monitor_update)) => {
                                                let update_err = self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update);
                                                let chan_id = chan.get().channel_id();
                                                match (update_err,
-                                                       handle_monitor_update_res!(self, update_err, channel_state, chan,
+                                                       handle_monitor_update_res!(self, update_err, chan,
                                                                RAACommitmentOrder::CommitmentFirst, false, true))
                                                {
                                                        (ChannelMonitorUpdateStatus::PermanentFailure, Err(e)) => break Err(e),
@@ -2531,7 +2556,12 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        },
                                        None => { },
                                }
-                       } else { unreachable!(); }
+                       } else {
+                               // The channel was likely removed after we fetched the id from the
+                               // `short_to_chan_info` map, but before we successfully locked the `by_id` map.
+                               // This can occur as no consistency guarantees exists between the two maps.
+                               return Err(APIError::ChannelUnavailable{err: "No channel available with first hop!".to_owned()});
+                       }
                        return Ok(());
                };
 
@@ -3120,87 +3150,90 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                let mut channel_state_lock = self.channel_state.lock().unwrap();
                                let channel_state = &mut *channel_state_lock;
                                if short_chan_id != 0 {
-                                       let forward_chan_id = match channel_state.short_to_chan_info.get(&short_chan_id) {
-                                               Some((_cp_id, chan_id)) => chan_id.clone(),
-                                               None => {
+                                       macro_rules! forwarding_channel_not_found {
+                                               () => {
                                                        for forward_info in pending_forwards.drain(..) {
                                                                match forward_info {
-                                                                       HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo {
-                                                                               routing, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value },
-                                                                               prev_funding_outpoint } => {
-                                                                                       macro_rules! failure_handler {
-                                                                                               ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr, $next_hop_unknown: expr) => {
-                                                                                                       log_info!(self.logger, "Failed to accept/forward incoming HTLC: {}", $msg);
-
-                                                                                                       let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
-                                                                                                               short_channel_id: prev_short_channel_id,
-                                                                                                               outpoint: prev_funding_outpoint,
-                                                                                                               htlc_id: prev_htlc_id,
-                                                                                                               incoming_packet_shared_secret: incoming_shared_secret,
-                                                                                                               phantom_shared_secret: $phantom_ss,
-                                                                                                       });
-
-                                                                                                       let reason = if $next_hop_unknown {
-                                                                                                               HTLCDestination::UnknownNextHop { requested_forward_scid: short_chan_id }
-                                                                                                       } else {
-                                                                                                               HTLCDestination::FailedPayment{ payment_hash }
-                                                                                                       };
-
-                                                                                                       failed_forwards.push((htlc_source, payment_hash,
-                                                                                                               HTLCFailReason::Reason { failure_code: $err_code, data: $err_data },
-                                                                                                               reason
-                                                                                                       ));
-                                                                                                       continue;
-                                                                                               }
+                                                                       HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
+                                                                               prev_short_channel_id, prev_htlc_id, prev_funding_outpoint,
+                                                                               forward_info: PendingHTLCInfo {
+                                                                                       routing, incoming_shared_secret, payment_hash, amt_to_forward,
+                                                                                       outgoing_cltv_value, amt_incoming: _
+                                                                               }
+                                                                       }) => {
+                                                                               macro_rules! failure_handler {
+                                                                                       ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr, $next_hop_unknown: expr) => {
+                                                                                               log_info!(self.logger, "Failed to accept/forward incoming HTLC: {}", $msg);
+
+                                                                                               let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
+                                                                                                       short_channel_id: prev_short_channel_id,
+                                                                                                       outpoint: prev_funding_outpoint,
+                                                                                                       htlc_id: prev_htlc_id,
+                                                                                                       incoming_packet_shared_secret: incoming_shared_secret,
+                                                                                                       phantom_shared_secret: $phantom_ss,
+                                                                                               });
+
+                                                                                               let reason = if $next_hop_unknown {
+                                                                                                       HTLCDestination::UnknownNextHop { requested_forward_scid: short_chan_id }
+                                                                                               } else {
+                                                                                                       HTLCDestination::FailedPayment{ payment_hash }
+                                                                                               };
+
+                                                                                               failed_forwards.push((htlc_source, payment_hash,
+                                                                                                       HTLCFailReason::Reason { failure_code: $err_code, data: $err_data },
+                                                                                                       reason
+                                                                                               ));
+                                                                                               continue;
                                                                                        }
-                                                                                       macro_rules! fail_forward {
-                                                                                               ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
-                                                                                                       {
-                                                                                                               failure_handler!($msg, $err_code, $err_data, $phantom_ss, true);
-                                                                                                       }
+                                                                               }
+                                                                               macro_rules! fail_forward {
+                                                                                       ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
+                                                                                               {
+                                                                                                       failure_handler!($msg, $err_code, $err_data, $phantom_ss, true);
                                                                                                }
                                                                                        }
-                                                                                       macro_rules! failed_payment {
-                                                                                               ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
-                                                                                                       {
-                                                                                                               failure_handler!($msg, $err_code, $err_data, $phantom_ss, false);
-                                                                                                       }
+                                                                               }
+                                                                               macro_rules! failed_payment {
+                                                                                       ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
+                                                                                               {
+                                                                                                       failure_handler!($msg, $err_code, $err_data, $phantom_ss, false);
                                                                                                }
                                                                                        }
-                                                                                       if let PendingHTLCRouting::Forward { onion_packet, .. } = routing {
-                                                                                               let phantom_secret_res = self.keys_manager.get_node_secret(Recipient::PhantomNode);
-                                                                                               if phantom_secret_res.is_ok() && fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, short_chan_id) {
-                                                                                                       let phantom_shared_secret = SharedSecret::new(&onion_packet.public_key.unwrap(), &phantom_secret_res.unwrap()).secret_bytes();
-                                                                                                       let next_hop = match onion_utils::decode_next_payment_hop(phantom_shared_secret, &onion_packet.hop_data, onion_packet.hmac, payment_hash) {
-                                                                                                               Ok(res) => res,
-                                                                                                               Err(onion_utils::OnionDecodeErr::Malformed { err_msg, err_code }) => {
-                                                                                                                       let sha256_of_onion = Sha256::hash(&onion_packet.hop_data).into_inner();
-                                                                                                                       // In this scenario, the phantom would have sent us an
-                                                                                                                       // `update_fail_malformed_htlc`, meaning here we encrypt the error as
-                                                                                                                       // if it came from us (the second-to-last hop) but contains the sha256
-                                                                                                                       // of the onion.
-                                                                                                                       failed_payment!(err_msg, err_code, sha256_of_onion.to_vec(), None);
-                                                                                                               },
-                                                                                                               Err(onion_utils::OnionDecodeErr::Relay { err_msg, err_code }) => {
-                                                                                                                       failed_payment!(err_msg, err_code, Vec::new(), Some(phantom_shared_secret));
-                                                                                                               },
-                                                                                                       };
-                                                                                                       match next_hop {
-                                                                                                               onion_utils::Hop::Receive(hop_data) => {
-                                                                                                                       match self.construct_recv_pending_htlc_info(hop_data, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value, Some(phantom_shared_secret)) {
-                                                                                                                               Ok(info) => phantom_receives.push((prev_short_channel_id, prev_funding_outpoint, vec![(info, prev_htlc_id)])),
-                                                                                                                               Err(ReceiveError { err_code, err_data, msg }) => failed_payment!(msg, err_code, err_data, Some(phantom_shared_secret))
-                                                                                                                       }
-                                                                                                               },
-                                                                                                               _ => panic!(),
-                                                                                                       }
-                                                                                               } else {
-                                                                                                       fail_forward!(format!("Unknown short channel id {} for forward HTLC", short_chan_id), 0x4000 | 10, Vec::new(), None);
+                                                                               }
+                                                                               if let PendingHTLCRouting::Forward { onion_packet, .. } = routing {
+                                                                                       let phantom_secret_res = self.keys_manager.get_node_secret(Recipient::PhantomNode);
+                                                                                       if phantom_secret_res.is_ok() && fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, short_chan_id, &self.genesis_hash) {
+                                                                                               let phantom_shared_secret = SharedSecret::new(&onion_packet.public_key.unwrap(), &phantom_secret_res.unwrap()).secret_bytes();
+                                                                                               let next_hop = match onion_utils::decode_next_payment_hop(phantom_shared_secret, &onion_packet.hop_data, onion_packet.hmac, payment_hash) {
+                                                                                                       Ok(res) => res,
+                                                                                                       Err(onion_utils::OnionDecodeErr::Malformed { err_msg, err_code }) => {
+                                                                                                               let sha256_of_onion = Sha256::hash(&onion_packet.hop_data).into_inner();
+                                                                                                               // In this scenario, the phantom would have sent us an
+                                                                                                               // `update_fail_malformed_htlc`, meaning here we encrypt the error as
+                                                                                                               // if it came from us (the second-to-last hop) but contains the sha256
+                                                                                                               // of the onion.
+                                                                                                               failed_payment!(err_msg, err_code, sha256_of_onion.to_vec(), None);
+                                                                                                       },
+                                                                                                       Err(onion_utils::OnionDecodeErr::Relay { err_msg, err_code }) => {
+                                                                                                               failed_payment!(err_msg, err_code, Vec::new(), Some(phantom_shared_secret));
+                                                                                                       },
+                                                                                               };
+                                                                                               match next_hop {
+                                                                                                       onion_utils::Hop::Receive(hop_data) => {
+                                                                                                               match self.construct_recv_pending_htlc_info(hop_data, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value, Some(phantom_shared_secret)) {
+                                                                                                                       Ok(info) => phantom_receives.push((prev_short_channel_id, prev_funding_outpoint, vec![(info, prev_htlc_id)])),
+                                                                                                                       Err(ReceiveError { err_code, err_data, msg }) => failed_payment!(msg, err_code, err_data, Some(phantom_shared_secret))
+                                                                                                               }
+                                                                                                       },
+                                                                                                       _ => panic!(),
                                                                                                }
                                                                                        } else {
                                                                                                fail_forward!(format!("Unknown short channel id {} for forward HTLC", short_chan_id), 0x4000 | 10, Vec::new(), None);
                                                                                        }
-                                                                               },
+                                                                               } else {
+                                                                                       fail_forward!(format!("Unknown short channel id {} for forward HTLC", short_chan_id), 0x4000 | 10, Vec::new(), None);
+                                                                               }
+                                                                       },
                                                                        HTLCForwardInfo::FailHTLC { .. } => {
                                                                                // Channel went away before we could fail it. This implies
                                                                                // the channel is now on chain and our counterparty is
@@ -3209,143 +3242,158 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                        }
                                                                }
                                                        }
+                                               }
+                                       }
+                                       let forward_chan_id = match self.short_to_chan_info.read().unwrap().get(&short_chan_id) {
+                                               Some((_cp_id, chan_id)) => chan_id.clone(),
+                                               None => {
+                                                       forwarding_channel_not_found!();
                                                        continue;
                                                }
                                        };
-                                       if let hash_map::Entry::Occupied(mut chan) = channel_state.by_id.entry(forward_chan_id) {
-                                               let mut add_htlc_msgs = Vec::new();
-                                               let mut fail_htlc_msgs = Vec::new();
-                                               for forward_info in pending_forwards.drain(..) {
-                                                       match forward_info {
-                                                               HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo {
-                                                                               routing: PendingHTLCRouting::Forward {
-                                                                                       onion_packet, ..
-                                                                               }, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value },
-                                                                               prev_funding_outpoint } => {
-                                                                       log_trace!(self.logger, "Adding HTLC from short id {} with payment_hash {} to channel with short id {} after delay", prev_short_channel_id, log_bytes!(payment_hash.0), short_chan_id);
-                                                                       let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
-                                                                               short_channel_id: prev_short_channel_id,
-                                                                               outpoint: prev_funding_outpoint,
-                                                                               htlc_id: prev_htlc_id,
-                                                                               incoming_packet_shared_secret: incoming_shared_secret,
-                                                                               // Phantom payments are only PendingHTLCRouting::Receive.
-                                                                               phantom_shared_secret: None,
-                                                                       });
-                                                                       match chan.get_mut().send_htlc(amt_to_forward, payment_hash, outgoing_cltv_value, htlc_source.clone(), onion_packet, &self.logger) {
-                                                                               Err(e) => {
-                                                                                       if let ChannelError::Ignore(msg) = e {
-                                                                                               log_trace!(self.logger, "Failed to forward HTLC with payment_hash {}: {}", log_bytes!(payment_hash.0), msg);
-                                                                                       } else {
-                                                                                               panic!("Stated return value requirements in send_htlc() were not met");
-                                                                                       }
-                                                                                       let (failure_code, data) = self.get_htlc_temp_fail_err_and_data(0x1000|7, short_chan_id, chan.get());
-                                                                                       failed_forwards.push((htlc_source, payment_hash,
-                                                                                               HTLCFailReason::Reason { failure_code, data },
-                                                                                               HTLCDestination::NextHopChannel { node_id: Some(chan.get().get_counterparty_node_id()), channel_id: forward_chan_id }
-                                                                                       ));
-                                                                                       continue;
+                                       match channel_state.by_id.entry(forward_chan_id) {
+                                               hash_map::Entry::Vacant(_) => {
+                                                       forwarding_channel_not_found!();
+                                                       continue;
+                                               },
+                                               hash_map::Entry::Occupied(mut chan) => {
+                                                       let mut add_htlc_msgs = Vec::new();
+                                                       let mut fail_htlc_msgs = Vec::new();
+                                                       for forward_info in pending_forwards.drain(..) {
+                                                               match forward_info {
+                                                                       HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
+                                                                               prev_short_channel_id, prev_htlc_id, prev_funding_outpoint ,
+                                                                               forward_info: PendingHTLCInfo {
+                                                                                       incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value,
+                                                                                       routing: PendingHTLCRouting::Forward { onion_packet, .. }, amt_incoming: _,
                                                                                },
-                                                                               Ok(update_add) => {
-                                                                                       match update_add {
-                                                                                               Some(msg) => { add_htlc_msgs.push(msg); },
-                                                                                               None => {
-                                                                                                       // Nothing to do here...we're waiting on a remote
-                                                                                                       // revoke_and_ack before we can add anymore HTLCs. The Channel
-                                                                                                       // will automatically handle building the update_add_htlc and
-                                                                                                       // commitment_signed messages when we can.
-                                                                                                       // TODO: Do some kind of timer to set the channel as !is_live()
-                                                                                                       // as we don't really want others relying on us relaying through
-                                                                                                       // this channel currently :/.
+                                                                       }) => {
+                                                                               log_trace!(self.logger, "Adding HTLC from short id {} with payment_hash {} to channel with short id {} after delay", prev_short_channel_id, log_bytes!(payment_hash.0), short_chan_id);
+                                                                               let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
+                                                                                       short_channel_id: prev_short_channel_id,
+                                                                                       outpoint: prev_funding_outpoint,
+                                                                                       htlc_id: prev_htlc_id,
+                                                                                       incoming_packet_shared_secret: incoming_shared_secret,
+                                                                                       // Phantom payments are only PendingHTLCRouting::Receive.
+                                                                                       phantom_shared_secret: None,
+                                                                               });
+                                                                               match chan.get_mut().send_htlc(amt_to_forward, payment_hash, outgoing_cltv_value, htlc_source.clone(), onion_packet, &self.logger) {
+                                                                                       Err(e) => {
+                                                                                               if let ChannelError::Ignore(msg) = e {
+                                                                                                       log_trace!(self.logger, "Failed to forward HTLC with payment_hash {}: {}", log_bytes!(payment_hash.0), msg);
+                                                                                               } else {
+                                                                                                       panic!("Stated return value requirements in send_htlc() were not met");
+                                                                                               }
+                                                                                               let (failure_code, data) = self.get_htlc_temp_fail_err_and_data(0x1000|7, short_chan_id, chan.get());
+                                                                                               failed_forwards.push((htlc_source, payment_hash,
+                                                                                                       HTLCFailReason::Reason { failure_code, data },
+                                                                                                       HTLCDestination::NextHopChannel { node_id: Some(chan.get().get_counterparty_node_id()), channel_id: forward_chan_id }
+                                                                                               ));
+                                                                                               continue;
+                                                                                       },
+                                                                                       Ok(update_add) => {
+                                                                                               match update_add {
+                                                                                                       Some(msg) => { add_htlc_msgs.push(msg); },
+                                                                                                       None => {
+                                                                                                               // Nothing to do here...we're waiting on a remote
+                                                                                                               // revoke_and_ack before we can add anymore HTLCs. The Channel
+                                                                                                               // will automatically handle building the update_add_htlc and
+                                                                                                               // commitment_signed messages when we can.
+                                                                                                               // TODO: Do some kind of timer to set the channel as !is_live()
+                                                                                                               // as we don't really want others relying on us relaying through
+                                                                                                               // this channel currently :/.
+                                                                                                       }
                                                                                                }
                                                                                        }
                                                                                }
-                                                                       }
-                                                               },
-                                                               HTLCForwardInfo::AddHTLC { .. } => {
-                                                                       panic!("short_channel_id != 0 should imply any pending_forward entries are of type Forward");
-                                                               },
-                                                               HTLCForwardInfo::FailHTLC { htlc_id, err_packet } => {
-                                                                       log_trace!(self.logger, "Failing HTLC back to channel with short id {} (backward HTLC ID {}) after delay", short_chan_id, htlc_id);
-                                                                       match chan.get_mut().get_update_fail_htlc(htlc_id, err_packet, &self.logger) {
-                                                                               Err(e) => {
-                                                                                       if let ChannelError::Ignore(msg) = e {
-                                                                                               log_trace!(self.logger, "Failed to fail HTLC with ID {} backwards to short_id {}: {}", htlc_id, short_chan_id, msg);
-                                                                                       } else {
-                                                                                               panic!("Stated return value requirements in get_update_fail_htlc() were not met");
+                                                                       },
+                                                                       HTLCForwardInfo::AddHTLC { .. } => {
+                                                                               panic!("short_channel_id != 0 should imply any pending_forward entries are of type Forward");
+                                                                       },
+                                                                       HTLCForwardInfo::FailHTLC { htlc_id, err_packet } => {
+                                                                               log_trace!(self.logger, "Failing HTLC back to channel with short id {} (backward HTLC ID {}) after delay", short_chan_id, htlc_id);
+                                                                               match chan.get_mut().get_update_fail_htlc(htlc_id, err_packet, &self.logger) {
+                                                                                       Err(e) => {
+                                                                                               if let ChannelError::Ignore(msg) = e {
+                                                                                                       log_trace!(self.logger, "Failed to fail HTLC with ID {} backwards to short_id {}: {}", htlc_id, short_chan_id, msg);
+                                                                                               } else {
+                                                                                                       panic!("Stated return value requirements in get_update_fail_htlc() were not met");
+                                                                                               }
+                                                                                               // fail-backs are best-effort, we probably already have one
+                                                                                               // pending, and if not that's OK, if not, the channel is on
+                                                                                               // the chain and sending the HTLC-Timeout is their problem.
+                                                                                               continue;
+                                                                                       },
+                                                                                       Ok(Some(msg)) => { fail_htlc_msgs.push(msg); },
+                                                                                       Ok(None) => {
+                                                                                               // Nothing to do here...we're waiting on a remote
+                                                                                               // revoke_and_ack before we can update the commitment
+                                                                                               // transaction. The Channel will automatically handle
+                                                                                               // building the update_fail_htlc and commitment_signed
+                                                                                               // messages when we can.
+                                                                                               // We don't need any kind of timer here as they should fail
+                                                                                               // the channel onto the chain if they can't get our
+                                                                                               // update_fail_htlc in time, it's not our problem.
                                                                                        }
-                                                                                       // fail-backs are best-effort, we probably already have one
-                                                                                       // pending, and if not that's OK, if not, the channel is on
-                                                                                       // the chain and sending the HTLC-Timeout is their problem.
-                                                                                       continue;
-                                                                               },
-                                                                               Ok(Some(msg)) => { fail_htlc_msgs.push(msg); },
-                                                                               Ok(None) => {
-                                                                                       // Nothing to do here...we're waiting on a remote
-                                                                                       // revoke_and_ack before we can update the commitment
-                                                                                       // transaction. The Channel will automatically handle
-                                                                                       // building the update_fail_htlc and commitment_signed
-                                                                                       // messages when we can.
-                                                                                       // We don't need any kind of timer here as they should fail
-                                                                                       // the channel onto the chain if they can't get our
-                                                                                       // update_fail_htlc in time, it's not our problem.
                                                                                }
-                                                                       }
-                                                               },
+                                                                       },
+                                                               }
                                                        }
-                                               }
 
-                                               if !add_htlc_msgs.is_empty() || !fail_htlc_msgs.is_empty() {
-                                                       let (commitment_msg, monitor_update) = match chan.get_mut().send_commitment(&self.logger) {
-                                                               Ok(res) => res,
-                                                               Err(e) => {
-                                                                       // We surely failed send_commitment due to bad keys, in that case
-                                                                       // close channel and then send error message to peer.
-                                                                       let counterparty_node_id = chan.get().get_counterparty_node_id();
-                                                                       let err: Result<(), _>  = match e {
-                                                                               ChannelError::Ignore(_) | ChannelError::Warn(_) => {
-                                                                                       panic!("Stated return value requirements in send_commitment() were not met");
-                                                                               }
-                                                                               ChannelError::Close(msg) => {
-                                                                                       log_trace!(self.logger, "Closing channel {} due to Close-required error: {}", log_bytes!(chan.key()[..]), msg);
-                                                                                       let mut channel = remove_channel!(self, channel_state, chan);
-                                                                                       // ChannelClosed event is generated by handle_error for us.
-                                                                                       Err(MsgHandleErrInternal::from_finish_shutdown(msg, channel.channel_id(), channel.get_user_id(), channel.force_shutdown(true), self.get_channel_update_for_broadcast(&channel).ok()))
-                                                                               },
-                                                                       };
-                                                                       handle_errors.push((counterparty_node_id, err));
-                                                                       continue;
-                                                               }
-                                                       };
-                                                       match self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update) {
-                                                               ChannelMonitorUpdateStatus::Completed => {},
-                                                               e => {
-                                                                       handle_errors.push((chan.get().get_counterparty_node_id(), handle_monitor_update_res!(self, e, channel_state, chan, RAACommitmentOrder::CommitmentFirst, false, true)));
-                                                                       continue;
+                                                       if !add_htlc_msgs.is_empty() || !fail_htlc_msgs.is_empty() {
+                                                               let (commitment_msg, monitor_update) = match chan.get_mut().send_commitment(&self.logger) {
+                                                                       Ok(res) => res,
+                                                                       Err(e) => {
+                                                                               // We surely failed send_commitment due to bad keys, in that case
+                                                                               // close channel and then send error message to peer.
+                                                                               let counterparty_node_id = chan.get().get_counterparty_node_id();
+                                                                               let err: Result<(), _>  = match e {
+                                                                                       ChannelError::Ignore(_) | ChannelError::Warn(_) => {
+                                                                                               panic!("Stated return value requirements in send_commitment() were not met");
+                                                                                       }
+                                                                                       ChannelError::Close(msg) => {
+                                                                                               log_trace!(self.logger, "Closing channel {} due to Close-required error: {}", log_bytes!(chan.key()[..]), msg);
+                                                                                               let mut channel = remove_channel!(self, chan);
+                                                                                               // ChannelClosed event is generated by handle_error for us.
+                                                                                               Err(MsgHandleErrInternal::from_finish_shutdown(msg, channel.channel_id(), channel.get_user_id(), channel.force_shutdown(true), self.get_channel_update_for_broadcast(&channel).ok()))
+                                                                                       },
+                                                                               };
+                                                                               handle_errors.push((counterparty_node_id, err));
+                                                                               continue;
+                                                                       }
+                                                               };
+                                                               match self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update) {
+                                                                       ChannelMonitorUpdateStatus::Completed => {},
+                                                                       e => {
+                                                                               handle_errors.push((chan.get().get_counterparty_node_id(), handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, false, true)));
+                                                                               continue;
+                                                                       }
                                                                }
+                                                               log_debug!(self.logger, "Forwarding HTLCs resulted in a commitment update with {} HTLCs added and {} HTLCs failed for channel {}",
+                                                                       add_htlc_msgs.len(), fail_htlc_msgs.len(), log_bytes!(chan.get().channel_id()));
+                                                               channel_state.pending_msg_events.push(events::MessageSendEvent::UpdateHTLCs {
+                                                                       node_id: chan.get().get_counterparty_node_id(),
+                                                                       updates: msgs::CommitmentUpdate {
+                                                                               update_add_htlcs: add_htlc_msgs,
+                                                                               update_fulfill_htlcs: Vec::new(),
+                                                                               update_fail_htlcs: fail_htlc_msgs,
+                                                                               update_fail_malformed_htlcs: Vec::new(),
+                                                                               update_fee: None,
+                                                                               commitment_signed: commitment_msg,
+                                                                       },
+                                                               });
                                                        }
-                                                       log_debug!(self.logger, "Forwarding HTLCs resulted in a commitment update with {} HTLCs added and {} HTLCs failed for channel {}",
-                                                               add_htlc_msgs.len(), fail_htlc_msgs.len(), log_bytes!(chan.get().channel_id()));
-                                                       channel_state.pending_msg_events.push(events::MessageSendEvent::UpdateHTLCs {
-                                                               node_id: chan.get().get_counterparty_node_id(),
-                                                               updates: msgs::CommitmentUpdate {
-                                                                       update_add_htlcs: add_htlc_msgs,
-                                                                       update_fulfill_htlcs: Vec::new(),
-                                                                       update_fail_htlcs: fail_htlc_msgs,
-                                                                       update_fail_malformed_htlcs: Vec::new(),
-                                                                       update_fee: None,
-                                                                       commitment_signed: commitment_msg,
-                                                               },
-                                                       });
                                                }
-                                       } else {
-                                               unreachable!();
                                        }
                                } else {
                                        for forward_info in pending_forwards.drain(..) {
                                                match forward_info {
-                                                       HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo {
-                                                                       routing, incoming_shared_secret, payment_hash, amt_to_forward, .. },
-                                                                       prev_funding_outpoint } => {
+                                                       HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
+                                                               prev_short_channel_id, prev_htlc_id, prev_funding_outpoint,
+                                                               forward_info: PendingHTLCInfo {
+                                                                       routing, incoming_shared_secret, payment_hash, amt_to_forward, ..
+                                                               }
+                                                       }) => {
                                                                let (cltv_expiry, onion_payload, payment_data, phantom_shared_secret) = match routing {
                                                                        PendingHTLCRouting::Receive { payment_data, incoming_cltv_expiry, phantom_shared_secret } => {
                                                                                let _legacy_hop_data = Some(payment_data.clone());
@@ -3563,7 +3611,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                self.process_background_events();
        }
 
-       fn update_channel_fee(&self, short_to_chan_info: &mut HashMap<u64, (PublicKey, [u8; 32])>, pending_msg_events: &mut Vec<events::MessageSendEvent>, chan_id: &[u8; 32], chan: &mut Channel<<K::Target as KeysInterface>::Signer>, new_feerate: u32) -> (bool, NotifyOption, Result<(), MsgHandleErrInternal>) {
+       fn update_channel_fee(&self, pending_msg_events: &mut Vec<events::MessageSendEvent>, chan_id: &[u8; 32], chan: &mut Channel<<K::Target as KeysInterface>::Signer>, new_feerate: u32) -> (bool, NotifyOption, Result<(), MsgHandleErrInternal>) {
                if !chan.is_outbound() { return (true, NotifyOption::SkipPersist, Ok(())); }
                // If the feerate has decreased by less than half, don't bother
                if new_feerate <= chan.get_feerate() && new_feerate * 2 > chan.get_feerate() {
@@ -3583,7 +3631,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                let res = match chan.send_update_fee_and_commit(new_feerate, &self.logger) {
                        Ok(res) => Ok(res),
                        Err(e) => {
-                               let (drop, res) = convert_chan_err!(self, e, short_to_chan_info, chan, chan_id);
+                               let (drop, res) = convert_chan_err!(self, e, chan, chan_id);
                                if drop { retain_channel = false; }
                                Err(res)
                        }
@@ -3606,7 +3654,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                Ok(())
                                        },
                                        e => {
-                                               let (res, drop) = handle_monitor_update_res!(self, e, short_to_chan_info, chan, RAACommitmentOrder::CommitmentFirst, chan_id, COMMITMENT_UPDATE_ONLY);
+                                               let (res, drop) = handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, chan_id, COMMITMENT_UPDATE_ONLY);
                                                if drop { retain_channel = false; }
                                                res
                                        }
@@ -3634,9 +3682,8 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                let mut channel_state_lock = self.channel_state.lock().unwrap();
                                let channel_state = &mut *channel_state_lock;
                                let pending_msg_events = &mut channel_state.pending_msg_events;
-                               let short_to_chan_info = &mut channel_state.short_to_chan_info;
                                channel_state.by_id.retain(|chan_id, chan| {
-                                       let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(short_to_chan_info, pending_msg_events, chan_id, chan, new_feerate);
+                                       let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(pending_msg_events, chan_id, chan, new_feerate);
                                        if chan_needs_persist == NotifyOption::DoPersist { should_persist = NotifyOption::DoPersist; }
                                        if err.is_err() {
                                                handle_errors.push(err);
@@ -3713,10 +3760,9 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                let mut channel_state_lock = self.channel_state.lock().unwrap();
                                let channel_state = &mut *channel_state_lock;
                                let pending_msg_events = &mut channel_state.pending_msg_events;
-                               let short_to_chan_info = &mut channel_state.short_to_chan_info;
                                channel_state.by_id.retain(|chan_id, chan| {
                                        let counterparty_node_id = chan.get_counterparty_node_id();
-                                       let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(short_to_chan_info, pending_msg_events, chan_id, chan, new_feerate);
+                                       let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(pending_msg_events, chan_id, chan, new_feerate);
                                        if chan_needs_persist == NotifyOption::DoPersist { should_persist = NotifyOption::DoPersist; }
                                        if err.is_err() {
                                                handle_errors.push((err, counterparty_node_id));
@@ -3724,7 +3770,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        if !retain_channel { return false; }
 
                                        if let Err(e) = chan.timer_check_closing_negotiation_progress() {
-                                               let (needs_close, err) = convert_chan_err!(self, e, short_to_chan_info, chan, chan_id);
+                                               let (needs_close, err) = convert_chan_err!(self, e, chan, chan_id);
                                                handle_errors.push((Err(err), chan.get_counterparty_node_id()));
                                                if needs_close { return false; }
                                        }
@@ -4142,10 +4188,19 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        for htlc in sources.iter() {
-                               if let None = channel_state.short_to_chan_info.get(&htlc.prev_hop.short_channel_id) {
+                               let chan_id = match self.short_to_chan_info.read().unwrap().get(&htlc.prev_hop.short_channel_id) {
+                                       Some((_cp_id, chan_id)) => chan_id.clone(),
+                                       None => {
+                                               valid_mpp = false;
+                                               break;
+                                       }
+                               };
+
+                               if let None = channel_state.by_id.get(&chan_id) {
                                        valid_mpp = false;
                                        break;
                                }
+
                                if expected_amt_msat.is_some() && expected_amt_msat != Some(htlc.total_msat) {
                                        log_error!(self.logger, "Somehow ended up with an MPP payment with different total amounts - this should not be reachable!");
                                        debug_assert!(false);
@@ -4229,7 +4284,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        fn claim_funds_from_hop(&self, channel_state_lock: &mut MutexGuard<ChannelHolder<<K::Target as KeysInterface>::Signer>>, prev_hop: HTLCPreviousHopData, payment_preimage: PaymentPreimage) -> ClaimFundsFromHop {
                //TODO: Delay the claimed_funds relaying just like we do outbound relay!
                let channel_state = &mut **channel_state_lock;
-               let chan_id = match channel_state.short_to_chan_info.get(&prev_hop.short_channel_id) {
+               let chan_id = match self.short_to_chan_info.read().unwrap().get(&prev_hop.short_channel_id) {
                        Some((_cp_id, chan_id)) => chan_id.clone(),
                        None => {
                                return ClaimFundsFromHop::PrevHopForceClosed
@@ -4248,7 +4303,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                        payment_preimage, e);
                                                                return ClaimFundsFromHop::MonitorUpdateFail(
                                                                        chan.get().get_counterparty_node_id(),
-                                                                       handle_monitor_update_res!(self, e, channel_state, chan, RAACommitmentOrder::CommitmentFirst, false, msgs.is_some()).unwrap_err(),
+                                                                       handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, false, msgs.is_some()).unwrap_err(),
                                                                        Some(htlc_value_msat)
                                                                );
                                                        }
@@ -4283,14 +4338,14 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                },
                                        }
                                        let counterparty_node_id = chan.get().get_counterparty_node_id();
-                                       let (drop, res) = convert_chan_err!(self, e, channel_state.short_to_chan_info, chan.get_mut(), &chan_id);
+                                       let (drop, res) = convert_chan_err!(self, e, chan.get_mut(), &chan_id);
                                        if drop {
                                                chan.remove_entry();
                                        }
                                        return ClaimFundsFromHop::MonitorUpdateFail(counterparty_node_id, res, None);
                                },
                        }
-               } else { unreachable!(); }
+               } else { return ClaimFundsFromHop::PrevHopForceClosed }
        }
 
        fn finalize_claims(&self, mut sources: Vec<HTLCSource>) {
@@ -4538,7 +4593,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                }
                                        };
                                        channel_state.pending_msg_events.push(send_msg_err_event);
-                                       let _ = remove_channel!(self, channel_state, channel);
+                                       let _ = remove_channel!(self, channel);
                                        return Err(APIError::APIMisuseError { err: "Please use accept_inbound_channel_from_trusted_peer_0conf to accept channels with zero confirmations.".to_owned() });
                                }
 
@@ -4618,7 +4673,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.temporary_channel_id));
                                        }
-                                       try_chan_entry!(self, chan.get_mut().accept_channel(&msg, &self.default_configuration.channel_handshake_limits, &their_features), channel_state, chan);
+                                       try_chan_entry!(self, chan.get_mut().accept_channel(&msg, &self.default_configuration.channel_handshake_limits, &their_features), chan);
                                        (chan.get().get_value_satoshis(), chan.get().get_funding_redeemscript().to_v0_p2wsh(), chan.get().get_user_id())
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.temporary_channel_id))
@@ -4645,7 +4700,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.temporary_channel_id));
                                        }
-                                       (try_chan_entry!(self, chan.get_mut().funding_created(msg, best_block, &self.logger), channel_state, chan), chan.remove())
+                                       (try_chan_entry!(self, chan.get_mut().funding_created(msg, best_block, &self.logger), chan), chan.remove())
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.temporary_channel_id))
                        }
@@ -4698,7 +4753,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        msg: funding_msg,
                                });
                                if let Some(msg) = channel_ready {
-                                       send_channel_ready!(channel_state.short_to_chan_info, channel_state.pending_msg_events, chan, msg);
+                                       send_channel_ready!(self, channel_state.pending_msg_events, chan, msg);
                                }
                                e.insert(chan);
                        }
@@ -4718,12 +4773,12 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        }
                                        let (monitor, funding_tx, channel_ready) = match chan.get_mut().funding_signed(&msg, best_block, &self.logger) {
                                                Ok(update) => update,
-                                               Err(e) => try_chan_entry!(self, Err(e), channel_state, chan),
+                                               Err(e) => try_chan_entry!(self, Err(e), chan),
                                        };
                                        match self.chain_monitor.watch_channel(chan.get().get_funding_txo().unwrap(), monitor) {
                                                ChannelMonitorUpdateStatus::Completed => {},
                                                e => {
-                                                       let mut res = handle_monitor_update_res!(self, e, channel_state, chan, RAACommitmentOrder::RevokeAndACKFirst, channel_ready.is_some(), OPTIONALLY_RESEND_FUNDING_LOCKED);
+                                                       let mut res = handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::RevokeAndACKFirst, channel_ready.is_some(), OPTIONALLY_RESEND_FUNDING_LOCKED);
                                                        if let Err(MsgHandleErrInternal { ref mut shutdown_finish, .. }) = res {
                                                                // We weren't able to watch the channel to begin with, so no updates should be made on
                                                                // it. Previously, full_stack_target found an (unreachable) panic when the
@@ -4736,7 +4791,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                },
                                        }
                                        if let Some(msg) = channel_ready {
-                                               send_channel_ready!(channel_state.short_to_chan_info, channel_state.pending_msg_events, chan.get(), msg);
+                                               send_channel_ready!(self, channel_state.pending_msg_events, chan.get(), msg);
                                        }
                                        funding_tx
                                },
@@ -4757,7 +4812,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
                                let announcement_sigs_opt = try_chan_entry!(self, chan.get_mut().channel_ready(&msg, self.get_our_node_id(),
-                                       self.genesis_hash.clone(), &self.best_block.read().unwrap(), &self.logger), channel_state, chan);
+                                       self.genesis_hash.clone(), &self.best_block.read().unwrap(), &self.logger), chan);
                                if let Some(announcement_sigs) = announcement_sigs_opt {
                                        log_trace!(self.logger, "Sending announcement_signatures for channel {}", log_bytes!(chan.get().channel_id()));
                                        channel_state.pending_msg_events.push(events::MessageSendEvent::SendAnnouncementSignatures {
@@ -4805,16 +4860,16 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                        if chan_entry.get().sent_shutdown() { " after we initiated shutdown" } else { "" });
                                        }
 
-                                       let (shutdown, monitor_update, htlcs) = try_chan_entry!(self, chan_entry.get_mut().shutdown(&self.keys_manager, &their_features, &msg), channel_state, chan_entry);
+                                       let (shutdown, monitor_update, htlcs) = try_chan_entry!(self, chan_entry.get_mut().shutdown(&self.keys_manager, &their_features, &msg), chan_entry);
                                        dropped_htlcs = htlcs;
 
                                        // Update the monitor with the shutdown script if necessary.
                                        if let Some(monitor_update) = monitor_update {
                                                let update_res = self.chain_monitor.update_channel(chan_entry.get().get_funding_txo().unwrap(), monitor_update);
                                                let (result, is_permanent) =
-                                                       handle_monitor_update_res!(self, update_res, channel_state.short_to_chan_info, chan_entry.get_mut(), RAACommitmentOrder::CommitmentFirst, chan_entry.key(), NO_UPDATE);
+                                                       handle_monitor_update_res!(self, update_res, chan_entry.get_mut(), RAACommitmentOrder::CommitmentFirst, chan_entry.key(), NO_UPDATE);
                                                if is_permanent {
-                                                       remove_channel!(self, channel_state, chan_entry);
+                                                       remove_channel!(self, chan_entry);
                                                        break result;
                                                }
                                        }
@@ -4849,7 +4904,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        if chan_entry.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                        }
-                                       let (closing_signed, tx) = try_chan_entry!(self, chan_entry.get_mut().closing_signed(&self.fee_estimator, &msg), channel_state, chan_entry);
+                                       let (closing_signed, tx) = try_chan_entry!(self, chan_entry.get_mut().closing_signed(&self.fee_estimator, &msg), chan_entry);
                                        if let Some(msg) = closing_signed {
                                                channel_state.pending_msg_events.push(events::MessageSendEvent::SendClosingSigned {
                                                        node_id: counterparty_node_id.clone(),
@@ -4862,7 +4917,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                // also implies there are no pending HTLCs left on the channel, so we can
                                                // fully delete it from tracking (the channel monitor is still around to
                                                // watch for old state broadcasts)!
-                                               (tx, Some(remove_channel!(self, channel_state, chan_entry)))
+                                               (tx, Some(remove_channel!(self, chan_entry)))
                                        } else { (tx, None) }
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
@@ -4926,7 +4981,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                _ => pending_forward_info
                                        }
                                };
-                               try_chan_entry!(self, chan.get_mut().update_add_htlc(&msg, pending_forward_info, create_pending_htlc_status, &self.logger), channel_state, chan);
+                               try_chan_entry!(self, chan.get_mut().update_add_htlc(&msg, pending_forward_info, create_pending_htlc_status, &self.logger), chan);
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                }
@@ -4942,7 +4997,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                        }
-                                       try_chan_entry!(self, chan.get_mut().update_fulfill_htlc(&msg), channel_state, chan)
+                                       try_chan_entry!(self, chan.get_mut().update_fulfill_htlc(&msg), chan)
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                        }
@@ -4959,7 +5014,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
-                               try_chan_entry!(self, chan.get_mut().update_fail_htlc(&msg, HTLCFailReason::LightningError { err: msg.reason.clone() }), channel_state, chan);
+                               try_chan_entry!(self, chan.get_mut().update_fail_htlc(&msg, HTLCFailReason::LightningError { err: msg.reason.clone() }), chan);
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                }
@@ -4976,9 +5031,9 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                }
                                if (msg.failure_code & 0x8000) == 0 {
                                        let chan_err: ChannelError = ChannelError::Close("Got update_fail_malformed_htlc with BADONION not set".to_owned());
-                                       try_chan_entry!(self, Err(chan_err), channel_state, chan);
+                                       try_chan_entry!(self, Err(chan_err), chan);
                                }
-                               try_chan_entry!(self, chan.get_mut().update_fail_malformed_htlc(&msg, HTLCFailReason::Reason { failure_code: msg.failure_code, data: Vec::new() }), channel_state, chan);
+                               try_chan_entry!(self, chan.get_mut().update_fail_malformed_htlc(&msg, HTLCFailReason::Reason { failure_code: msg.failure_code, data: Vec::new() }), chan);
                                Ok(())
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
@@ -4995,17 +5050,17 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                }
                                let (revoke_and_ack, commitment_signed, monitor_update) =
                                        match chan.get_mut().commitment_signed(&msg, &self.logger) {
-                                               Err((None, e)) => try_chan_entry!(self, Err(e), channel_state, chan),
+                                               Err((None, e)) => try_chan_entry!(self, Err(e), chan),
                                                Err((Some(update), e)) => {
                                                        assert!(chan.get().is_awaiting_monitor_update());
                                                        let _ = self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), update);
-                                                       try_chan_entry!(self, Err(e), channel_state, chan);
+                                                       try_chan_entry!(self, Err(e), chan);
                                                        unreachable!();
                                                },
                                                Ok(res) => res
                                        };
                                let update_res = self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update);
-                               if let Err(e) = handle_monitor_update_res!(self, update_res, channel_state, chan, RAACommitmentOrder::RevokeAndACKFirst, true, commitment_signed.is_some()) {
+                               if let Err(e) = handle_monitor_update_res!(self, update_res, chan, RAACommitmentOrder::RevokeAndACKFirst, true, commitment_signed.is_some()) {
                                        return Err(e);
                                }
 
@@ -5048,12 +5103,12 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                        PendingHTLCRouting::ReceiveKeysend { .. } => 0,
                                        }) {
                                                hash_map::Entry::Occupied(mut entry) => {
-                                                       entry.get_mut().push(HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_funding_outpoint,
-                                                                                                       prev_htlc_id, forward_info });
+                                                       entry.get_mut().push(HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
+                                                               prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, forward_info }));
                                                },
                                                hash_map::Entry::Vacant(entry) => {
-                                                       entry.insert(vec!(HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_funding_outpoint,
-                                                                                                    prev_htlc_id, forward_info }));
+                                                       entry.insert(vec!(HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
+                                                               prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, forward_info })));
                                                }
                                        }
                                }
@@ -5082,7 +5137,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        }
                                        let was_paused_for_mon_update = chan.get().is_awaiting_monitor_update();
                                        let raa_updates = break_chan_entry!(self,
-                                               chan.get_mut().revoke_and_ack(&msg, &self.logger), channel_state, chan);
+                                               chan.get_mut().revoke_and_ack(&msg, &self.logger), chan);
                                        htlcs_to_fail = raa_updates.holding_cell_failed_htlcs;
                                        let update_res = self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), raa_updates.monitor_update);
                                        if was_paused_for_mon_update {
@@ -5094,7 +5149,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                break Err(MsgHandleErrInternal::ignore_no_close("Existing pending monitor update prevented responses to RAA".to_owned()));
                                        }
                                        if update_res != ChannelMonitorUpdateStatus::Completed {
-                                               if let Err(e) = handle_monitor_update_res!(self, update_res, channel_state, chan,
+                                               if let Err(e) = handle_monitor_update_res!(self, update_res, chan,
                                                                RAACommitmentOrder::CommitmentFirst, false,
                                                                raa_updates.commitment_update.is_some(), false,
                                                                raa_updates.accepted_htlcs, raa_updates.failed_htlcs,
@@ -5142,7 +5197,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
-                               try_chan_entry!(self, chan.get_mut().update_fee(&self.fee_estimator, &msg), channel_state, chan);
+                               try_chan_entry!(self, chan.get_mut().update_fee(&self.fee_estimator, &msg), chan);
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                }
@@ -5164,7 +5219,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
                                channel_state.pending_msg_events.push(events::MessageSendEvent::BroadcastChannelAnnouncement {
                                        msg: try_chan_entry!(self, chan.get_mut().announcement_signatures(
-                                               self.get_our_node_id(), self.genesis_hash.clone(), self.best_block.read().unwrap().height(), msg), channel_state, chan),
+                                               self.get_our_node_id(), self.genesis_hash.clone(), self.best_block.read().unwrap().height(), msg), chan),
                                        // Note that announcement_signatures fails if the channel cannot be announced,
                                        // so get_channel_update_for_broadcast will never fail by the time we get here.
                                        update_msg: self.get_channel_update_for_broadcast(chan.get()).unwrap(),
@@ -5177,15 +5232,15 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
        /// Returns ShouldPersist if anything changed, otherwise either SkipPersist or an Err.
        fn internal_channel_update(&self, counterparty_node_id: &PublicKey, msg: &msgs::ChannelUpdate) -> Result<NotifyOption, MsgHandleErrInternal> {
-               let mut channel_state_lock = self.channel_state.lock().unwrap();
-               let channel_state = &mut *channel_state_lock;
-               let chan_id = match channel_state.short_to_chan_info.get(&msg.contents.short_channel_id) {
+               let chan_id = match self.short_to_chan_info.read().unwrap().get(&msg.contents.short_channel_id) {
                        Some((_cp_id, chan_id)) => chan_id.clone(),
                        None => {
                                // It's not a local channel
                                return Ok(NotifyOption::SkipPersist)
                        }
                };
+               let mut channel_state_lock = self.channel_state.lock().unwrap();
+               let channel_state = &mut *channel_state_lock;
                match channel_state.by_id.entry(chan_id) {
                        hash_map::Entry::Occupied(mut chan) => {
                                if chan.get().get_counterparty_node_id() != *counterparty_node_id {
@@ -5203,10 +5258,10 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        return Ok(NotifyOption::SkipPersist);
                                } else {
                                        log_debug!(self.logger, "Received channel_update for channel {}.", log_bytes!(chan_id));
-                                       try_chan_entry!(self, chan.get_mut().channel_update(&msg), channel_state, chan);
+                                       try_chan_entry!(self, chan.get_mut().channel_update(&msg), chan);
                                }
                        },
-                       hash_map::Entry::Vacant(_) => unreachable!()
+                       hash_map::Entry::Vacant(_) => return Ok(NotifyOption::SkipPersist)
                }
                Ok(NotifyOption::DoPersist)
        }
@@ -5228,7 +5283,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                        // add-HTLCs on disconnect, we may be handed HTLCs to fail backwards here.
                                        let responses = try_chan_entry!(self, chan.get_mut().channel_reestablish(
                                                msg, &self.logger, self.our_network_pubkey.clone(), self.genesis_hash,
-                                               &*self.best_block.read().unwrap()), channel_state, chan);
+                                               &*self.best_block.read().unwrap()), chan);
                                        let mut channel_update = None;
                                        if let Some(msg) = responses.shutdown_msg {
                                                channel_state.pending_msg_events.push(events::MessageSendEvent::SendShutdown {
@@ -5292,7 +5347,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                let by_id = &mut channel_state.by_id;
                                                let pending_msg_events = &mut channel_state.pending_msg_events;
                                                if let hash_map::Entry::Occupied(chan_entry) = by_id.entry(funding_outpoint.to_channel_id()) {
-                                                       let mut chan = remove_channel!(self, channel_state, chan_entry);
+                                                       let mut chan = remove_channel!(self, chan_entry);
                                                        failed_channels.push(chan.force_shutdown(false));
                                                        if let Ok(update) = self.get_channel_update_for_broadcast(&chan) {
                                                                pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
@@ -5351,7 +5406,6 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        let by_id = &mut channel_state.by_id;
-                       let short_to_chan_info = &mut channel_state.short_to_chan_info;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
 
                        by_id.retain(|channel_id, chan| {
@@ -5374,7 +5428,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                                },
                                                                e => {
                                                                        has_monitor_update = true;
-                                                                       let (res, close_channel) = handle_monitor_update_res!(self, e, short_to_chan_info, chan, RAACommitmentOrder::CommitmentFirst, channel_id, COMMITMENT_UPDATE_ONLY);
+                                                                       let (res, close_channel) = handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, channel_id, COMMITMENT_UPDATE_ONLY);
                                                                        handle_errors.push((chan.get_counterparty_node_id(), res));
                                                                        if close_channel { return false; }
                                                                },
@@ -5383,7 +5437,7 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                                                true
                                        },
                                        Err(e) => {
-                                               let (close_channel, res) = convert_chan_err!(self, e, short_to_chan_info, chan, channel_id);
+                                               let (close_channel, res) = convert_chan_err!(self, e, chan, channel_id);
                                                handle_errors.push((chan.get_counterparty_node_id(), Err(res)));
                                                // ChannelClosed event is generated by handle_error for us
                                                !close_channel
@@ -5414,7 +5468,6 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        let by_id = &mut channel_state.by_id;
-                       let short_to_chan_info = &mut channel_state.short_to_chan_info;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
 
                        by_id.retain(|channel_id, chan| {
@@ -5439,13 +5492,13 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
 
                                                        log_info!(self.logger, "Broadcasting {}", log_tx!(tx));
                                                        self.tx_broadcaster.broadcast_transaction(&tx);
-                                                       update_maps_on_chan_removal!(self, short_to_chan_info, chan);
+                                                       update_maps_on_chan_removal!(self, chan);
                                                        false
                                                } else { true }
                                        },
                                        Err(e) => {
                                                has_update = true;
-                                               let (close_channel, res) = convert_chan_err!(self, e, short_to_chan_info, chan, channel_id);
+                                               let (close_channel, res) = convert_chan_err!(self, e, chan, channel_id);
                                                handle_errors.push((chan.get_counterparty_node_id(), Err(res)));
                                                !close_channel
                                        }
@@ -5635,14 +5688,14 @@ impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F
        ///
        /// [phantom node payments]: crate::chain::keysinterface::PhantomKeysManager
        pub fn get_phantom_scid(&self) -> u64 {
-               let mut channel_state = self.channel_state.lock().unwrap();
-               let best_block = self.best_block.read().unwrap();
+               let best_block_height = self.best_block.read().unwrap().height();
+               let short_to_chan_info = self.short_to_chan_info.read().unwrap();
                loop {
-                       let scid_candidate = fake_scid::Namespace::Phantom.get_fake_scid(best_block.height(), &self.genesis_hash, &self.fake_scid_rand_bytes, &self.keys_manager);
+                       let scid_candidate = fake_scid::Namespace::Phantom.get_fake_scid(best_block_height, &self.genesis_hash, &self.fake_scid_rand_bytes, &self.keys_manager);
                        // Ensure the generated scid doesn't conflict with a real channel.
-                       match channel_state.short_to_chan_info.entry(scid_candidate) {
-                               hash_map::Entry::Occupied(_) => continue,
-                               hash_map::Entry::Vacant(_) => return scid_candidate
+                       match short_to_chan_info.get(&scid_candidate) {
+                               Some(_) => continue,
+                               None => return scid_candidate
                        }
                }
        }
@@ -5855,7 +5908,7 @@ where
 
        fn get_relevant_txids(&self) -> Vec<(Txid, Option<BlockHash>)> {
                let channel_state = self.channel_state.lock().unwrap();
-               let mut res = Vec::with_capacity(channel_state.short_to_chan_info.len());
+               let mut res = Vec::with_capacity(channel_state.by_id.len());
                for chan in channel_state.by_id.values() {
                        if let (Some(funding_txo), block_hash) = (chan.get_funding_txo(), chan.get_funding_tx_confirmed_in()) {
                                res.push((funding_txo.txid, block_hash));
@@ -5898,7 +5951,6 @@ where
                {
                        let mut channel_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_lock;
-                       let short_to_chan_info = &mut channel_state.short_to_chan_info;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
                        channel_state.by_id.retain(|_, channel| {
                                let res = f(channel);
@@ -5910,7 +5962,7 @@ where
                                                }, HTLCDestination::NextHopChannel { node_id: Some(channel.get_counterparty_node_id()), channel_id: channel.channel_id() }));
                                        }
                                        if let Some(channel_ready) = channel_ready_opt {
-                                               send_channel_ready!(short_to_chan_info, pending_msg_events, channel, channel_ready);
+                                               send_channel_ready!(self, pending_msg_events, channel, channel_ready);
                                                if channel.is_usable() {
                                                        log_trace!(self.logger, "Sending channel_ready with private initial channel_update for our counterparty on channel {}", log_bytes!(channel.channel_id()));
                                                        if let Ok(msg) = self.get_channel_update_for_unicast(channel) {
@@ -5951,6 +6003,7 @@ where
                                                        // enforce option_scid_alias then), and if the funding tx is ever
                                                        // un-confirmed we force-close the channel, ensuring short_to_chan_info
                                                        // is always consistent.
+                                                       let mut short_to_chan_info = self.short_to_chan_info.write().unwrap();
                                                        let scid_insert = short_to_chan_info.insert(real_scid, (channel.get_counterparty_node_id(), channel.channel_id()));
                                                        assert!(scid_insert.is_none() || scid_insert.unwrap() == (channel.get_counterparty_node_id(), channel.channel_id()),
                                                                "SCIDs should never collide - ensure you weren't behind by a full {} blocks when creating channels",
@@ -5958,7 +6011,7 @@ where
                                                }
                                        }
                                } else if let Err(reason) = res {
-                                       update_maps_on_chan_removal!(self, short_to_chan_info, channel);
+                                       update_maps_on_chan_removal!(self, channel);
                                        // It looks like our counterparty went on-chain or funding transaction was
                                        // reorged out of the main chain. Close the channel.
                                        failed_channels.push(channel.force_shutdown(true));
@@ -6154,14 +6207,13 @@ impl<M: Deref , T: Deref , K: Deref , F: Deref , L: Deref >
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
-                       let short_to_chan_info = &mut channel_state.short_to_chan_info;
                        log_debug!(self.logger, "Marking channels with {} disconnected and generating channel_updates. We believe we {} make future connections to this peer.",
                                log_pubkey!(counterparty_node_id), if no_connection_possible { "cannot" } else { "can" });
                        channel_state.by_id.retain(|_, chan| {
                                if chan.get_counterparty_node_id() == *counterparty_node_id {
                                        chan.remove_uncommitted_htlcs_and_mark_paused(&self.logger);
                                        if chan.is_shutdown() {
-                                               update_maps_on_chan_removal!(self, short_to_chan_info, chan);
+                                               update_maps_on_chan_removal!(self, chan);
                                                self.issue_channel_close_events(chan, ClosureReason::DisconnectedPeer);
                                                return false;
                                        } else {
@@ -6421,7 +6473,8 @@ impl_writeable_tlv_based!(PendingHTLCInfo, {
        (2, incoming_shared_secret, required),
        (4, payment_hash, required),
        (6, amt_to_forward, required),
-       (8, outgoing_cltv_value, required)
+       (8, outgoing_cltv_value, required),
+       (9, amt_incoming, option),
 });
 
 
@@ -6620,7 +6673,7 @@ impl Writeable for HTLCSource {
                                        (1, payment_id_opt, option),
                                        (2, first_hop_htlc_msat, required),
                                        (3, payment_secret, option),
-                                       (4, path, vec_type),
+                                       (4, *path, vec_type),
                                        (5, payment_params, option),
                                 });
                        }
@@ -6643,18 +6696,20 @@ impl_writeable_tlv_based_enum!(HTLCFailReason,
        },
 ;);
 
+impl_writeable_tlv_based!(PendingAddHTLCInfo, {
+       (0, forward_info, required),
+       (2, prev_short_channel_id, required),
+       (4, prev_htlc_id, required),
+       (6, prev_funding_outpoint, required),
+});
+
 impl_writeable_tlv_based_enum!(HTLCForwardInfo,
-       (0, AddHTLC) => {
-               (0, forward_info, required),
-               (2, prev_short_channel_id, required),
-               (4, prev_htlc_id, required),
-               (6, prev_funding_outpoint, required),
-       },
        (1, FailHTLC) => {
                (0, htlc_id, required),
                (2, err_packet, required),
-       },
-;);
+       };
+       (0, AddHTLC)
+);
 
 impl_writeable_tlv_based!(PendingInboundPayment, {
        (0, payment_secret, required),
@@ -7351,7 +7406,6 @@ impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 
                        channel_state: Mutex::new(ChannelHolder {
                                by_id,
-                               short_to_chan_info,
                                claimable_htlcs,
                                pending_msg_events: Vec::new(),
                        }),
@@ -7362,6 +7416,7 @@ impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                        forward_htlcs: Mutex::new(forward_htlcs),
                        outbound_scid_aliases: Mutex::new(outbound_scid_aliases),
                        id_to_peer: Mutex::new(id_to_peer),
+                       short_to_chan_info: FairRwLock::new(short_to_chan_info),
                        fake_scid_rand_bytes: fake_scid_rand_bytes.unwrap(),
 
                        probing_cookie_secret: probing_cookie_secret.unwrap(),
index e8a3e6d26facd524c3ae3036ed87ec54a08b8937..77d0fa4529fb2ea526682079229b81fa9165ddf6 100644 (file)
@@ -157,6 +157,7 @@ mod sealed {
                // Byte 2
                BasicMPP,
        ]);
+       define_context!(OfferContext, []);
        // This isn't a "real" feature context, and is only used in the channel_type field in an
        // `OpenChannel` message.
        define_context!(ChannelTypeContext, [
@@ -366,7 +367,7 @@ mod sealed {
                supports_keysend, requires_keysend);
 
        #[cfg(test)]
-       define_feature!(123456789, UnknownFeature, [NodeContext, ChannelContext, InvoiceContext],
+       define_feature!(123456789, UnknownFeature, [NodeContext, ChannelContext, InvoiceContext, OfferContext],
                "Feature flags for an unknown feature used in testing.", set_unknown_feature_optional,
                set_unknown_feature_required, supports_unknown_test_feature, requires_unknown_test_feature);
 }
@@ -425,6 +426,8 @@ pub type NodeFeatures = Features<sealed::NodeContext>;
 pub type ChannelFeatures = Features<sealed::ChannelContext>;
 /// Features used within an invoice.
 pub type InvoiceFeatures = Features<sealed::InvoiceContext>;
+/// Features used within an offer.
+pub type OfferFeatures = Features<sealed::OfferContext>;
 
 /// Features used within the channel_type field in an OpenChannel message.
 ///
@@ -684,6 +687,15 @@ impl<T: sealed::Wumbo> Features<T> {
        }
 }
 
+#[cfg(test)]
+impl<T: sealed::UnknownFeature> Features<T> {
+       pub(crate) fn unknown() -> Self {
+               let mut features = Self::empty();
+               features.set_unknown_feature_required();
+               features
+       }
+}
+
 macro_rules! impl_feature_len_prefixed_write {
        ($features: ident) => {
                impl Writeable for $features {
@@ -704,21 +716,26 @@ impl_feature_len_prefixed_write!(ChannelFeatures);
 impl_feature_len_prefixed_write!(NodeFeatures);
 impl_feature_len_prefixed_write!(InvoiceFeatures);
 
-// Because ChannelTypeFeatures only appears inside of TLVs, it doesn't have a length prefix when
-// serialized. Thus, we can't use `impl_feature_len_prefixed_write`, above, and have to write our
-// own serialization.
-impl Writeable for ChannelTypeFeatures {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               self.write_be(w)
-       }
-}
-impl Readable for ChannelTypeFeatures {
-       fn read<R: io::Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let v = io_extras::read_to_end(r)?;
-               Ok(Self::from_be_bytes(v))
+// Some features only appear inside of TLVs, so they don't have a length prefix when serialized.
+macro_rules! impl_feature_tlv_write {
+       ($features: ident) => {
+               impl Writeable for $features {
+                       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+                               self.write_be(w)
+                       }
+               }
+               impl Readable for $features {
+                       fn read<R: io::Read>(r: &mut R) -> Result<Self, DecodeError> {
+                               let v = io_extras::read_to_end(r)?;
+                               Ok(Self::from_be_bytes(v))
+                       }
+               }
        }
 }
 
+impl_feature_tlv_write!(ChannelTypeFeatures);
+impl_feature_tlv_write!(OfferFeatures);
+
 #[cfg(test)]
 mod tests {
        use super::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, InvoiceFeatures, NodeFeatures, sealed};
index 14def2f6b7a3d9a133681da6bc67087a10700400..fb630abb9a88e71b9551f8e51a98707d77a9e9fb 100644 (file)
@@ -15,7 +15,7 @@ use crate::chain::channelmonitor::{ChannelMonitor, CLTV_CLAIM_BUFFER, LATENCY_GR
 use crate::chain::keysinterface::{KeysInterface, Recipient};
 use crate::ln::{PaymentHash, PaymentSecret};
 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
-use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, HTLCForwardInfo, CLTV_FAR_FAR_AWAY, MIN_CLTV_EXPIRY_DELTA, PendingHTLCInfo, PendingHTLCRouting, PaymentId};
+use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, HTLCForwardInfo, CLTV_FAR_FAR_AWAY, MIN_CLTV_EXPIRY_DELTA, PendingAddHTLCInfo, PendingHTLCInfo, PendingHTLCRouting, PaymentId};
 use crate::ln::onion_utils;
 use crate::routing::gossip::{NetworkUpdate, RoutingFees, NodeId};
 use crate::routing::router::{get_route, PaymentParameters, Route, RouteHint, RouteHintHop};
@@ -554,7 +554,7 @@ fn test_onion_failure() {
                for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
                        for f in pending_forwards.iter_mut() {
                                match f {
-                                       &mut HTLCForwardInfo::AddHTLC { ref mut forward_info, .. } =>
+                                       &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) =>
                                                forward_info.outgoing_cltv_value += 1,
                                        _ => {},
                                }
@@ -567,7 +567,7 @@ fn test_onion_failure() {
                for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
                        for f in pending_forwards.iter_mut() {
                                match f {
-                                       &mut HTLCForwardInfo::AddHTLC { ref mut forward_info, .. } =>
+                                       &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) =>
                                                forward_info.amt_to_forward -= 1,
                                        _ => {},
                                }
@@ -1035,12 +1035,12 @@ fn test_phantom_onion_hmac_failure() {
                let mut forward_htlcs = nodes[1].node.forward_htlcs.lock().unwrap();
                let mut pending_forward = forward_htlcs.get_mut(&phantom_scid).unwrap();
                match pending_forward[0] {
-                       HTLCForwardInfo::AddHTLC {
+                       HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
                                forward_info: PendingHTLCInfo {
                                        routing: PendingHTLCRouting::Forward { ref mut onion_packet, .. },
                                        ..
                                }, ..
-                       } => {
+                       }) => {
                                onion_packet.hmac[onion_packet.hmac.len() - 1] ^= 1;
                                Sha256::hash(&onion_packet.hop_data).into_inner().to_vec()
                        },
@@ -1095,12 +1095,12 @@ fn test_phantom_invalid_onion_payload() {
        for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
                for f in pending_forwards.iter_mut() {
                        match f {
-                               &mut HTLCForwardInfo::AddHTLC {
+                               &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
                                        forward_info: PendingHTLCInfo {
                                                routing: PendingHTLCRouting::Forward { ref mut onion_packet, .. },
                                                ..
                                        }, ..
-                               } => {
+                               }) => {
                                        // Construct the onion payloads for the entire route and an invalid amount.
                                        let height = nodes[0].best_block_info().1;
                                        let session_priv = SecretKey::from_slice(&session_priv).unwrap();
@@ -1166,9 +1166,9 @@ fn test_phantom_final_incorrect_cltv_expiry() {
        for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
                for f in pending_forwards.iter_mut() {
                        match f {
-                               &mut HTLCForwardInfo::AddHTLC {
+                               &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
                                        forward_info: PendingHTLCInfo { ref mut outgoing_cltv_value, .. }, ..
-                               } => {
+                               }) => {
                                        *outgoing_cltv_value += 1;
                                },
                                _ => panic!("Unexpected forward"),
index 1e26d33920ac5862dc7430cbb4b34d73f9537ddd..a445f420a9c96011238a28a0eff4c7e0e8fe4426 100644 (file)
@@ -276,7 +276,7 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, use_funding_
 
        let channel_state = nodes[0].node.channel_state.lock().unwrap();
        assert_eq!(channel_state.by_id.len(), 1);
-       assert_eq!(channel_state.short_to_chan_info.len(), 2);
+       assert_eq!(nodes[0].node.short_to_chan_info.read().unwrap().len(), 2);
        mem::drop(channel_state);
 
        if !reorg_after_reload {
@@ -305,7 +305,7 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, use_funding_
                {
                        let channel_state = nodes[0].node.channel_state.lock().unwrap();
                        assert_eq!(channel_state.by_id.len(), 0);
-                       assert_eq!(channel_state.short_to_chan_info.len(), 0);
+                       assert_eq!(nodes[0].node.short_to_chan_info.read().unwrap().len(), 0);
                }
        }
 
@@ -383,7 +383,7 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, use_funding_
                {
                        let channel_state = nodes[0].node.channel_state.lock().unwrap();
                        assert_eq!(channel_state.by_id.len(), 0);
-                       assert_eq!(channel_state.short_to_chan_info.len(), 0);
+                       assert_eq!(nodes[0].node.short_to_chan_info.read().unwrap().len(), 0);
                }
        }
        // With expect_channel_force_closed set the TestChainMonitor will enforce that the next update
diff --git a/lightning/src/offers/mod.rs b/lightning/src/offers/mod.rs
new file mode 100644 (file)
index 0000000..2f961a0
--- /dev/null
@@ -0,0 +1,15 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Implementation of Lightning Offers
+//! ([BOLT 12](https://github.com/lightning/bolts/blob/master/12-offer-encoding.md)).
+//!
+//! Offers are a flexible protocol for Lightning payments.
+
+pub mod offer;
diff --git a/lightning/src/offers/offer.rs b/lightning/src/offers/offer.rs
new file mode 100644 (file)
index 0000000..bf569fb
--- /dev/null
@@ -0,0 +1,709 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Data structures and encoding for `offer` messages.
+//!
+//! An [`Offer`] represents an "offer to be paid." It is typically constructed by a merchant and
+//! published as a QR code to be scanned by a customer. The customer uses the offer to request an
+//! invoice from the merchant to be paid.
+//!
+//! ```ignore
+//! extern crate bitcoin;
+//! extern crate core;
+//! extern crate lightning;
+//!
+//! use core::num::NonZeroU64;
+//! use core::time::Duration;
+//!
+//! use bitcoin::secp256k1::{KeyPair, PublicKey, Secp256k1, SecretKey};
+//! use lightning::offers::offer::{OfferBuilder, Quantity};
+//!
+//! # use bitcoin::secp256k1;
+//! # use lightning::onion_message::BlindedPath;
+//! # #[cfg(feature = "std")]
+//! # use std::time::SystemTime;
+//! #
+//! # fn create_blinded_path() -> BlindedPath { unimplemented!() }
+//! # fn create_another_blinded_path() -> BlindedPath { unimplemented!() }
+//! #
+//! # #[cfg(feature = "std")]
+//! # fn build() -> Result<(), secp256k1::Error> {
+//! let secp_ctx = Secp256k1::new();
+//! let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32])?);
+//! let pubkey = PublicKey::from(keys);
+//!
+//! let expiration = SystemTime::now() + Duration::from_secs(24 * 60 * 60);
+//! let offer = OfferBuilder::new("coffee, large".to_string(), pubkey)
+//!     .amount_msats(20_000)
+//!     .supported_quantity(Quantity::Unbounded)
+//!     .absolute_expiry(expiration.duration_since(SystemTime::UNIX_EPOCH).unwrap())
+//!     .issuer("Foo Bar".to_string())
+//!     .path(create_blinded_path())
+//!     .path(create_another_blinded_path())
+//!     .build()
+//!     .unwrap();
+//! # Ok(())
+//! # }
+//! ```
+
+use bitcoin::blockdata::constants::ChainHash;
+use bitcoin::network::constants::Network;
+use bitcoin::secp256k1::PublicKey;
+use core::num::NonZeroU64;
+use core::time::Duration;
+use crate::io;
+use crate::ln::features::OfferFeatures;
+use crate::ln::msgs::MAX_VALUE_MSAT;
+use crate::onion_message::BlindedPath;
+use crate::util::ser::{HighZeroBytesDroppedBigSize, WithoutLength, Writeable, Writer};
+use crate::util::string::PrintableString;
+
+use crate::prelude::*;
+
+#[cfg(feature = "std")]
+use std::time::SystemTime;
+
+/// Builds an [`Offer`] for the "offer to be paid" flow.
+///
+/// See [module-level documentation] for usage.
+///
+/// [module-level documentation]: self
+pub struct OfferBuilder {
+       offer: OfferContents,
+}
+
+impl OfferBuilder {
+       /// Creates a new builder for an offer setting the [`Offer::description`] and using the
+       /// [`Offer::signing_pubkey`] for signing invoices. The associated secret key must be remembered
+       /// while the offer is valid.
+       ///
+       /// Use a different pubkey per offer to avoid correlating offers.
+       pub fn new(description: String, signing_pubkey: PublicKey) -> Self {
+               let offer = OfferContents {
+                       chains: None, metadata: None, amount: None, description,
+                       features: OfferFeatures::empty(), absolute_expiry: None, issuer: None, paths: None,
+                       supported_quantity: Quantity::one(), signing_pubkey: Some(signing_pubkey),
+               };
+               OfferBuilder { offer }
+       }
+
+       /// Adds the chain hash of the given [`Network`] to [`Offer::chains`]. If not called,
+       /// the chain hash of [`Network::Bitcoin`] is assumed to be the only one supported.
+       ///
+       /// See [`Offer::chains`] on how this relates to the payment currency.
+       ///
+       /// Successive calls to this method will add another chain hash.
+       pub fn chain(mut self, network: Network) -> Self {
+               let chains = self.offer.chains.get_or_insert_with(Vec::new);
+               let chain = ChainHash::using_genesis_block(network);
+               if !chains.contains(&chain) {
+                       chains.push(chain);
+               }
+
+               self
+       }
+
+       /// Sets the [`Offer::metadata`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       pub fn metadata(mut self, metadata: Vec<u8>) -> Self {
+               self.offer.metadata = Some(metadata);
+               self
+       }
+
+       /// Sets the [`Offer::amount`] as an [`Amount::Bitcoin`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       pub fn amount_msats(mut self, amount_msats: u64) -> Self {
+               self.amount(Amount::Bitcoin { amount_msats })
+       }
+
+       /// Sets the [`Offer::amount`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       fn amount(mut self, amount: Amount) -> Self {
+               self.offer.amount = Some(amount);
+               self
+       }
+
+       /// Sets the [`Offer::features`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       #[cfg(test)]
+       pub fn features(mut self, features: OfferFeatures) -> Self {
+               self.offer.features = features;
+               self
+       }
+
+       /// Sets the [`Offer::absolute_expiry`] as seconds since the Unix epoch. Any expiry that has
+       /// already passed is valid and can be checked for using [`Offer::is_expired`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       pub fn absolute_expiry(mut self, absolute_expiry: Duration) -> Self {
+               self.offer.absolute_expiry = Some(absolute_expiry);
+               self
+       }
+
+       /// Sets the [`Offer::issuer`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       pub fn issuer(mut self, issuer: String) -> Self {
+               self.offer.issuer = Some(issuer);
+               self
+       }
+
+       /// Adds a blinded path to [`Offer::paths`]. Must include at least one path if only connected by
+       /// private channels or if [`Offer::signing_pubkey`] is not a public node id.
+       ///
+       /// Successive calls to this method will add another blinded path. Caller is responsible for not
+       /// adding duplicate paths.
+       pub fn path(mut self, path: BlindedPath) -> Self {
+               self.offer.paths.get_or_insert_with(Vec::new).push(path);
+               self
+       }
+
+       /// Sets the quantity of items for [`Offer::supported_quantity`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       pub fn supported_quantity(mut self, quantity: Quantity) -> Self {
+               self.offer.supported_quantity = quantity;
+               self
+       }
+
+       /// Builds an [`Offer`] from the builder's settings.
+       pub fn build(mut self) -> Result<Offer, ()> {
+               match self.offer.amount {
+                       Some(Amount::Bitcoin { amount_msats }) => {
+                               if amount_msats > MAX_VALUE_MSAT {
+                                       return Err(());
+                               }
+                       },
+                       Some(Amount::Currency { .. }) => unreachable!(),
+                       None => {},
+               }
+
+               if let Some(chains) = &self.offer.chains {
+                       if chains.len() == 1 && chains[0] == self.offer.implied_chain() {
+                               self.offer.chains = None;
+                       }
+               }
+
+               let mut bytes = Vec::new();
+               self.offer.write(&mut bytes).unwrap();
+
+               Ok(Offer {
+                       bytes,
+                       contents: self.offer,
+               })
+       }
+}
+
+/// An `Offer` is a potentially long-lived proposal for payment of a good or service.
+///
+/// An offer is a precursor to an `InvoiceRequest`. A merchant publishes an offer from which a
+/// customer may request an `Invoice` for a specific quantity and using an amount sufficient to
+/// cover that quantity (i.e., at least `quantity * amount`). See [`Offer::amount`].
+///
+/// Offers may be denominated in currency other than bitcoin but are ultimately paid using the
+/// latter.
+///
+/// Through the use of [`BlindedPath`]s, offers provide recipient privacy.
+#[derive(Clone, Debug)]
+pub struct Offer {
+       // The serialized offer. Needed when creating an `InvoiceRequest` if the offer contains unknown
+       // fields.
+       bytes: Vec<u8>,
+       contents: OfferContents,
+}
+
+/// The contents of an [`Offer`], which may be shared with an `InvoiceRequest` or an `Invoice`.
+#[derive(Clone, Debug)]
+pub(crate) struct OfferContents {
+       chains: Option<Vec<ChainHash>>,
+       metadata: Option<Vec<u8>>,
+       amount: Option<Amount>,
+       description: String,
+       features: OfferFeatures,
+       absolute_expiry: Option<Duration>,
+       issuer: Option<String>,
+       paths: Option<Vec<BlindedPath>>,
+       supported_quantity: Quantity,
+       signing_pubkey: Option<PublicKey>,
+}
+
+impl Offer {
+       // TODO: Return a slice once ChainHash has constants.
+       // - https://github.com/rust-bitcoin/rust-bitcoin/pull/1283
+       // - https://github.com/rust-bitcoin/rust-bitcoin/pull/1286
+       /// The chains that may be used when paying a requested invoice (e.g., bitcoin mainnet).
+       /// Payments must be denominated in units of the minimal lightning-payable unit (e.g., msats)
+       /// for the selected chain.
+       pub fn chains(&self) -> Vec<ChainHash> {
+               self.contents.chains
+                       .as_ref()
+                       .cloned()
+                       .unwrap_or_else(|| vec![self.contents.implied_chain()])
+       }
+
+       // TODO: Link to corresponding method in `InvoiceRequest`.
+       /// Opaque bytes set by the originator. Useful for authentication and validating fields since it
+       /// is reflected in `invoice_request` messages along with all the other fields from the `offer`.
+       pub fn metadata(&self) -> Option<&Vec<u8>> {
+               self.contents.metadata.as_ref()
+       }
+
+       /// The minimum amount required for a successful payment of a single item.
+       pub fn amount(&self) -> Option<&Amount> {
+               self.contents.amount.as_ref()
+       }
+
+       /// A complete description of the purpose of the payment. Intended to be displayed to the user
+       /// but with the caveat that it has not been verified in any way.
+       pub fn description(&self) -> PrintableString {
+               PrintableString(&self.contents.description)
+       }
+
+       /// Features pertaining to the offer.
+       pub fn features(&self) -> &OfferFeatures {
+               &self.contents.features
+       }
+
+       /// Duration since the Unix epoch when an invoice should no longer be requested.
+       ///
+       /// If `None`, the offer does not expire.
+       pub fn absolute_expiry(&self) -> Option<Duration> {
+               self.contents.absolute_expiry
+       }
+
+       /// Whether the offer has expired.
+       #[cfg(feature = "std")]
+       pub fn is_expired(&self) -> bool {
+               match self.absolute_expiry() {
+                       Some(seconds_from_epoch) => match SystemTime::UNIX_EPOCH.elapsed() {
+                               Ok(elapsed) => elapsed > seconds_from_epoch,
+                               Err(_) => false,
+                       },
+                       None => false,
+               }
+       }
+
+       /// The issuer of the offer, possibly beginning with `user@domain` or `domain`. Intended to be
+       /// displayed to the user but with the caveat that it has not been verified in any way.
+       pub fn issuer(&self) -> Option<PrintableString> {
+               self.contents.issuer.as_ref().map(|issuer| PrintableString(issuer.as_str()))
+       }
+
+       /// Paths to the recipient originating from publicly reachable nodes. Blinded paths provide
+       /// recipient privacy by obfuscating its node id.
+       pub fn paths(&self) -> &[BlindedPath] {
+               self.contents.paths.as_ref().map(|paths| paths.as_slice()).unwrap_or(&[])
+       }
+
+       /// The quantity of items supported.
+       pub fn supported_quantity(&self) -> Quantity {
+               self.contents.supported_quantity()
+       }
+
+       /// The public key used by the recipient to sign invoices.
+       pub fn signing_pubkey(&self) -> PublicKey {
+               self.contents.signing_pubkey.unwrap()
+       }
+
+       #[cfg(test)]
+       fn as_tlv_stream(&self) -> OfferTlvStreamRef {
+               self.contents.as_tlv_stream()
+       }
+}
+
+impl OfferContents {
+       pub fn implied_chain(&self) -> ChainHash {
+               ChainHash::using_genesis_block(Network::Bitcoin)
+       }
+
+       pub fn supported_quantity(&self) -> Quantity {
+               self.supported_quantity
+       }
+
+       fn as_tlv_stream(&self) -> OfferTlvStreamRef {
+               let (currency, amount) = match &self.amount {
+                       None => (None, None),
+                       Some(Amount::Bitcoin { amount_msats }) => (None, Some(*amount_msats)),
+                       Some(Amount::Currency { iso4217_code, amount }) => (
+                               Some(iso4217_code), Some(*amount)
+                       ),
+               };
+
+               let features = {
+                       if self.features == OfferFeatures::empty() { None } else { Some(&self.features) }
+               };
+
+               OfferTlvStreamRef {
+                       chains: self.chains.as_ref(),
+                       metadata: self.metadata.as_ref(),
+                       currency,
+                       amount,
+                       description: Some(&self.description),
+                       features,
+                       absolute_expiry: self.absolute_expiry.map(|duration| duration.as_secs()),
+                       paths: self.paths.as_ref(),
+                       issuer: self.issuer.as_ref(),
+                       quantity_max: self.supported_quantity.to_tlv_record(),
+                       node_id: self.signing_pubkey.as_ref(),
+               }
+       }
+}
+
+impl Writeable for OfferContents {
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
+               self.as_tlv_stream().write(writer)
+       }
+}
+
+/// The minimum amount required for an item in an [`Offer`], denominated in either bitcoin or
+/// another currency.
+#[derive(Clone, Debug, PartialEq)]
+pub enum Amount {
+       /// An amount of bitcoin.
+       Bitcoin {
+               /// The amount in millisatoshi.
+               amount_msats: u64,
+       },
+       /// An amount of currency specified using ISO 4712.
+       Currency {
+               /// The currency that the amount is denominated in.
+               iso4217_code: CurrencyCode,
+               /// The amount in the currency unit adjusted by the ISO 4712 exponent (e.g., USD cents).
+               amount: u64,
+       },
+}
+
+/// An ISO 4712 three-letter currency code (e.g., USD).
+pub type CurrencyCode = [u8; 3];
+
+/// Quantity of items supported by an [`Offer`].
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum Quantity {
+       /// Up to a specific number of items (inclusive).
+       Bounded(NonZeroU64),
+       /// One or more items.
+       Unbounded,
+}
+
+impl Quantity {
+       fn one() -> Self {
+               Quantity::Bounded(NonZeroU64::new(1).unwrap())
+       }
+
+       fn to_tlv_record(&self) -> Option<u64> {
+               match self {
+                       Quantity::Bounded(n) => {
+                               let n = n.get();
+                               if n == 1 { None } else { Some(n) }
+                       },
+                       Quantity::Unbounded => Some(0),
+               }
+       }
+}
+
+tlv_stream!(OfferTlvStream, OfferTlvStreamRef, {
+       (2, chains: (Vec<ChainHash>, WithoutLength)),
+       (4, metadata: (Vec<u8>, WithoutLength)),
+       (6, currency: CurrencyCode),
+       (8, amount: (u64, HighZeroBytesDroppedBigSize)),
+       (10, description: (String, WithoutLength)),
+       (12, features: OfferFeatures),
+       (14, absolute_expiry: (u64, HighZeroBytesDroppedBigSize)),
+       (16, paths: (Vec<BlindedPath>, WithoutLength)),
+       (18, issuer: (String, WithoutLength)),
+       (20, quantity_max: (u64, HighZeroBytesDroppedBigSize)),
+       (22, node_id: PublicKey),
+});
+
+#[cfg(test)]
+mod tests {
+       use super::{Amount, OfferBuilder, Quantity};
+
+       use bitcoin::blockdata::constants::ChainHash;
+       use bitcoin::network::constants::Network;
+       use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey};
+       use core::num::NonZeroU64;
+       use core::time::Duration;
+       use crate::ln::features::OfferFeatures;
+       use crate::ln::msgs::MAX_VALUE_MSAT;
+       use crate::onion_message::{BlindedHop, BlindedPath};
+       use crate::util::ser::Writeable;
+       use crate::util::string::PrintableString;
+
+       fn pubkey(byte: u8) -> PublicKey {
+               let secp_ctx = Secp256k1::new();
+               PublicKey::from_secret_key(&secp_ctx, &privkey(byte))
+       }
+
+       fn privkey(byte: u8) -> SecretKey {
+               SecretKey::from_slice(&[byte; 32]).unwrap()
+       }
+
+       #[test]
+       fn builds_offer_with_defaults() {
+               let offer = OfferBuilder::new("foo".into(), pubkey(42)).build().unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               let mut buffer = Vec::new();
+               offer.contents.write(&mut buffer).unwrap();
+
+               assert_eq!(offer.bytes, buffer.as_slice());
+               assert_eq!(offer.chains(), vec![ChainHash::using_genesis_block(Network::Bitcoin)]);
+               assert_eq!(offer.metadata(), None);
+               assert_eq!(offer.amount(), None);
+               assert_eq!(offer.description(), PrintableString("foo"));
+               assert_eq!(offer.features(), &OfferFeatures::empty());
+               assert_eq!(offer.absolute_expiry(), None);
+               #[cfg(feature = "std")]
+               assert!(!offer.is_expired());
+               assert_eq!(offer.paths(), &[]);
+               assert_eq!(offer.issuer(), None);
+               assert_eq!(offer.supported_quantity(), Quantity::one());
+               assert_eq!(offer.signing_pubkey(), pubkey(42));
+
+               assert_eq!(tlv_stream.chains, None);
+               assert_eq!(tlv_stream.metadata, None);
+               assert_eq!(tlv_stream.currency, None);
+               assert_eq!(tlv_stream.amount, None);
+               assert_eq!(tlv_stream.description, Some(&String::from("foo")));
+               assert_eq!(tlv_stream.features, None);
+               assert_eq!(tlv_stream.absolute_expiry, None);
+               assert_eq!(tlv_stream.paths, None);
+               assert_eq!(tlv_stream.issuer, None);
+               assert_eq!(tlv_stream.quantity_max, None);
+               assert_eq!(tlv_stream.node_id, Some(&pubkey(42)));
+       }
+
+       #[test]
+       fn builds_offer_with_chains() {
+               let mainnet = ChainHash::using_genesis_block(Network::Bitcoin);
+               let testnet = ChainHash::using_genesis_block(Network::Testnet);
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .chain(Network::Bitcoin)
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.chains(), vec![mainnet]);
+               assert_eq!(offer.as_tlv_stream().chains, None);
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .chain(Network::Testnet)
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.chains(), vec![testnet]);
+               assert_eq!(offer.as_tlv_stream().chains, Some(&vec![testnet]));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .chain(Network::Testnet)
+                       .chain(Network::Testnet)
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.chains(), vec![testnet]);
+               assert_eq!(offer.as_tlv_stream().chains, Some(&vec![testnet]));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .chain(Network::Bitcoin)
+                       .chain(Network::Testnet)
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.chains(), vec![mainnet, testnet]);
+               assert_eq!(offer.as_tlv_stream().chains, Some(&vec![mainnet, testnet]));
+       }
+
+       #[test]
+       fn builds_offer_with_metadata() {
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .metadata(vec![42; 32])
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.metadata(), Some(&vec![42; 32]));
+               assert_eq!(offer.as_tlv_stream().metadata, Some(&vec![42; 32]));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .metadata(vec![42; 32])
+                       .metadata(vec![43; 32])
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.metadata(), Some(&vec![43; 32]));
+               assert_eq!(offer.as_tlv_stream().metadata, Some(&vec![43; 32]));
+       }
+
+       #[test]
+       fn builds_offer_with_amount() {
+               let bitcoin_amount = Amount::Bitcoin { amount_msats: 1000 };
+               let currency_amount = Amount::Currency { iso4217_code: *b"USD", amount: 10 };
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .amount_msats(1000)
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(offer.amount(), Some(&bitcoin_amount));
+               assert_eq!(tlv_stream.amount, Some(1000));
+               assert_eq!(tlv_stream.currency, None);
+
+               let builder = OfferBuilder::new("foo".into(), pubkey(42))
+                       .amount(currency_amount.clone());
+               let tlv_stream = builder.offer.as_tlv_stream();
+               assert_eq!(builder.offer.amount, Some(currency_amount.clone()));
+               assert_eq!(tlv_stream.amount, Some(10));
+               assert_eq!(tlv_stream.currency, Some(b"USD"));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .amount(currency_amount.clone())
+                       .amount(bitcoin_amount.clone())
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(tlv_stream.amount, Some(1000));
+               assert_eq!(tlv_stream.currency, None);
+
+               let invalid_amount = Amount::Bitcoin { amount_msats: MAX_VALUE_MSAT + 1 };
+               match OfferBuilder::new("foo".into(), pubkey(42)).amount(invalid_amount).build() {
+                       Ok(_) => panic!("expected error"),
+                       Err(e) => assert_eq!(e, ()),
+               }
+       }
+
+       #[test]
+       fn builds_offer_with_features() {
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .features(OfferFeatures::unknown())
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.features(), &OfferFeatures::unknown());
+               assert_eq!(offer.as_tlv_stream().features, Some(&OfferFeatures::unknown()));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .features(OfferFeatures::unknown())
+                       .features(OfferFeatures::empty())
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.features(), &OfferFeatures::empty());
+               assert_eq!(offer.as_tlv_stream().features, None);
+       }
+
+       #[test]
+       fn builds_offer_with_absolute_expiry() {
+               let future_expiry = Duration::from_secs(u64::max_value());
+               let past_expiry = Duration::from_secs(0);
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .absolute_expiry(future_expiry)
+                       .build()
+                       .unwrap();
+               #[cfg(feature = "std")]
+               assert!(!offer.is_expired());
+               assert_eq!(offer.absolute_expiry(), Some(future_expiry));
+               assert_eq!(offer.as_tlv_stream().absolute_expiry, Some(future_expiry.as_secs()));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .absolute_expiry(future_expiry)
+                       .absolute_expiry(past_expiry)
+                       .build()
+                       .unwrap();
+               #[cfg(feature = "std")]
+               assert!(offer.is_expired());
+               assert_eq!(offer.absolute_expiry(), Some(past_expiry));
+               assert_eq!(offer.as_tlv_stream().absolute_expiry, Some(past_expiry.as_secs()));
+       }
+
+       #[test]
+       fn builds_offer_with_paths() {
+               let paths = vec![
+                       BlindedPath {
+                               introduction_node_id: pubkey(40),
+                               blinding_point: pubkey(41),
+                               blinded_hops: vec![
+                                       BlindedHop { blinded_node_id: pubkey(43), encrypted_payload: vec![0; 43] },
+                                       BlindedHop { blinded_node_id: pubkey(44), encrypted_payload: vec![0; 44] },
+                               ],
+                       },
+                       BlindedPath {
+                               introduction_node_id: pubkey(40),
+                               blinding_point: pubkey(41),
+                               blinded_hops: vec![
+                                       BlindedHop { blinded_node_id: pubkey(45), encrypted_payload: vec![0; 45] },
+                                       BlindedHop { blinded_node_id: pubkey(46), encrypted_payload: vec![0; 46] },
+                               ],
+                       },
+               ];
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .path(paths[0].clone())
+                       .path(paths[1].clone())
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(offer.paths(), paths.as_slice());
+               assert_eq!(offer.signing_pubkey(), pubkey(42));
+               assert_ne!(pubkey(42), pubkey(44));
+               assert_eq!(tlv_stream.paths, Some(&paths));
+               assert_eq!(tlv_stream.node_id, Some(&pubkey(42)));
+       }
+
+       #[test]
+       fn builds_offer_with_issuer() {
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .issuer("bar".into())
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.issuer(), Some(PrintableString("bar")));
+               assert_eq!(offer.as_tlv_stream().issuer, Some(&String::from("bar")));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .issuer("bar".into())
+                       .issuer("baz".into())
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.issuer(), Some(PrintableString("baz")));
+               assert_eq!(offer.as_tlv_stream().issuer, Some(&String::from("baz")));
+       }
+
+       #[test]
+       fn builds_offer_with_supported_quantity() {
+               let ten = NonZeroU64::new(10).unwrap();
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .supported_quantity(Quantity::one())
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(offer.supported_quantity(), Quantity::one());
+               assert_eq!(tlv_stream.quantity_max, None);
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .supported_quantity(Quantity::Unbounded)
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(offer.supported_quantity(), Quantity::Unbounded);
+               assert_eq!(tlv_stream.quantity_max, Some(0));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .supported_quantity(Quantity::Bounded(ten))
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(offer.supported_quantity(), Quantity::Bounded(ten));
+               assert_eq!(tlv_stream.quantity_max, Some(10));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .supported_quantity(Quantity::Bounded(ten))
+                       .supported_quantity(Quantity::one())
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(offer.supported_quantity(), Quantity::one());
+               assert_eq!(tlv_stream.quantity_max, None);
+       }
+}
index 82a325a9e046bbd714d2b1fc6a53b89c1808d014..8a8d9924f9fb0ce33fcaca28602ff29a8598816d 100644 (file)
@@ -28,31 +28,33 @@ use crate::prelude::*;
 
 /// Onion messages can be sent and received to blinded routes, which serve to hide the identity of
 /// the recipient.
+#[derive(Clone, Debug, PartialEq)]
 pub struct BlindedRoute {
        /// To send to a blinded route, the sender first finds a route to the unblinded
        /// `introduction_node_id`, which can unblind its [`encrypted_payload`] to find out the onion
        /// message's next hop and forward it along.
        ///
        /// [`encrypted_payload`]: BlindedHop::encrypted_payload
-       pub(super) introduction_node_id: PublicKey,
+       pub(crate) introduction_node_id: PublicKey,
        /// Used by the introduction node to decrypt its [`encrypted_payload`] to forward the onion
        /// message.
        ///
        /// [`encrypted_payload`]: BlindedHop::encrypted_payload
-       pub(super) blinding_point: PublicKey,
+       pub(crate) blinding_point: PublicKey,
        /// The hops composing the blinded route.
-       pub(super) blinded_hops: Vec<BlindedHop>,
+       pub(crate) blinded_hops: Vec<BlindedHop>,
 }
 
 /// Used to construct the blinded hops portion of a blinded route. These hops cannot be identified
 /// by outside observers and thus can be used to hide the identity of the recipient.
+#[derive(Clone, Debug, PartialEq)]
 pub struct BlindedHop {
        /// The blinded node id of this hop in a blinded route.
-       pub(super) blinded_node_id: PublicKey,
+       pub(crate) blinded_node_id: PublicKey,
        /// The encrypted payload intended for this hop in a blinded route.
        // The node sending to this blinded route will later encode this payload into the onion packet for
        // this hop.
-       pub(super) encrypted_payload: Vec<u8>,
+       pub(crate) encrypted_payload: Vec<u8>,
 }
 
 impl BlindedRoute {
index 3c522e30c51bf367c25311b6987418f3fe802931..89c1545a7df29c59adfe0956bc9faf1f769e9b80 100644 (file)
@@ -28,6 +28,6 @@ mod utils;
 mod functional_tests;
 
 // Re-export structs so they can be imported with just the `onion_message::` module prefix.
-pub use self::blinded_route::{BlindedRoute, BlindedHop};
+pub use self::blinded_route::{BlindedRoute, BlindedRoute as BlindedPath, BlindedHop};
 pub use self::messenger::{CustomOnionMessageContents, CustomOnionMessageHandler, Destination, OnionMessageContents, OnionMessenger, SendError, SimpleArcOnionMessenger, SimpleRefOnionMessenger};
 pub(crate) use self::packet::Packet;
index 73f58036c865c4c18f12a8cd3dd7191328ebaa87..8fff53642e128682638ec187eab0c8a31afca260 100644 (file)
@@ -164,7 +164,7 @@ impl<T: CustomOnionMessageContents> Writeable for (Payload<T>, [u8; 32]) {
                match &self.0 {
                        Payload::Forward(ForwardControlTlvs::Blinded(encrypted_bytes)) => {
                                encode_varint_length_prefixed_tlv!(w, {
-                                       (4, encrypted_bytes, vec_type)
+                                       (4, *encrypted_bytes, vec_type)
                                })
                        },
                        Payload::Receive {
@@ -172,7 +172,7 @@ impl<T: CustomOnionMessageContents> Writeable for (Payload<T>, [u8; 32]) {
                        } => {
                                encode_varint_length_prefixed_tlv!(w, {
                                        (2, reply_path, option),
-                                       (4, encrypted_bytes, vec_type),
+                                       (4, *encrypted_bytes, vec_type),
                                        (message.tlv_type(), message, required)
                                })
                        },
index 6b0f88d09a26a180431376f4f4a0d3be4204f375..00846a7ba794a1e3b9ba52319d1daf97641097b6 100644 (file)
@@ -31,6 +31,7 @@ use crate::util::ser::{Readable, ReadableArgs, Writeable, Writer, MaybeReadable}
 use crate::util::logger::{Logger, Level};
 use crate::util::events::{Event, EventHandler, MessageSendEvent, MessageSendEventsProvider};
 use crate::util::scid_utils::{block_from_scid, scid_from_parts, MAX_SCID_BLOCK};
+use crate::util::string::PrintableString;
 
 use crate::io;
 use crate::io_extras::{copy, sink};
@@ -1022,23 +1023,17 @@ pub struct NodeAlias(pub [u8; 32]);
 
 impl fmt::Display for NodeAlias {
        fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-               let control_symbol = core::char::REPLACEMENT_CHARACTER;
                let first_null = self.0.iter().position(|b| *b == 0).unwrap_or(self.0.len());
                let bytes = self.0.split_at(first_null).0;
                match core::str::from_utf8(bytes) {
-                       Ok(alias) => {
-                               for c in alias.chars() {
-                                       let mut bytes = [0u8; 4];
-                                       let c = if !c.is_control() { c } else { control_symbol };
-                                       f.write_str(c.encode_utf8(&mut bytes))?;
-                               }
-                       },
+                       Ok(alias) => PrintableString(alias).fmt(f)?,
                        Err(_) => {
+                               use core::fmt::Write;
                                for c in bytes.iter().map(|b| *b as char) {
                                        // Display printable ASCII characters
-                                       let mut bytes = [0u8; 4];
+                                       let control_symbol = core::char::REPLACEMENT_CHARACTER;
                                        let c = if c >= '\x20' && c <= '\x7e' { c } else { control_symbol };
-                                       f.write_str(c.encode_utf8(&mut bytes))?;
+                                       f.write_char(c)?;
                                }
                        },
                };
index 5a0a9d0cc0a7cf584f7509fda22669fb28b9d5df..a9687d28802266dfe0aa89868d1cf9aef61e845d 100644 (file)
@@ -24,7 +24,7 @@ use crate::ln::msgs;
 use crate::ln::msgs::DecodeError;
 use crate::ln::{PaymentPreimage, PaymentHash, PaymentSecret};
 use crate::routing::gossip::NetworkUpdate;
-use crate::util::ser::{BigSize, FixedLengthReader, Writeable, Writer, MaybeReadable, Readable, VecReadWrapper, VecWriteWrapper, OptionDeserWrapper};
+use crate::util::ser::{BigSize, FixedLengthReader, Writeable, Writer, MaybeReadable, Readable, WithoutLength, OptionDeserWrapper};
 use crate::routing::router::{RouteHop, RouteParameters};
 
 use bitcoin::{PackedLockTime, Transaction};
@@ -785,7 +785,7 @@ impl Writeable for Event {
                                        (1, network_update, option),
                                        (2, payment_failed_permanently, required),
                                        (3, all_paths_failed, required),
-                                       (5, path, vec_type),
+                                       (5, *path, vec_type),
                                        (7, short_channel_id, option),
                                        (9, retry, option),
                                        (11, payment_id, option),
@@ -799,7 +799,7 @@ impl Writeable for Event {
                        &Event::SpendableOutputs { ref outputs } => {
                                5u8.write(writer)?;
                                write_tlv_fields!(writer, {
-                                       (0, VecWriteWrapper(outputs), required),
+                                       (0, WithoutLength(outputs), required),
                                });
                        },
                        &Event::PaymentForwarded { fee_earned_msat, prev_channel_id, claim_from_onchain_tx, next_channel_id } => {
@@ -831,7 +831,7 @@ impl Writeable for Event {
                                write_tlv_fields!(writer, {
                                        (0, payment_id, required),
                                        (2, payment_hash, option),
-                                       (4, path, vec_type)
+                                       (4, *path, vec_type)
                                })
                        },
                        &Event::PaymentFailed { ref payment_id, ref payment_hash } => {
@@ -859,7 +859,7 @@ impl Writeable for Event {
                                write_tlv_fields!(writer, {
                                        (0, payment_id, required),
                                        (2, payment_hash, required),
-                                       (4, path, vec_type)
+                                       (4, *path, vec_type)
                                })
                        },
                        &Event::ProbeFailed { ref payment_id, ref payment_hash, ref path, ref short_channel_id } => {
@@ -867,7 +867,7 @@ impl Writeable for Event {
                                write_tlv_fields!(writer, {
                                        (0, payment_id, required),
                                        (2, payment_hash, required),
-                                       (4, path, vec_type),
+                                       (4, *path, vec_type),
                                        (6, short_channel_id, option),
                                })
                        },
@@ -1007,7 +1007,7 @@ impl MaybeReadable for Event {
                        4u8 => Ok(None),
                        5u8 => {
                                let f = || {
-                                       let mut outputs = VecReadWrapper(Vec::new());
+                                       let mut outputs = WithoutLength(Vec::new());
                                        read_tlv_fields!(reader, {
                                                (0, outputs, required),
                                        });
index 9ffe1b7494fae459bdbab3e26a593c35678ee421..730131f224197dc38e5719be8db2336b92089eae 100644 (file)
@@ -21,6 +21,7 @@ pub mod ser;
 pub mod message_signing;
 pub mod invoice;
 pub mod persist;
+pub mod string;
 pub mod wakers;
 
 pub(crate) mod atomic_counter;
index b851cca81aa06150327bcd5b1c45cc2238c54fd9..c3529a6cbd229aa0a35eddd65a0187faf8a10ac6 100644 (file)
@@ -73,7 +73,7 @@ pub(crate) mod fake_scid {
        use core::convert::TryInto;
        use core::ops::Deref;
 
-       const TEST_SEGWIT_ACTIVATION_HEIGHT: u32 = 0;
+       const TEST_SEGWIT_ACTIVATION_HEIGHT: u32 = 1;
        const MAINNET_SEGWIT_ACTIVATION_HEIGHT: u32 = 481_824;
        const MAX_TX_INDEX: u32 = 2_500;
        const MAX_NAMESPACES: u8 = 8; // We allocate 3 bits for the namespace identifier.
@@ -151,12 +151,13 @@ pub(crate) mod fake_scid {
        }
 
        /// Returns whether the given fake scid falls into the given namespace.
-       pub fn is_valid_phantom(fake_scid_rand_bytes: &[u8; 32], scid: u64) -> bool {
+       pub fn is_valid_phantom(fake_scid_rand_bytes: &[u8; 32], scid: u64, genesis_hash: &BlockHash) -> bool {
                let block_height = scid_utils::block_from_scid(&scid);
                let tx_index = scid_utils::tx_index_from_scid(&scid);
                let namespace = Namespace::Phantom;
                let valid_vout = namespace.get_encrypted_vout(block_height, tx_index, fake_scid_rand_bytes);
-               valid_vout == scid_utils::vout_from_scid(&scid) as u8
+               block_height >= segwit_activation_height(genesis_hash)
+                       && valid_vout == scid_utils::vout_from_scid(&scid) as u8
        }
 
        #[cfg(test)]
@@ -194,11 +195,12 @@ pub(crate) mod fake_scid {
                fn test_is_valid_phantom() {
                        let namespace = Namespace::Phantom;
                        let fake_scid_rand_bytes = [0; 32];
+                       let testnet_genesis = genesis_block(Network::Testnet).header.block_hash();
                        let valid_encrypted_vout = namespace.get_encrypted_vout(0, 0, &fake_scid_rand_bytes);
-                       let valid_fake_scid = scid_utils::scid_from_parts(0, 0, valid_encrypted_vout as u64).unwrap();
-                       assert!(is_valid_phantom(&fake_scid_rand_bytes, valid_fake_scid));
-                       let invalid_fake_scid = scid_utils::scid_from_parts(0, 0, 12).unwrap();
-                       assert!(!is_valid_phantom(&fake_scid_rand_bytes, invalid_fake_scid));
+                       let valid_fake_scid = scid_utils::scid_from_parts(1, 0, valid_encrypted_vout as u64).unwrap();
+                       assert!(is_valid_phantom(&fake_scid_rand_bytes, valid_fake_scid, &testnet_genesis));
+                       let invalid_fake_scid = scid_utils::scid_from_parts(1, 0, 12).unwrap();
+                       assert!(!is_valid_phantom(&fake_scid_rand_bytes, invalid_fake_scid, &testnet_genesis));
                }
 
                #[test]
index 5ff6dc86a0bf91d20df6fb110f67185f8cd5f355..7c4ba7fd50f99888745a6944971c10f3f00f77a9 100644 (file)
@@ -22,6 +22,7 @@ use core::ops::Deref;
 use bitcoin::secp256k1::{PublicKey, SecretKey};
 use bitcoin::secp256k1::constants::{PUBLIC_KEY_SIZE, SECRET_KEY_SIZE, COMPACT_SIGNATURE_SIZE};
 use bitcoin::secp256k1::ecdsa::Signature;
+use bitcoin::blockdata::constants::ChainHash;
 use bitcoin::blockdata::script::Script;
 use bitcoin::blockdata::transaction::{OutPoint, Transaction, TxOut};
 use bitcoin::consensus;
@@ -283,39 +284,6 @@ impl<T: Readable> From<T> for OptionDeserWrapper<T> {
        fn from(t: T) -> OptionDeserWrapper<T> { OptionDeserWrapper(Some(t)) }
 }
 
-/// Wrapper to write each element of a Vec with no length prefix
-pub(crate) struct VecWriteWrapper<'a, T: Writeable>(pub &'a Vec<T>);
-impl<'a, T: Writeable> Writeable for VecWriteWrapper<'a, T> {
-       #[inline]
-       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
-               for ref v in self.0.iter() {
-                       v.write(writer)?;
-               }
-               Ok(())
-       }
-}
-
-/// Wrapper to read elements from a given stream until it reaches the end of the stream.
-pub(crate) struct VecReadWrapper<T>(pub Vec<T>);
-impl<T: MaybeReadable> Readable for VecReadWrapper<T> {
-       #[inline]
-       fn read<R: Read>(mut reader: &mut R) -> Result<Self, DecodeError> {
-               let mut values = Vec::new();
-               loop {
-                       let mut track_read = ReadTrackingReader::new(&mut reader);
-                       match MaybeReadable::read(&mut track_read) {
-                               Ok(Some(v)) => { values.push(v); },
-                               Ok(None) => { },
-                               // If we failed to read any bytes at all, we reached the end of our TLV
-                               // stream and have simply exhausted all entries.
-                               Err(ref e) if e == &DecodeError::ShortRead && !track_read.have_read => break,
-                               Err(e) => return Err(e),
-                       }
-               }
-               Ok(Self(values))
-       }
-}
-
 pub(crate) struct U48(pub u64);
 impl Writeable for U48 {
        #[inline]
@@ -451,6 +419,9 @@ macro_rules! impl_writeable_primitive {
                                }
                        }
                }
+               impl From<$val_type> for HighZeroBytesDroppedBigSize<$val_type> {
+                       fn from(val: $val_type) -> Self { Self(val) }
+               }
        }
 }
 
@@ -514,7 +485,7 @@ macro_rules! impl_array {
        );
 }
 
-impl_array!(3); // for rgb
+impl_array!(3); // for rgb, ISO 4712 code
 impl_array!(4); // for IPv4
 impl_array!(12); // for OnionV2
 impl_array!(16); // for IPv6
@@ -546,6 +517,59 @@ impl Readable for [u16; 8] {
        }
 }
 
+/// For variable-length values within TLV record where the length is encoded as part of the record.
+/// Used to prevent encoding the length twice.
+pub(crate) struct WithoutLength<T>(pub T);
+
+impl Writeable for WithoutLength<&String> {
+       #[inline]
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               w.write_all(self.0.as_bytes())
+       }
+}
+impl Readable for WithoutLength<String> {
+       #[inline]
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let v: WithoutLength<Vec<u8>> = Readable::read(r)?;
+               Ok(Self(String::from_utf8(v.0).map_err(|_| DecodeError::InvalidValue)?))
+       }
+}
+impl<'a> From<&'a String> for WithoutLength<&'a String> {
+       fn from(s: &'a String) -> Self { Self(s) }
+}
+
+impl<'a, T: Writeable> Writeable for WithoutLength<&'a Vec<T>> {
+       #[inline]
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
+               for ref v in self.0.iter() {
+                       v.write(writer)?;
+               }
+               Ok(())
+       }
+}
+
+impl<T: MaybeReadable> Readable for WithoutLength<Vec<T>> {
+       #[inline]
+       fn read<R: Read>(mut reader: &mut R) -> Result<Self, DecodeError> {
+               let mut values = Vec::new();
+               loop {
+                       let mut track_read = ReadTrackingReader::new(&mut reader);
+                       match MaybeReadable::read(&mut track_read) {
+                               Ok(Some(v)) => { values.push(v); },
+                               Ok(None) => { },
+                               // If we failed to read any bytes at all, we reached the end of our TLV
+                               // stream and have simply exhausted all entries.
+                               Err(ref e) if e == &DecodeError::ShortRead && !track_read.have_read => break,
+                               Err(e) => return Err(e),
+                       }
+               }
+               Ok(Self(values))
+       }
+}
+impl<'a, T> From<&'a Vec<T>> for WithoutLength<&'a Vec<T>> {
+       fn from(v: &'a Vec<T>) -> Self { Self(v) }
+}
+
 // HashMap
 impl<K, V> Writeable for HashMap<K, V>
        where K: Writeable + Eq + Hash,
@@ -860,6 +884,19 @@ impl Readable for BlockHash {
        }
 }
 
+impl Writeable for ChainHash {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               w.write_all(self.as_bytes())
+       }
+}
+
+impl Readable for ChainHash {
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let buf: [u8; 32] = Readable::read(r)?;
+               Ok(ChainHash::from(&buf[..]))
+       }
+}
+
 impl Writeable for OutPoint {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                self.txid.write(w)?;
index 74b206d3a5774ca8885a7693f2562c05187348be..3ec0848680ffe62a2532e03d47d1fc5b62d7f4b8 100644 (file)
@@ -17,7 +17,7 @@ macro_rules! encode_tlv {
                $field.write($stream)?;
        };
        ($stream: expr, $type: expr, $field: expr, vec_type) => {
-               encode_tlv!($stream, $type, $crate::util::ser::VecWriteWrapper(&$field), required);
+               encode_tlv!($stream, $type, $crate::util::ser::WithoutLength(&$field), required);
        };
        ($stream: expr, $optional_type: expr, $optional_field: expr, option) => {
                if let Some(ref field) = $optional_field {
@@ -26,6 +26,12 @@ macro_rules! encode_tlv {
                        field.write($stream)?;
                }
        };
+       ($stream: expr, $type: expr, $field: expr, (option, encoding: ($fieldty: ty, $encoding: ident))) => {
+               encode_tlv!($stream, $type, $field.map(|f| $encoding(f)), option);
+       };
+       ($stream: expr, $type: expr, $field: expr, (option, encoding: $fieldty: ty)) => {
+               encode_tlv!($stream, $type, $field, option);
+       };
 }
 
 macro_rules! encode_tlv_stream {
@@ -66,7 +72,7 @@ macro_rules! get_varint_length_prefixed_tlv_length {
                $len.0 += field_len;
        };
        ($len: expr, $type: expr, $field: expr, vec_type) => {
-               get_varint_length_prefixed_tlv_length!($len, $type, $crate::util::ser::VecWriteWrapper(&$field), required);
+               get_varint_length_prefixed_tlv_length!($len, $type, $crate::util::ser::WithoutLength(&$field), required);
        };
        ($len: expr, $optional_type: expr, $optional_field: expr, option) => {
                if let Some(ref field) = $optional_field {
@@ -121,6 +127,9 @@ macro_rules! check_tlv_order {
        ($last_seen_type: expr, $typ: expr, $type: expr, $field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {{
                // no-op
        }};
+       ($last_seen_type: expr, $typ: expr, $type: expr, $field: ident, (option, encoding: $encoding: tt)) => {{
+               // no-op
+       }};
 }
 
 macro_rules! check_missing_tlv {
@@ -150,6 +159,9 @@ macro_rules! check_missing_tlv {
        ($last_seen_type: expr, $type: expr, $field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {{
                // no-op
        }};
+       ($last_seen_type: expr, $type: expr, $field: ident, (option, encoding: $encoding: tt)) => {{
+               // no-op
+       }};
 }
 
 macro_rules! decode_tlv {
@@ -160,7 +172,7 @@ macro_rules! decode_tlv {
                $field = $crate::util::ser::Readable::read(&mut $reader)?;
        }};
        ($reader: expr, $field: ident, vec_type) => {{
-               let f: $crate::util::ser::VecReadWrapper<_> = $crate::util::ser::Readable::read(&mut $reader)?;
+               let f: $crate::util::ser::WithoutLength<Vec<_>> = $crate::util::ser::Readable::read(&mut $reader)?;
                $field = Some(f.0);
        }};
        ($reader: expr, $field: ident, option) => {{
@@ -172,6 +184,15 @@ macro_rules! decode_tlv {
        ($reader: expr, $field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {{
                $field = Some($trait::read(&mut $reader $(, $read_arg)*)?);
        }};
+       ($reader: expr, $field: ident, (option, encoding: ($fieldty: ty, $encoding: ident))) => {{
+               $field = {
+                       let field: $encoding<$fieldty> = ser::Readable::read(&mut $reader)?;
+                       Some(field.0)
+               };
+       }};
+       ($reader: expr, $field: ident, (option, encoding: $fieldty: ty)) => {{
+               decode_tlv!($reader, $field, option);
+       }};
 }
 
 // `$decode_custom_tlv` is a closure that may be optionally provided to handle custom message types.
@@ -441,6 +462,75 @@ macro_rules! impl_writeable_tlv_based {
        }
 }
 
+/// Defines a struct for a TLV stream and a similar struct using references for non-primitive types,
+/// implementing [`Readable`] for the former and [`Writeable`] for the latter. Useful as an
+/// intermediary format when reading or writing a type encoded as a TLV stream. Note that each field
+/// representing a TLV record has its type wrapped with an [`Option`]. A tuple consisting of a type
+/// and a serialization wrapper may be given in place of a type when custom serialization is
+/// required.
+///
+/// [`Readable`]: crate::util::ser::Readable
+/// [`Writeable`]: crate::util::ser::Writeable
+macro_rules! tlv_stream {
+       ($name:ident, $nameref:ident, {
+               $(($type:expr, $field:ident : $fieldty:tt)),* $(,)*
+       }) => {
+               #[derive(Debug)]
+               struct $name {
+                       $(
+                               $field: Option<tlv_record_type!($fieldty)>,
+                       )*
+               }
+
+               pub(crate) struct $nameref<'a> {
+                       $(
+                               pub(crate) $field: Option<tlv_record_ref_type!($fieldty)>,
+                       )*
+               }
+
+               impl<'a> $crate::util::ser::Writeable for $nameref<'a> {
+                       fn write<W: $crate::util::ser::Writer>(&self, writer: &mut W) -> Result<(), $crate::io::Error> {
+                               encode_tlv_stream!(writer, {
+                                       $(($type, self.$field, (option, encoding: $fieldty))),*
+                               });
+                               Ok(())
+                       }
+               }
+
+               impl $crate::util::ser::Readable for $name {
+                       fn read<R: $crate::io::Read>(reader: &mut R) -> Result<Self, $crate::ln::msgs::DecodeError> {
+                               $(
+                                       init_tlv_field_var!($field, option);
+                               )*
+                               decode_tlv_stream!(reader, {
+                                       $(($type, $field, (option, encoding: $fieldty))),*
+                               });
+
+                               Ok(Self {
+                                       $(
+                                               $field: $field
+                                       ),*
+                               })
+                       }
+               }
+       }
+}
+
+macro_rules! tlv_record_type {
+       (($type:ty, $wrapper:ident)) => { $type };
+       ($type:ty) => { $type };
+}
+
+macro_rules! tlv_record_ref_type {
+       (char) => { char };
+       (u8) => { u8 };
+       ((u16, $wrapper: ident)) => { u16 };
+       ((u32, $wrapper: ident)) => { u32 };
+       ((u64, $wrapper: ident)) => { u64 };
+       (($type:ty, $wrapper:ident)) => { &'a $type };
+       ($type:ty) => { &'a $type };
+}
+
 macro_rules! _impl_writeable_tlv_based_enum_common {
        ($st: ident, $(($variant_id: expr, $variant_name: ident) =>
                {$(($type: expr, $field: ident, $fieldty: tt)),* $(,)*}
@@ -453,7 +543,7 @@ macro_rules! _impl_writeable_tlv_based_enum_common {
                                                let id: u8 = $variant_id;
                                                id.write(writer)?;
                                                write_tlv_fields!(writer, {
-                                                       $(($type, $field, $fieldty)),*
+                                                       $(($type, *$field, $fieldty)),*
                                                });
                                        }),*
                                        $($st::$tuple_variant_name (ref field) => {
diff --git a/lightning/src/util/string.rs b/lightning/src/util/string.rs
new file mode 100644 (file)
index 0000000..42eb96f
--- /dev/null
@@ -0,0 +1,42 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Utilities for strings.
+
+use core::fmt;
+
+/// A string that displays only printable characters, replacing control characters with
+/// [`core::char::REPLACEMENT_CHARACTER`].
+#[derive(Debug, PartialEq)]
+pub struct PrintableString<'a>(pub &'a str);
+
+impl<'a> fmt::Display for PrintableString<'a> {
+       fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+               use core::fmt::Write;
+               for c in self.0.chars() {
+                       let c = if c.is_control() { core::char::REPLACEMENT_CHARACTER } else { c };
+                       f.write_char(c)?;
+               }
+
+               Ok(())
+       }
+}
+
+#[cfg(test)]
+mod tests {
+       use super::PrintableString;
+
+       #[test]
+       fn displays_printable_string() {
+               assert_eq!(
+                       format!("{}", PrintableString("I \u{1F496} LDK!\t\u{26A1}")),
+                       "I \u{1F496} LDK!\u{FFFD}\u{26A1}",
+               );
+       }
+}