Merge pull request #2199 from tnull/2023-04-fix-async-event-processing
authorMatt Corallo <649246+TheBlueMatt@users.noreply.github.com>
Sat, 22 Apr 2023 01:45:06 +0000 (01:45 +0000)
committerGitHub <noreply@github.com>
Sat, 22 Apr 2023 01:45:06 +0000 (01:45 +0000)
Allow async events processing without holding `total_consistency_lock`

43 files changed:
README.md
ci/ci-tests.sh
fuzz/src/chanmon_consistency.rs
fuzz/src/full_stack.rs
lightning-background-processor/src/lib.rs
lightning-invoice/src/de.rs
lightning-invoice/src/lib.rs
lightning-invoice/src/payment.rs
lightning-invoice/src/ser.rs
lightning-invoice/tests/ser_de.rs
lightning/Cargo.toml
lightning/src/chain/chainmonitor.rs
lightning/src/chain/channelmonitor.rs
lightning/src/chain/keysinterface.rs
lightning/src/chain/onchaintx.rs
lightning/src/chain/package.rs
lightning/src/events/mod.rs
lightning/src/ln/chan_utils.rs
lightning/src/ln/channel.rs
lightning/src/ln/channelmanager.rs
lightning/src/ln/features.rs
lightning/src/ln/functional_test_utils.rs
lightning/src/ln/functional_tests.rs
lightning/src/ln/inbound_payment.rs
lightning/src/ln/monitor_tests.rs
lightning/src/ln/msgs.rs
lightning/src/ln/onion_route_tests.rs
lightning/src/ln/onion_utils.rs
lightning/src/ln/outbound_payment.rs
lightning/src/ln/payment_tests.rs
lightning/src/ln/reorg_tests.rs
lightning/src/offers/invoice.rs
lightning/src/offers/invoice_request.rs
lightning/src/offers/merkle.rs
lightning/src/offers/mod.rs
lightning/src/offers/offer.rs
lightning/src/offers/parse.rs
lightning/src/offers/payer.rs
lightning/src/offers/refund.rs
lightning/src/offers/signer.rs [new file with mode: 0644]
lightning/src/offers/test_utils.rs [new file with mode: 0644]
lightning/src/util/crypto.rs
lightning/src/util/test_utils.rs

index 7f95427a6ab13bfb29d1fe1614e5238fb28b1bcb..a4ab59b53833afd6e672d1d67686a233b98f67ff 100644 (file)
--- a/README.md
+++ b/README.md
@@ -94,7 +94,7 @@ In general, `rust-lightning` does not provide (but LDK has implementations of):
   wanted to run your full Lightning node on a hardware wallet, you could, by
   piping the Lightning network messages over USB/serial and then sending them in
   a TCP socket from another machine.
-* private keys - again we have "default implementations", but users can chose to
+* private keys - again we have "default implementations", but users can choose to
   provide private keys to RL/LDK in any way they wish following a simple API. We
   even support a generic API for signing transactions, allowing users to run
   RL/LDK without any private keys in memory/putting private keys only on
index 37d8e06de1ca408e4ce35e80abe45b96bbca482d..f948cc0d63e925f94b803f2738ed0475f01cbc2b 100755 (executable)
@@ -35,13 +35,14 @@ for DIR in lightning lightning-invoice lightning-rapid-gossip-sync; do
        cargo test --verbose --color always --no-default-features --features no-std
        # check if there is a conflict between no-std and the default std feature
        cargo test --verbose --color always --features no-std
-       # check that things still pass without grind_signatures
-       # note that outbound_commitment_test only runs in this mode, because of hardcoded signature values
-       cargo test --verbose --color always --no-default-features --features std
        # check if there is a conflict between no-std and the c_bindings cfg
        RUSTFLAGS="--cfg=c_bindings" cargo test --verbose --color always --no-default-features --features=no-std
        popd
 done
+# Note that outbound_commitment_test only runs in this mode because of hardcoded signature values
+pushd lightning
+cargo test --verbose --color always --no-default-features --features=std,_test_vectors
+popd
 # This one only works for lightning-invoice
 pushd lightning-invoice
 # check that compile with no-std and serde works in lightning-invoice
index c10b5c0e364ae770467dd03a74e8d1d1fed01c15..a00404497fa1c4146d42abfb20a8cb2474de3220 100644 (file)
@@ -240,6 +240,7 @@ impl SignerProvider for KeyProvider {
                        [id, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, self.node_secret[31]],
                        channel_value_satoshis,
                        channel_keys_id,
+                       channel_keys_id,
                );
                let revoked_commitment = self.make_enforcement_state_cell(keys.commitment_seed);
                EnforcingSigner::new_with_revoked(keys, revoked_commitment, false)
@@ -248,7 +249,7 @@ impl SignerProvider for KeyProvider {
        fn read_chan_signer(&self, buffer: &[u8]) -> Result<Self::Signer, DecodeError> {
                let mut reader = std::io::Cursor::new(buffer);
 
-               let inner: InMemorySigner = Readable::read(&mut reader)?;
+               let inner: InMemorySigner = ReadableArgs::read(&mut reader, self)?;
                let state = self.make_enforcement_state_cell(inner.commitment_seed);
 
                Ok(EnforcingSigner {
index 59cb41dbb451da7c666afa8e0fcb8f1461c28b00..876a412da5fef668c74dd5b215999ece3d2901d1 100644 (file)
@@ -48,7 +48,7 @@ use lightning::util::config::UserConfig;
 use lightning::util::errors::APIError;
 use lightning::util::enforcing_trait_impls::{EnforcingSigner, EnforcementState};
 use lightning::util::logger::Logger;
-use lightning::util::ser::{Readable, Writeable};
+use lightning::util::ser::{Readable, ReadableArgs, Writeable};
 
 use crate::utils::test_logger;
 use crate::utils::test_persister::TestPersister;
@@ -347,6 +347,7 @@ impl SignerProvider for KeyProvider {
                                [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, ctr],
                                channel_value_satoshis,
                                channel_keys_id,
+                               channel_keys_id,
                        )
                } else {
                        InMemorySigner::new(
@@ -359,12 +360,13 @@ impl SignerProvider for KeyProvider {
                                [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, ctr],
                                channel_value_satoshis,
                                channel_keys_id,
+                               channel_keys_id,
                        )
                }, state, false)
        }
 
        fn read_chan_signer(&self, mut data: &[u8]) -> Result<EnforcingSigner, DecodeError> {
-               let inner: InMemorySigner = Readable::read(&mut data)?;
+               let inner: InMemorySigner = ReadableArgs::read(&mut data, self)?;
                let state = Arc::new(Mutex::new(EnforcementState::new()));
 
                Ok(EnforcingSigner::new_with_revoked(
index 7b7a75d60e65c184519b358773f6c920ab7c03a3..b249303b57a8be1d3476f88d1d422102657370a7 100644 (file)
@@ -64,8 +64,8 @@ use alloc::vec::Vec;
 /// * Monitoring whether the [`ChannelManager`] needs to be re-persisted to disk, and if so,
 ///   writing it to disk/backups by invoking the callback given to it at startup.
 ///   [`ChannelManager`] persistence should be done in the background.
-/// * Calling [`ChannelManager::timer_tick_occurred`] and [`PeerManager::timer_tick_occurred`]
-///   at the appropriate intervals.
+/// * Calling [`ChannelManager::timer_tick_occurred`], [`ChainMonitor::rebroadcast_pending_claims`]
+///   and [`PeerManager::timer_tick_occurred`] at the appropriate intervals.
 /// * Calling [`NetworkGraph::remove_stale_channels_and_tracking`] (if a [`GossipSync`] with a
 ///   [`NetworkGraph`] is provided to [`BackgroundProcessor::start`]).
 ///
@@ -116,12 +116,17 @@ const FIRST_NETWORK_PRUNE_TIMER: u64 = 60;
 #[cfg(test)]
 const FIRST_NETWORK_PRUNE_TIMER: u64 = 1;
 
+#[cfg(not(test))]
+const REBROADCAST_TIMER: u64 = 30;
+#[cfg(test)]
+const REBROADCAST_TIMER: u64 = 1;
+
 #[cfg(feature = "futures")]
 /// core::cmp::min is not currently const, so we define a trivial (and equivalent) replacement
 const fn min_u64(a: u64, b: u64) -> u64 { if a < b { a } else { b } }
 #[cfg(feature = "futures")]
 const FASTEST_TIMER: u64 = min_u64(min_u64(FRESHNESS_TIMER, PING_TIMER),
-       min_u64(SCORER_PERSIST_TIMER, FIRST_NETWORK_PRUNE_TIMER));
+       min_u64(SCORER_PERSIST_TIMER, min_u64(FIRST_NETWORK_PRUNE_TIMER, REBROADCAST_TIMER)));
 
 /// Either [`P2PGossipSync`] or [`RapidGossipSync`].
 pub enum GossipSync<
@@ -270,11 +275,14 @@ macro_rules! define_run_body {
        => { {
                log_trace!($logger, "Calling ChannelManager's timer_tick_occurred on startup");
                $channel_manager.timer_tick_occurred();
+               log_trace!($logger, "Rebroadcasting monitor's pending claims on startup");
+               $chain_monitor.rebroadcast_pending_claims();
 
                let mut last_freshness_call = $get_timer(FRESHNESS_TIMER);
                let mut last_ping_call = $get_timer(PING_TIMER);
                let mut last_prune_call = $get_timer(FIRST_NETWORK_PRUNE_TIMER);
                let mut last_scorer_persist_call = $get_timer(SCORER_PERSIST_TIMER);
+               let mut last_rebroadcast_call = $get_timer(REBROADCAST_TIMER);
                let mut have_pruned = false;
 
                loop {
@@ -374,6 +382,12 @@ macro_rules! define_run_body {
                                }
                                last_scorer_persist_call = $get_timer(SCORER_PERSIST_TIMER);
                        }
+
+                       if $timer_elapsed(&mut last_rebroadcast_call, REBROADCAST_TIMER) {
+                               log_trace!($logger, "Rebroadcasting monitor's pending claims");
+                               $chain_monitor.rebroadcast_pending_claims();
+                               last_rebroadcast_call = $get_timer(REBROADCAST_TIMER);
+                       }
                }
 
                // After we exit, ensure we persist the ChannelManager one final time - this avoids
@@ -1194,8 +1208,9 @@ mod tests {
 
        #[test]
        fn test_timer_tick_called() {
-               // Test that ChannelManager's and PeerManager's `timer_tick_occurred` is called every
-               // `FRESHNESS_TIMER`.
+               // Test that `ChannelManager::timer_tick_occurred` is called every `FRESHNESS_TIMER`,
+               // `ChainMonitor::rebroadcast_pending_claims` is called every `REBROADCAST_TIMER`, and
+               // `PeerManager::timer_tick_occurred` every `PING_TIMER`.
                let nodes = create_nodes(1, "test_timer_tick_called".to_string());
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir));
@@ -1203,10 +1218,12 @@ mod tests {
                let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
                loop {
                        let log_entries = nodes[0].logger.lines.lock().unwrap();
-                       let desired_log = "Calling ChannelManager's timer_tick_occurred".to_string();
-                       let second_desired_log = "Calling PeerManager's timer_tick_occurred".to_string();
-                       if log_entries.get(&("lightning_background_processor".to_string(), desired_log)).is_some() &&
-                                       log_entries.get(&("lightning_background_processor".to_string(), second_desired_log)).is_some() {
+                       let desired_log_1 = "Calling ChannelManager's timer_tick_occurred".to_string();
+                       let desired_log_2 = "Calling PeerManager's timer_tick_occurred".to_string();
+                       let desired_log_3 = "Rebroadcasting monitor's pending claims".to_string();
+                       if log_entries.get(&("lightning_background_processor".to_string(), desired_log_1)).is_some() &&
+                               log_entries.get(&("lightning_background_processor".to_string(), desired_log_2)).is_some() &&
+                               log_entries.get(&("lightning_background_processor".to_string(), desired_log_3)).is_some() {
                                break
                        }
                }
index 925d7265c553526d993f94b29354adcac1db9ac1..01adf67d1af0816bcc78ddeeacd6473f680b71bf 100644 (file)
@@ -460,6 +460,8 @@ impl FromBase32 for TaggedField {
                                Ok(TaggedField::PrivateRoute(PrivateRoute::from_base32(field_data)?)),
                        constants::TAG_PAYMENT_SECRET =>
                                Ok(TaggedField::PaymentSecret(PaymentSecret::from_base32(field_data)?)),
+                       constants::TAG_PAYMENT_METADATA =>
+                               Ok(TaggedField::PaymentMetadata(Vec::<u8>::from_base32(field_data)?)),
                        constants::TAG_FEATURES =>
                                Ok(TaggedField::Features(InvoiceFeatures::from_base32(field_data)?)),
                        _ => {
index 94a93fe62b7f1d160c5f1dc0c89ba66b56bf34c1..0923fcc1756d4fd05305ab53b2653d08539949f9 100644 (file)
@@ -218,10 +218,13 @@ pub const DEFAULT_MIN_FINAL_CLTV_EXPIRY_DELTA: u64 = 18;
 ///  * `D`: exactly one [`TaggedField::Description`] or [`TaggedField::DescriptionHash`]
 ///  * `H`: exactly one [`TaggedField::PaymentHash`]
 ///  * `T`: the timestamp is set
+///  * `C`: the CLTV expiry is set
+///  * `S`: the payment secret is set
+///  * `M`: payment metadata is set
 ///
 /// This is not exported to bindings users as we likely need to manually select one set of boolean type parameters.
 #[derive(Eq, PartialEq, Debug, Clone)]
-pub struct InvoiceBuilder<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> {
+pub struct InvoiceBuilder<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool, M: tb::Bool> {
        currency: Currency,
        amount: Option<u64>,
        si_prefix: Option<SiPrefix>,
@@ -234,6 +237,7 @@ pub struct InvoiceBuilder<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S:
        phantom_t: core::marker::PhantomData<T>,
        phantom_c: core::marker::PhantomData<C>,
        phantom_s: core::marker::PhantomData<S>,
+       phantom_m: core::marker::PhantomData<M>,
 }
 
 /// Represents a syntactically and semantically correct lightning BOLT11 invoice.
@@ -442,6 +446,7 @@ pub enum TaggedField {
        Fallback(Fallback),
        PrivateRoute(PrivateRoute),
        PaymentSecret(PaymentSecret),
+       PaymentMetadata(Vec<u8>),
        Features(InvoiceFeatures),
 }
 
@@ -506,15 +511,16 @@ pub mod constants {
        pub const TAG_FALLBACK: u8 = 9;
        pub const TAG_PRIVATE_ROUTE: u8 = 3;
        pub const TAG_PAYMENT_SECRET: u8 = 16;
+       pub const TAG_PAYMENT_METADATA: u8 = 27;
        pub const TAG_FEATURES: u8 = 5;
 }
 
-impl InvoiceBuilder<tb::False, tb::False, tb::False, tb::False, tb::False> {
+impl InvoiceBuilder<tb::False, tb::False, tb::False, tb::False, tb::False, tb::False> {
        /// Construct new, empty `InvoiceBuilder`. All necessary fields have to be filled first before
        /// `InvoiceBuilder::build(self)` becomes available.
-       pub fn new(currrency: Currency) -> Self {
+       pub fn new(currency: Currency) -> Self {
                InvoiceBuilder {
-                       currency: currrency,
+                       currency,
                        amount: None,
                        si_prefix: None,
                        timestamp: None,
@@ -526,14 +532,15 @@ impl InvoiceBuilder<tb::False, tb::False, tb::False, tb::False, tb::False> {
                        phantom_t: core::marker::PhantomData,
                        phantom_c: core::marker::PhantomData,
                        phantom_s: core::marker::PhantomData,
+                       phantom_m: core::marker::PhantomData,
                }
        }
 }
 
-impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, T, C, S> {
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool, M: tb::Bool> InvoiceBuilder<D, H, T, C, S, M> {
        /// Helper function to set the completeness flags.
-       fn set_flags<DN: tb::Bool, HN: tb::Bool, TN: tb::Bool, CN: tb::Bool, SN: tb::Bool>(self) -> InvoiceBuilder<DN, HN, TN, CN, SN> {
-               InvoiceBuilder::<DN, HN, TN, CN, SN> {
+       fn set_flags<DN: tb::Bool, HN: tb::Bool, TN: tb::Bool, CN: tb::Bool, SN: tb::Bool, MN: tb::Bool>(self) -> InvoiceBuilder<DN, HN, TN, CN, SN, MN> {
+               InvoiceBuilder::<DN, HN, TN, CN, SN, MN> {
                        currency: self.currency,
                        amount: self.amount,
                        si_prefix: self.si_prefix,
@@ -546,6 +553,7 @@ impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBui
                        phantom_t: core::marker::PhantomData,
                        phantom_c: core::marker::PhantomData,
                        phantom_s: core::marker::PhantomData,
+                       phantom_m: core::marker::PhantomData,
                }
        }
 
@@ -590,7 +598,7 @@ impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBui
        }
 }
 
-impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, tb::True, C, S> {
+impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool, M: tb::Bool> InvoiceBuilder<D, H, tb::True, C, S, M> {
        /// Builds a [`RawInvoice`] if no [`CreationError`] occurred while construction any of the
        /// fields.
        pub fn build_raw(self) -> Result<RawInvoice, CreationError> {
@@ -624,9 +632,9 @@ impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, tb
        }
 }
 
-impl<H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<tb::False, H, T, C, S> {
+impl<H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool, M: tb::Bool> InvoiceBuilder<tb::False, H, T, C, S, M> {
        /// Set the description. This function is only available if no description (hash) was set.
-       pub fn description(mut self, description: String) -> InvoiceBuilder<tb::True, H, T, C, S> {
+       pub fn description(mut self, description: String) -> InvoiceBuilder<tb::True, H, T, C, S, M> {
                match Description::new(description) {
                        Ok(d) => self.tagged_fields.push(TaggedField::Description(d)),
                        Err(e) => self.error = Some(e),
@@ -635,13 +643,13 @@ impl<H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<tb::Fals
        }
 
        /// Set the description hash. This function is only available if no description (hash) was set.
-       pub fn description_hash(mut self, description_hash: sha256::Hash) -> InvoiceBuilder<tb::True, H, T, C, S> {
+       pub fn description_hash(mut self, description_hash: sha256::Hash) -> InvoiceBuilder<tb::True, H, T, C, S, M> {
                self.tagged_fields.push(TaggedField::DescriptionHash(Sha256(description_hash)));
                self.set_flags()
        }
 
        /// Set the description or description hash. This function is only available if no description (hash) was set.
-       pub fn invoice_description(self, description: InvoiceDescription) -> InvoiceBuilder<tb::True, H, T, C, S> {
+       pub fn invoice_description(self, description: InvoiceDescription) -> InvoiceBuilder<tb::True, H, T, C, S, M> {
                match description {
                        InvoiceDescription::Direct(desc) => {
                                self.description(desc.clone().into_inner())
@@ -653,18 +661,18 @@ impl<H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<tb::Fals
        }
 }
 
-impl<D: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, tb::False, T, C, S> {
+impl<D: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool, M: tb::Bool> InvoiceBuilder<D, tb::False, T, C, S, M> {
        /// Set the payment hash. This function is only available if no payment hash was set.
-       pub fn payment_hash(mut self, hash: sha256::Hash) -> InvoiceBuilder<D, tb::True, T, C, S> {
+       pub fn payment_hash(mut self, hash: sha256::Hash) -> InvoiceBuilder<D, tb::True, T, C, S, M> {
                self.tagged_fields.push(TaggedField::PaymentHash(Sha256(hash)));
                self.set_flags()
        }
 }
 
-impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, tb::False, C, S> {
+impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool, M: tb::Bool> InvoiceBuilder<D, H, tb::False, C, S, M> {
        /// Sets the timestamp to a specific [`SystemTime`].
        #[cfg(feature = "std")]
-       pub fn timestamp(mut self, time: SystemTime) -> InvoiceBuilder<D, H, tb::True, C, S> {
+       pub fn timestamp(mut self, time: SystemTime) -> InvoiceBuilder<D, H, tb::True, C, S, M> {
                match PositiveTimestamp::from_system_time(time) {
                        Ok(t) => self.timestamp = Some(t),
                        Err(e) => self.error = Some(e),
@@ -675,7 +683,7 @@ impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, tb
 
        /// Sets the timestamp to a duration since the Unix epoch, dropping the subsecond part (which
        /// is not representable in BOLT 11 invoices).
-       pub fn duration_since_epoch(mut self, time: Duration) -> InvoiceBuilder<D, H, tb::True, C, S> {
+       pub fn duration_since_epoch(mut self, time: Duration) -> InvoiceBuilder<D, H, tb::True, C, S, M> {
                match PositiveTimestamp::from_duration_since_epoch(time) {
                        Ok(t) => self.timestamp = Some(t),
                        Err(e) => self.error = Some(e),
@@ -686,34 +694,82 @@ impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, tb
 
        /// Sets the timestamp to the current system time.
        #[cfg(feature = "std")]
-       pub fn current_timestamp(mut self) -> InvoiceBuilder<D, H, tb::True, C, S> {
+       pub fn current_timestamp(mut self) -> InvoiceBuilder<D, H, tb::True, C, S, M> {
                let now = PositiveTimestamp::from_system_time(SystemTime::now());
                self.timestamp = Some(now.expect("for the foreseeable future this shouldn't happen"));
                self.set_flags()
        }
 }
 
-impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, T, tb::False, S> {
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, S: tb::Bool, M: tb::Bool> InvoiceBuilder<D, H, T, tb::False, S, M> {
        /// Sets `min_final_cltv_expiry_delta`.
-       pub fn min_final_cltv_expiry_delta(mut self, min_final_cltv_expiry_delta: u64) -> InvoiceBuilder<D, H, T, tb::True, S> {
+       pub fn min_final_cltv_expiry_delta(mut self, min_final_cltv_expiry_delta: u64) -> InvoiceBuilder<D, H, T, tb::True, S, M> {
                self.tagged_fields.push(TaggedField::MinFinalCltvExpiryDelta(MinFinalCltvExpiryDelta(min_final_cltv_expiry_delta)));
                self.set_flags()
        }
 }
 
-impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool> InvoiceBuilder<D, H, T, C, tb::False> {
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, M: tb::Bool> InvoiceBuilder<D, H, T, C, tb::False, M> {
        /// Sets the payment secret and relevant features.
-       pub fn payment_secret(mut self, payment_secret: PaymentSecret) -> InvoiceBuilder<D, H, T, C, tb::True> {
-               let mut features = InvoiceFeatures::empty();
-               features.set_variable_length_onion_required();
-               features.set_payment_secret_required();
+       pub fn payment_secret(mut self, payment_secret: PaymentSecret) -> InvoiceBuilder<D, H, T, C, tb::True, M> {
+               let mut found_features = false;
+               for field in self.tagged_fields.iter_mut() {
+                       if let TaggedField::Features(f) = field {
+                               found_features = true;
+                               f.set_variable_length_onion_required();
+                               f.set_payment_secret_required();
+                       }
+               }
                self.tagged_fields.push(TaggedField::PaymentSecret(payment_secret));
-               self.tagged_fields.push(TaggedField::Features(features));
+               if !found_features {
+                       let mut features = InvoiceFeatures::empty();
+                       features.set_variable_length_onion_required();
+                       features.set_payment_secret_required();
+                       self.tagged_fields.push(TaggedField::Features(features));
+               }
                self.set_flags()
        }
 }
 
-impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool> InvoiceBuilder<D, H, T, C, tb::True> {
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, T, C, S, tb::False> {
+       /// Sets the payment metadata.
+       ///
+       /// By default features are set to *optionally* allow the sender to include the payment metadata.
+       /// If you wish to require that the sender include the metadata (and fail to parse the invoice if
+       /// they don't support payment metadata fields), you need to call
+       /// [`InvoiceBuilder::require_payment_metadata`] after this.
+       pub fn payment_metadata(mut self, payment_metadata: Vec<u8>) -> InvoiceBuilder<D, H, T, C, S, tb::True> {
+               self.tagged_fields.push(TaggedField::PaymentMetadata(payment_metadata));
+               let mut found_features = false;
+               for field in self.tagged_fields.iter_mut() {
+                       if let TaggedField::Features(f) = field {
+                               found_features = true;
+                               f.set_payment_metadata_optional();
+                       }
+               }
+               if !found_features {
+                       let mut features = InvoiceFeatures::empty();
+                       features.set_payment_metadata_optional();
+                       self.tagged_fields.push(TaggedField::Features(features));
+               }
+               self.set_flags()
+       }
+}
+
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, T, C, S, tb::True> {
+       /// Sets forwarding of payment metadata as required. A reader of the invoice which does not
+       /// support sending payment metadata will fail to read the invoice.
+       pub fn require_payment_metadata(mut self) -> InvoiceBuilder<D, H, T, C, S, tb::True> {
+               for field in self.tagged_fields.iter_mut() {
+                       if let TaggedField::Features(f) = field {
+                               f.set_payment_metadata_required();
+                       }
+               }
+               self
+       }
+}
+
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, M: tb::Bool> InvoiceBuilder<D, H, T, C, tb::True, M> {
        /// Sets the `basic_mpp` feature as optional.
        pub fn basic_mpp(mut self) -> Self {
                for field in self.tagged_fields.iter_mut() {
@@ -725,7 +781,7 @@ impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool> InvoiceBuilder<D, H, T,
        }
 }
 
-impl InvoiceBuilder<tb::True, tb::True, tb::True, tb::True, tb::True> {
+impl<M: tb::Bool> InvoiceBuilder<tb::True, tb::True, tb::True, tb::True, tb::True, M> {
        /// Builds and signs an invoice using the supplied `sign_function`. This function MAY NOT fail
        /// and MUST produce a recoverable signature valid for the given hash and if applicable also for
        /// the included payee public key.
@@ -977,6 +1033,10 @@ impl RawInvoice {
                find_extract!(self.known_tagged_fields(), TaggedField::PaymentSecret(ref x), x)
        }
 
+       pub fn payment_metadata(&self) -> Option<&Vec<u8>> {
+               find_extract!(self.known_tagged_fields(), TaggedField::PaymentMetadata(ref x), x)
+       }
+
        pub fn features(&self) -> Option<&InvoiceFeatures> {
                find_extract!(self.known_tagged_fields(), TaggedField::Features(ref x), x)
        }
@@ -1248,6 +1308,11 @@ impl Invoice {
                self.signed_invoice.payment_secret().expect("was checked by constructor")
        }
 
+       /// Get the payment metadata blob if one was included in the invoice
+       pub fn payment_metadata(&self) -> Option<&Vec<u8>> {
+               self.signed_invoice.payment_metadata()
+       }
+
        /// Get the invoice features if they were included in the invoice
        pub fn features(&self) -> Option<&InvoiceFeatures> {
                self.signed_invoice.features()
@@ -1396,6 +1461,7 @@ impl TaggedField {
                        TaggedField::Fallback(_) => constants::TAG_FALLBACK,
                        TaggedField::PrivateRoute(_) => constants::TAG_PRIVATE_ROUTE,
                        TaggedField::PaymentSecret(_) => constants::TAG_PAYMENT_SECRET,
+                       TaggedField::PaymentMetadata(_) => constants::TAG_PAYMENT_METADATA,
                        TaggedField::Features(_) => constants::TAG_FEATURES,
                };
 
index 00b17db7899d60e235d95057d45b3e97886eadf7..11757be2e3a2f8399661da3b9b81d51f8457d6a6 100644 (file)
@@ -145,8 +145,10 @@ fn pay_invoice_using_amount<P: Deref>(
        payer: P
 ) -> Result<(), PaymentError> where P::Target: Payer {
        let payment_hash = PaymentHash((*invoice.payment_hash()).into_inner());
-       let payment_secret = Some(*invoice.payment_secret());
-       let recipient_onion = RecipientOnionFields { payment_secret };
+       let recipient_onion = RecipientOnionFields {
+               payment_secret: Some(*invoice.payment_secret()),
+               payment_metadata: invoice.payment_metadata().map(|v| v.clone()),
+       };
        let mut payment_params = PaymentParameters::from_node_id(invoice.recover_payee_pub_key(),
                invoice.min_final_cltv_expiry_delta() as u32)
                .with_expiry_time(expiry_time_from_unix_epoch(invoice).as_secs())
@@ -213,6 +215,8 @@ mod tests {
        use super::*;
        use crate::{InvoiceBuilder, Currency};
        use bitcoin_hashes::sha256::Hash as Sha256;
+       use lightning::events::Event;
+       use lightning::ln::msgs::ChannelMessageHandler;
        use lightning::ln::{PaymentPreimage, PaymentSecret};
        use lightning::ln::functional_test_utils::*;
        use secp256k1::{SecretKey, Secp256k1};
@@ -350,4 +354,52 @@ mod tests {
                        _ => panic!()
                }
        }
+
+       #[test]
+       #[cfg(feature = "std")]
+       fn payment_metadata_end_to_end() {
+               // Test that a payment metadata read from an invoice passed to `pay_invoice` makes it all
+               // the way out through the `PaymentClaimable` event.
+               let chanmon_cfgs = create_chanmon_cfgs(2);
+               let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+               let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+               let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+               create_announced_chan_between_nodes(&nodes, 0, 1);
+
+               let payment_metadata = vec![42, 43, 44, 45, 46, 47, 48, 49, 42];
+
+               let (payment_hash, payment_secret) =
+                       nodes[1].node.create_inbound_payment(None, 7200, None).unwrap();
+
+               let invoice = InvoiceBuilder::new(Currency::Bitcoin)
+                       .description("test".into())
+                       .payment_hash(Sha256::from_slice(&payment_hash.0).unwrap())
+                       .payment_secret(payment_secret)
+                       .current_timestamp()
+                       .min_final_cltv_expiry_delta(144)
+                       .amount_milli_satoshis(50_000)
+                       .payment_metadata(payment_metadata.clone())
+                       .build_signed(|hash| {
+                               Secp256k1::new().sign_ecdsa_recoverable(hash,
+                                       &nodes[1].keys_manager.backing.get_node_secret_key())
+                       })
+                       .unwrap();
+
+               pay_invoice(&invoice, Retry::Attempts(0), nodes[0].node).unwrap();
+               check_added_monitors(&nodes[0], 1);
+               let send_event = SendEvent::from_node(&nodes[0]);
+               nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
+               commitment_signed_dance!(nodes[1], nodes[0], &send_event.commitment_msg, false);
+
+               expect_pending_htlcs_forwardable!(nodes[1]);
+
+               let mut events = nodes[1].node.get_and_clear_pending_events();
+               assert_eq!(events.len(), 1);
+               match events.pop().unwrap() {
+                       Event::PaymentClaimable { onion_fields, .. } => {
+                               assert_eq!(Some(payment_metadata), onion_fields.unwrap().payment_metadata);
+                       },
+                       _ => panic!("Unexpected event")
+               }
+       }
 }
index 29a2ca074edf0185c7d3ac97d74baa087dd1af29..0dca180cab370a79e9358b89d7f1e1d2e51a8c95 100644 (file)
@@ -446,6 +446,9 @@ impl ToBase32 for TaggedField {
                        TaggedField::PaymentSecret(ref payment_secret) => {
                                  write_tagged_field(writer, constants::TAG_PAYMENT_SECRET, payment_secret)
                        },
+                       TaggedField::PaymentMetadata(ref payment_metadata) => {
+                                 write_tagged_field(writer, constants::TAG_PAYMENT_METADATA, payment_metadata)
+                       },
                        TaggedField::Features(ref features) => {
                                write_tagged_field(writer, constants::TAG_FEATURES, features)
                        },
index 73e2ea8384a1190680c913cf50a5336317f6f171..ef5a4d32b30a0caa7c397217fb808a64ca2be877 100644 (file)
@@ -332,6 +332,56 @@ fn get_test_tuples() -> Vec<(String, SignedRawInvoice, bool, bool)> {
                        true, // Different features than set in InvoiceBuilder
                        true, // Some unknown fields
                ),
+               ( // Older version of the payment metadata test with a payment_pubkey set
+                       "lnbc10m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdp9wpshjmt9de6zqmt9w3skgct5vysxjmnnd9jx2mq8q8a04uqnp4q0n326hr8v9zprg8gsvezcch06gfaqqhde2aj730yg0durunfhv66sp5zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs9q2gqqqqqqsgqy9gw6ymamd20jumvdgpfphkhp8fzhhdhycw36egcmla5vlrtrmhs9t7psfy3hkkdqzm9eq64fjg558znccds5nhsfmxveha5xe0dykgpspdha0".to_owned(),
+                       InvoiceBuilder::new(Currency::Bitcoin)
+                               .amount_milli_satoshis(1_000_000_000)
+                               .duration_since_epoch(Duration::from_secs(1496314658))
+                               .payment_hash(sha256::Hash::from_hex(
+                                       "0001020304050607080900010203040506070809000102030405060708090102"
+                               ).unwrap())
+                               .description("payment metadata inside".to_owned())
+                               .payment_metadata(hex::decode("01fafaf0").unwrap())
+                               .require_payment_metadata()
+                               .payee_pub_key(PublicKey::from_slice(&hex::decode(
+                                               "03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad"
+                                       ).unwrap()).unwrap())
+                               .payment_secret(PaymentSecret([0x11; 32]))
+                               .build_raw()
+                               .unwrap()
+                               .sign(|_| {
+                                       RecoverableSignature::from_compact(
+                                               &hex::decode("2150ed137ddb54f9736c6a0290ded709d22bddb7261d1d6518dffb467c6b1eef02afc182491bdacd00b65c83554c914a1c53c61b0a4ef04eccccdfb4365ed259").unwrap(),
+                                               RecoveryId::from_i32(1).unwrap()
+                                       )
+                               }).unwrap(),
+                       false, // Different features than set in InvoiceBuilder
+                       true, // Some unknown fields
+               ),
+               (
+                       "lnbc10m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdp9wpshjmt9de6zqmt9w3skgct5vysxjmnnd9jx2mq8q8a04uqsp5zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs9q2gqqqqqqsgq7hf8he7ecf7n4ffphs6awl9t6676rrclv9ckg3d3ncn7fct63p6s365duk5wrk202cfy3aj5xnnp5gs3vrdvruverwwq7yzhkf5a3xqpd05wjc".to_owned(),
+                       InvoiceBuilder::new(Currency::Bitcoin)
+                               .amount_milli_satoshis(1_000_000_000)
+                               .duration_since_epoch(Duration::from_secs(1496314658))
+                               .payment_hash(sha256::Hash::from_hex(
+                                       "0001020304050607080900010203040506070809000102030405060708090102"
+                               ).unwrap())
+                               .description("payment metadata inside".to_owned())
+                               .payment_metadata(hex::decode("01fafaf0").unwrap())
+                               .require_payment_metadata()
+                               .payment_secret(PaymentSecret([0x11; 32]))
+                               .build_raw()
+                               .unwrap()
+                               .sign(|_| {
+                                       RecoverableSignature::from_compact(
+                                               &hex::decode("f5d27be7d9c27d3aa521bc35d77cabd6bda18f1f61716445b19e27e4e17a887508ea8de5a8e1d94f561248f65434e61a221160dac1f1991b9c0f1057b269d898").unwrap(),
+                                               RecoveryId::from_i32(1).unwrap()
+                                       )
+                               }).unwrap(),
+                       false, // Different features than set in InvoiceBuilder
+                       true, // Some unknown fields
+               ),
+
        ]
 }
 
index 17896ecb387a1756ed13c8adc5dc58d86f8bdba5..32755a7e43fe76beb3114182c66bbb5f1353391c 100644 (file)
@@ -29,6 +29,8 @@ max_level_trace = []
 # This is unsafe to use in production because it may result in the counterparty publishing taking our funds.
 unsafe_revoked_tx_signing = []
 _bench_unstable = []
+# Override signing to not include randomness when generating signatures for test vectors.
+_test_vectors = []
 
 no-std = ["hashbrown", "bitcoin/no-std", "core2/alloc"]
 std = ["bitcoin/std"]
index fdaa25f69b8ea7df39dc6b246d2f062095071019..e7c2b0f18ec001d703eb0249964b90a8a3f1a5ed 100644 (file)
@@ -217,8 +217,15 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> Deref for LockedChannelMonitor<
 /// or used independently to monitor channels remotely. See the [module-level documentation] for
 /// details.
 ///
+/// Note that `ChainMonitor` should regularly trigger rebroadcasts/fee bumps of pending claims from
+/// a force-closed channel. This is crucial in preventing certain classes of pinning attacks,
+/// detecting substantial mempool feerate changes between blocks, and ensuring reliability if
+/// broadcasting fails. We recommend invoking this every 30 seconds, or lower if running in an
+/// environment with spotty connections, like on mobile.
+///
 /// [`ChannelManager`]: crate::ln::channelmanager::ChannelManager
 /// [module-level documentation]: crate::chain::chainmonitor
+/// [`rebroadcast_pending_claims`]: Self::rebroadcast_pending_claims
 pub struct ChainMonitor<ChannelSigner: WriteableEcdsaChannelSigner, C: Deref, T: Deref, F: Deref, L: Deref, P: Deref>
        where C::Target: chain::Filter,
         T::Target: BroadcasterInterface,
@@ -533,6 +540,20 @@ where C::Target: chain::Filter,
        pub fn get_update_future(&self) -> Future {
                self.event_notifier.get_future()
        }
+
+       /// Triggers rebroadcasts/fee-bumps of pending claims from a force-closed channel. This is
+       /// crucial in preventing certain classes of pinning attacks, detecting substantial mempool
+       /// feerate changes between blocks, and ensuring reliability if broadcasting fails. We recommend
+       /// invoking this every 30 seconds, or lower if running in an environment with spotty
+       /// connections, like on mobile.
+       pub fn rebroadcast_pending_claims(&self) {
+               let monitors = self.monitors.read().unwrap();
+               for (_, monitor_holder) in &*monitors {
+                       monitor_holder.monitor.rebroadcast_pending_claims(
+                               &*self.broadcaster, &*self.fee_estimator, &*self.logger
+                       )
+               }
+       }
 }
 
 impl<ChannelSigner: WriteableEcdsaChannelSigner, C: Deref, T: Deref, F: Deref, L: Deref, P: Deref>
index 82fae49f7ce5374b33f80ec9b556d99f1dcb8eb4..cb2183f8509b78bf1b878066ff3289018665d891 100644 (file)
@@ -1467,6 +1467,27 @@ impl<Signer: WriteableEcdsaChannelSigner> ChannelMonitor<Signer> {
        pub fn current_best_block(&self) -> BestBlock {
                self.inner.lock().unwrap().best_block.clone()
        }
+
+       /// Triggers rebroadcasts/fee-bumps of pending claims from a force-closed channel. This is
+       /// crucial in preventing certain classes of pinning attacks, detecting substantial mempool
+       /// feerate changes between blocks, and ensuring reliability if broadcasting fails. We recommend
+       /// invoking this every 30 seconds, or lower if running in an environment with spotty
+       /// connections, like on mobile.
+       pub fn rebroadcast_pending_claims<B: Deref, F: Deref, L: Deref>(
+               &self, broadcaster: B, fee_estimator: F, logger: L,
+       )
+       where
+               B::Target: BroadcasterInterface,
+               F::Target: FeeEstimator,
+               L::Target: Logger,
+       {
+               let fee_estimator = LowerBoundedFeeEstimator::new(fee_estimator);
+               let mut inner = self.inner.lock().unwrap();
+               let current_height = inner.best_block.height;
+               inner.onchain_tx_handler.rebroadcast_pending_claims(
+                       current_height, &broadcaster, &fee_estimator, &logger,
+               );
+       }
 }
 
 impl<Signer: WriteableEcdsaChannelSigner> ChannelMonitorImpl<Signer> {
@@ -4235,6 +4256,7 @@ mod tests {
                        [41; 32],
                        0,
                        [0; 32],
+                       [0; 32],
                );
 
                let counterparty_pubkeys = ChannelPublicKeys {
index 4ad15db876bb4cad83106c89ba7ba835409b1544..b1290708b95f70a71ebf6193cceb1677aa3150bd 100644 (file)
@@ -32,8 +32,8 @@ use bitcoin::secp256k1::ecdsa::RecoverableSignature;
 use bitcoin::{PackedLockTime, secp256k1, Sequence, Witness};
 
 use crate::util::transaction_utils;
-use crate::util::crypto::{hkdf_extract_expand_twice, sign};
-use crate::util::ser::{Writeable, Writer, Readable};
+use crate::util::crypto::{hkdf_extract_expand_twice, sign, sign_with_aux_rand};
+use crate::util::ser::{Writeable, Writer, Readable, ReadableArgs};
 use crate::chain::transaction::OutPoint;
 #[cfg(anchors)]
 use crate::events::bump_transaction::HTLCDescriptor;
@@ -45,6 +45,7 @@ use crate::ln::script::ShutdownScript;
 
 use crate::prelude::*;
 use core::convert::TryInto;
+use core::ops::Deref;
 use core::sync::atomic::{AtomicUsize, Ordering};
 use crate::io::{self, Error};
 use crate::ln::msgs::{DecodeError, MAX_VALUE_MSAT};
@@ -553,7 +554,6 @@ pub trait SignerProvider {
        fn get_shutdown_scriptpubkey(&self) -> ShutdownScript;
 }
 
-#[derive(Clone)]
 /// A simple implementation of [`WriteableEcdsaChannelSigner`] that just keeps the private keys in memory.
 ///
 /// This implementation performs no policy checks and is insufficient by itself as
@@ -580,6 +580,30 @@ pub struct InMemorySigner {
        channel_value_satoshis: u64,
        /// Key derivation parameters.
        channel_keys_id: [u8; 32],
+       /// Seed from which all randomness produced is derived from.
+       rand_bytes_unique_start: [u8; 32],
+       /// Tracks the number of times we've produced randomness to ensure we don't return the same
+       /// bytes twice.
+       rand_bytes_index: AtomicCounter,
+}
+
+impl Clone for InMemorySigner {
+       fn clone(&self) -> Self {
+               Self {
+                       funding_key: self.funding_key.clone(),
+                       revocation_base_key: self.revocation_base_key.clone(),
+                       payment_key: self.payment_key.clone(),
+                       delayed_payment_base_key: self.delayed_payment_base_key.clone(),
+                       htlc_base_key: self.htlc_base_key.clone(),
+                       commitment_seed: self.commitment_seed.clone(),
+                       holder_channel_pubkeys: self.holder_channel_pubkeys.clone(),
+                       channel_parameters: self.channel_parameters.clone(),
+                       channel_value_satoshis: self.channel_value_satoshis,
+                       channel_keys_id: self.channel_keys_id,
+                       rand_bytes_unique_start: self.get_secure_random_bytes(),
+                       rand_bytes_index: AtomicCounter::new(),
+               }
+       }
 }
 
 impl InMemorySigner {
@@ -594,6 +618,7 @@ impl InMemorySigner {
                commitment_seed: [u8; 32],
                channel_value_satoshis: u64,
                channel_keys_id: [u8; 32],
+               rand_bytes_unique_start: [u8; 32],
        ) -> InMemorySigner {
                let holder_channel_pubkeys =
                        InMemorySigner::make_holder_keys(secp_ctx, &funding_key, &revocation_base_key,
@@ -610,6 +635,8 @@ impl InMemorySigner {
                        holder_channel_pubkeys,
                        channel_parameters: None,
                        channel_keys_id,
+                       rand_bytes_unique_start,
+                       rand_bytes_index: AtomicCounter::new(),
                }
        }
 
@@ -686,7 +713,7 @@ impl InMemorySigner {
                let remotepubkey = self.pubkeys().payment_point;
                let witness_script = bitcoin::Address::p2pkh(&::bitcoin::PublicKey{compressed: true, inner: remotepubkey}, Network::Testnet).script_pubkey();
                let sighash = hash_to_message!(&sighash::SighashCache::new(spend_tx).segwit_signature_hash(input_idx, &witness_script, descriptor.output.value, EcdsaSighashType::All).unwrap()[..]);
-               let remotesig = sign(secp_ctx, &sighash, &self.payment_key);
+               let remotesig = sign_with_aux_rand(secp_ctx, &sighash, &self.payment_key, &self);
                let payment_script = bitcoin::Address::p2wpkh(&::bitcoin::PublicKey{compressed: true, inner: remotepubkey}, Network::Bitcoin).unwrap().script_pubkey();
 
                if payment_script != descriptor.output.script_pubkey { return Err(()); }
@@ -722,7 +749,7 @@ impl InMemorySigner {
                let delayed_payment_pubkey = PublicKey::from_secret_key(&secp_ctx, &delayed_payment_key);
                let witness_script = chan_utils::get_revokeable_redeemscript(&descriptor.revocation_pubkey, descriptor.to_self_delay, &delayed_payment_pubkey);
                let sighash = hash_to_message!(&sighash::SighashCache::new(spend_tx).segwit_signature_hash(input_idx, &witness_script, descriptor.output.value, EcdsaSighashType::All).unwrap()[..]);
-               let local_delayedsig = sign(secp_ctx, &sighash, &delayed_payment_key);
+               let local_delayedsig = sign_with_aux_rand(secp_ctx, &sighash, &delayed_payment_key, &self);
                let payment_script = bitcoin::Address::p2wsh(&witness_script, Network::Bitcoin).script_pubkey();
 
                if descriptor.output.script_pubkey != payment_script { return Err(()); }
@@ -736,6 +763,15 @@ impl InMemorySigner {
        }
 }
 
+impl EntropySource for InMemorySigner {
+       fn get_secure_random_bytes(&self) -> [u8; 32] {
+               let index = self.rand_bytes_index.get_increment();
+               let mut nonce = [0u8; 16];
+               nonce[..8].copy_from_slice(&index.to_be_bytes());
+               ChaCha20::get_single_block(&self.rand_bytes_unique_start, &nonce)
+       }
+}
+
 impl ChannelSigner for InMemorySigner {
        fn get_per_commitment_point(&self, idx: u64, secp_ctx: &Secp256k1<secp256k1::All>) -> PublicKey {
                let commitment_secret = SecretKey::from_slice(&chan_utils::build_commitment_secret(&self.commitment_seed, idx)).unwrap();
@@ -774,7 +810,7 @@ impl EcdsaChannelSigner for InMemorySigner {
                let channel_funding_redeemscript = make_funding_redeemscript(&funding_pubkey, &self.counterparty_pubkeys().funding_pubkey);
 
                let built_tx = trusted_tx.built_transaction();
-               let commitment_sig = built_tx.sign(&self.funding_key, &channel_funding_redeemscript, self.channel_value_satoshis, secp_ctx);
+               let commitment_sig = built_tx.sign_counterparty_commitment(&self.funding_key, &channel_funding_redeemscript, self.channel_value_satoshis, secp_ctx);
                let commitment_txid = built_tx.txid;
 
                let mut htlc_sigs = Vec::with_capacity(commitment_tx.htlcs().len());
@@ -799,9 +835,9 @@ impl EcdsaChannelSigner for InMemorySigner {
                let funding_pubkey = PublicKey::from_secret_key(secp_ctx, &self.funding_key);
                let funding_redeemscript = make_funding_redeemscript(&funding_pubkey, &self.counterparty_pubkeys().funding_pubkey);
                let trusted_tx = commitment_tx.trust();
-               let sig = trusted_tx.built_transaction().sign(&self.funding_key, &funding_redeemscript, self.channel_value_satoshis, secp_ctx);
+               let sig = trusted_tx.built_transaction().sign_holder_commitment(&self.funding_key, &funding_redeemscript, self.channel_value_satoshis, &self, secp_ctx);
                let channel_parameters = self.get_channel_parameters();
-               let htlc_sigs = trusted_tx.get_htlc_sigs(&self.htlc_base_key, &channel_parameters.as_holder_broadcastable(), secp_ctx)?;
+               let htlc_sigs = trusted_tx.get_htlc_sigs(&self.htlc_base_key, &channel_parameters.as_holder_broadcastable(), &self, secp_ctx)?;
                Ok((sig, htlc_sigs))
        }
 
@@ -810,9 +846,9 @@ impl EcdsaChannelSigner for InMemorySigner {
                let funding_pubkey = PublicKey::from_secret_key(secp_ctx, &self.funding_key);
                let funding_redeemscript = make_funding_redeemscript(&funding_pubkey, &self.counterparty_pubkeys().funding_pubkey);
                let trusted_tx = commitment_tx.trust();
-               let sig = trusted_tx.built_transaction().sign(&self.funding_key, &funding_redeemscript, self.channel_value_satoshis, secp_ctx);
+               let sig = trusted_tx.built_transaction().sign_holder_commitment(&self.funding_key, &funding_redeemscript, self.channel_value_satoshis, &self, secp_ctx);
                let channel_parameters = self.get_channel_parameters();
-               let htlc_sigs = trusted_tx.get_htlc_sigs(&self.htlc_base_key, &channel_parameters.as_holder_broadcastable(), secp_ctx)?;
+               let htlc_sigs = trusted_tx.get_htlc_sigs(&self.htlc_base_key, &channel_parameters.as_holder_broadcastable(), &self, secp_ctx)?;
                Ok((sig, htlc_sigs))
        }
 
@@ -826,7 +862,7 @@ impl EcdsaChannelSigner for InMemorySigner {
                };
                let mut sighash_parts = sighash::SighashCache::new(justice_tx);
                let sighash = hash_to_message!(&sighash_parts.segwit_signature_hash(input, &witness_script, amount, EcdsaSighashType::All).unwrap()[..]);
-               return Ok(sign(secp_ctx, &sighash, &revocation_key))
+               return Ok(sign_with_aux_rand(secp_ctx, &sighash, &revocation_key, &self))
        }
 
        fn sign_justice_revoked_htlc(&self, justice_tx: &Transaction, input: usize, amount: u64, per_commitment_key: &SecretKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
@@ -840,7 +876,7 @@ impl EcdsaChannelSigner for InMemorySigner {
                };
                let mut sighash_parts = sighash::SighashCache::new(justice_tx);
                let sighash = hash_to_message!(&sighash_parts.segwit_signature_hash(input, &witness_script, amount, EcdsaSighashType::All).unwrap()[..]);
-               return Ok(sign(secp_ctx, &sighash, &revocation_key))
+               return Ok(sign_with_aux_rand(secp_ctx, &sighash, &revocation_key, &self))
        }
 
        #[cfg(anchors)]
@@ -858,7 +894,7 @@ impl EcdsaChannelSigner for InMemorySigner {
                let our_htlc_private_key = chan_utils::derive_private_key(
                        &secp_ctx, &per_commitment_point, &self.htlc_base_key
                );
-               Ok(sign(&secp_ctx, &hash_to_message!(sighash), &our_htlc_private_key))
+               Ok(sign_with_aux_rand(&secp_ctx, &hash_to_message!(sighash), &our_htlc_private_key, &self))
        }
 
        fn sign_counterparty_htlc_transaction(&self, htlc_tx: &Transaction, input: usize, amount: u64, per_commitment_point: &PublicKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
@@ -869,7 +905,7 @@ impl EcdsaChannelSigner for InMemorySigner {
                let witness_script = chan_utils::get_htlc_redeemscript_with_explicit_keys(&htlc, self.opt_anchors(), &counterparty_htlcpubkey, &htlcpubkey, &revocation_pubkey);
                let mut sighash_parts = sighash::SighashCache::new(htlc_tx);
                let sighash = hash_to_message!(&sighash_parts.segwit_signature_hash(input, &witness_script, amount, EcdsaSighashType::All).unwrap()[..]);
-               Ok(sign(secp_ctx, &sighash, &htlc_key))
+               Ok(sign_with_aux_rand(secp_ctx, &sighash, &htlc_key, &self))
        }
 
        fn sign_closing_transaction(&self, closing_tx: &ClosingTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
@@ -885,14 +921,14 @@ impl EcdsaChannelSigner for InMemorySigner {
                let sighash = sighash::SighashCache::new(&*anchor_tx).segwit_signature_hash(
                        input, &witness_script, ANCHOR_OUTPUT_VALUE_SATOSHI, EcdsaSighashType::All,
                ).unwrap();
-               Ok(sign(secp_ctx, &hash_to_message!(&sighash[..]), &self.funding_key))
+               Ok(sign_with_aux_rand(secp_ctx, &hash_to_message!(&sighash[..]), &self.funding_key, &self))
        }
 
        fn sign_channel_announcement_with_funding_key(
                &self, msg: &UnsignedChannelAnnouncement, secp_ctx: &Secp256k1<secp256k1::All>
        ) -> Result<Signature, ()> {
                let msghash = hash_to_message!(&Sha256dHash::hash(&msg.encode()[..])[..]);
-               Ok(sign(secp_ctx, &msghash, &self.funding_key))
+               Ok(secp_ctx.sign_ecdsa(&msghash, &self.funding_key))
        }
 }
 
@@ -922,8 +958,8 @@ impl Writeable for InMemorySigner {
        }
 }
 
-impl Readable for InMemorySigner {
-       fn read<R: io::Read>(reader: &mut R) -> Result<Self, DecodeError> {
+impl<ES: Deref> ReadableArgs<ES> for InMemorySigner where ES::Target: EntropySource {
+       fn read<R: io::Read>(reader: &mut R, entropy_source: ES) -> Result<Self, DecodeError> {
                let _ver = read_ver_prefix!(reader, SERIALIZATION_VERSION);
 
                let funding_key = Readable::read(reader)?;
@@ -953,6 +989,8 @@ impl Readable for InMemorySigner {
                        holder_channel_pubkeys,
                        channel_parameters: counterparty_channel_data,
                        channel_keys_id: keys_id,
+                       rand_bytes_unique_start: entropy_source.get_secure_random_bytes(),
+                       rand_bytes_index: AtomicCounter::new(),
                })
        }
 }
@@ -1107,6 +1145,7 @@ impl KeysManager {
                let payment_key = key_step!(b"payment key", revocation_base_key);
                let delayed_payment_base_key = key_step!(b"delayed payment base key", payment_key);
                let htlc_base_key = key_step!(b"HTLC base key", delayed_payment_base_key);
+               let prng_seed = self.get_secure_random_bytes();
 
                InMemorySigner::new(
                        &self.secp_ctx,
@@ -1118,6 +1157,7 @@ impl KeysManager {
                        commitment_seed,
                        channel_value_satoshis,
                        params.clone(),
+                       prng_seed,
                )
        }
 
@@ -1233,7 +1273,7 @@ impl KeysManager {
                                        if payment_script != output.script_pubkey { return Err(()); };
 
                                        let sighash = hash_to_message!(&sighash::SighashCache::new(&spend_tx).segwit_signature_hash(input_idx, &witness_script, output.value, EcdsaSighashType::All).unwrap()[..]);
-                                       let sig = sign(secp_ctx, &sighash, &secret.private_key);
+                                       let sig = sign_with_aux_rand(secp_ctx, &sighash, &secret.private_key, &self);
                                        let mut sig_ser = sig.serialize_der().to_vec();
                                        sig_ser.push(EcdsaSighashType::All as u8);
                                        spend_tx.input[input_idx].witness.push(sig_ser);
@@ -1295,7 +1335,7 @@ impl NodeSigner for KeysManager {
 
        fn sign_gossip_message(&self, msg: UnsignedGossipMessage) -> Result<Signature, ()> {
                let msg_hash = hash_to_message!(&Sha256dHash::hash(&msg.encode()[..])[..]);
-               Ok(sign(&self.secp_ctx, &msg_hash, &self.node_secret))
+               Ok(self.secp_ctx.sign_ecdsa(&msg_hash, &self.node_secret))
        }
 }
 
@@ -1323,7 +1363,7 @@ impl SignerProvider for KeysManager {
        }
 
        fn read_chan_signer(&self, reader: &[u8]) -> Result<Self::Signer, DecodeError> {
-               InMemorySigner::read(&mut io::Cursor::new(reader))
+               InMemorySigner::read(&mut io::Cursor::new(reader), self)
        }
 
        fn get_destination_script(&self) -> Script {
index f690d3664273b94e8ee9fe8d6d4c9f16d44ece0e..04776fbb0899038e1aef56ff2cf12361d7649c2c 100644 (file)
@@ -481,6 +481,59 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                events.into_iter().map(|(_, event)| event).collect()
        }
 
+       /// Triggers rebroadcasts/fee-bumps of pending claims from a force-closed channel. This is
+       /// crucial in preventing certain classes of pinning attacks, detecting substantial mempool
+       /// feerate changes between blocks, and ensuring reliability if broadcasting fails. We recommend
+       /// invoking this every 30 seconds, or lower if running in an environment with spotty
+       /// connections, like on mobile.
+       pub(crate) fn rebroadcast_pending_claims<B: Deref, F: Deref, L: Deref>(
+               &mut self, current_height: u32, broadcaster: &B, fee_estimator: &LowerBoundedFeeEstimator<F>,
+               logger: &L,
+       )
+       where
+               B::Target: BroadcasterInterface,
+               F::Target: FeeEstimator,
+               L::Target: Logger,
+       {
+               let mut bump_requests = Vec::with_capacity(self.pending_claim_requests.len());
+               for (package_id, request) in self.pending_claim_requests.iter() {
+                       let inputs = request.outpoints();
+                       log_info!(logger, "Triggering rebroadcast/fee-bump for request with inputs {:?}", inputs);
+                       bump_requests.push((*package_id, request.clone()));
+               }
+               for (package_id, request) in bump_requests {
+                       self.generate_claim(current_height, &request, false /* force_feerate_bump */, fee_estimator, logger)
+                               .map(|(_, new_feerate, claim)| {
+                                       let mut bumped_feerate = false;
+                                       if let Some(mut_request) = self.pending_claim_requests.get_mut(&package_id) {
+                                               bumped_feerate = request.previous_feerate() > new_feerate;
+                                               mut_request.set_feerate(new_feerate);
+                                       }
+                                       match claim {
+                                               OnchainClaim::Tx(tx) => {
+                                                       let log_start = if bumped_feerate { "Broadcasting RBF-bumped" } else { "Rebroadcasting" };
+                                                       log_info!(logger, "{} onchain {}", log_start, log_tx!(tx));
+                                                       broadcaster.broadcast_transaction(&tx);
+                                               },
+                                               #[cfg(anchors)]
+                                               OnchainClaim::Event(event) => {
+                                                       let log_start = if bumped_feerate { "Yielding fee-bumped" } else { "Replaying" };
+                                                       log_info!(logger, "{} onchain event to spend inputs {:?}", log_start,
+                                                               request.outpoints());
+                                                       #[cfg(debug_assertions)] {
+                                                               debug_assert!(request.requires_external_funding());
+                                                               let num_existing = self.pending_claim_events.iter()
+                                                                       .filter(|entry| entry.0 == package_id).count();
+                                                               assert!(num_existing == 0 || num_existing == 1);
+                                                       }
+                                                       self.pending_claim_events.retain(|event| event.0 != package_id);
+                                                       self.pending_claim_events.push((package_id, event));
+                                               }
+                                       }
+                               });
+               }
+       }
+
        /// Lightning security model (i.e being able to redeem/timeout HTLC or penalize counterparty
        /// onchain) lays on the assumption of claim transactions getting confirmed before timelock
        /// expiration (CSV or CLTV following cases). In case of high-fee spikes, claim tx may get stuck
@@ -489,9 +542,13 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
        ///
        /// Panics if there are signing errors, because signing operations in reaction to on-chain
        /// events are not expected to fail, and if they do, we may lose funds.
-       fn generate_claim<F: Deref, L: Deref>(&mut self, cur_height: u32, cached_request: &PackageTemplate, fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L) -> Option<(Option<u32>, u64, OnchainClaim)>
-               where F::Target: FeeEstimator,
-                                       L::Target: Logger,
+       fn generate_claim<F: Deref, L: Deref>(
+               &mut self, cur_height: u32, cached_request: &PackageTemplate, force_feerate_bump: bool,
+               fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L,
+       ) -> Option<(u32, u64, OnchainClaim)>
+       where
+               F::Target: FeeEstimator,
+               L::Target: Logger,
        {
                let request_outpoints = cached_request.outpoints();
                if request_outpoints.is_empty() {
@@ -533,13 +590,14 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
 
                // Compute new height timer to decide when we need to regenerate a new bumped version of the claim tx (if we
                // didn't receive confirmation of it before, or not enough reorg-safe depth on top of it).
-               let new_timer = Some(cached_request.get_height_timer(cur_height));
+               let new_timer = cached_request.get_height_timer(cur_height);
                if cached_request.is_malleable() {
                        #[cfg(anchors)]
                        { // Attributes are not allowed on if expressions on our current MSRV of 1.41.
                                if cached_request.requires_external_funding() {
-                                       let target_feerate_sat_per_1000_weight = cached_request
-                                               .compute_package_feerate(fee_estimator, ConfirmationTarget::HighPriority);
+                                       let target_feerate_sat_per_1000_weight = cached_request.compute_package_feerate(
+                                               fee_estimator, ConfirmationTarget::HighPriority, force_feerate_bump
+                                       );
                                        if let Some(htlcs) = cached_request.construct_malleable_package_with_external_funding(self) {
                                                return Some((
                                                        new_timer,
@@ -558,14 +616,15 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
 
                        let predicted_weight = cached_request.package_weight(&self.destination_script);
                        if let Some((output_value, new_feerate)) = cached_request.compute_package_output(
-                               predicted_weight, self.destination_script.dust_value().to_sat(), fee_estimator, logger,
+                               predicted_weight, self.destination_script.dust_value().to_sat(),
+                               force_feerate_bump, fee_estimator, logger,
                        ) {
                                assert!(new_feerate != 0);
 
                                let transaction = cached_request.finalize_malleable_package(
                                        cur_height, self, output_value, self.destination_script.clone(), logger
                                ).unwrap();
-                               log_trace!(logger, "...with timer {} and feerate {}", new_timer.unwrap(), new_feerate);
+                               log_trace!(logger, "...with timer {} and feerate {}", new_timer, new_feerate);
                                assert!(predicted_weight >= transaction.weight());
                                return Some((new_timer, new_feerate, OnchainClaim::Tx(transaction)));
                        }
@@ -583,7 +642,7 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                                None => return None,
                        };
                        if !cached_request.requires_external_funding() {
-                               return Some((None, 0, OnchainClaim::Tx(tx)));
+                               return Some((new_timer, 0, OnchainClaim::Tx(tx)));
                        }
                        #[cfg(anchors)]
                        return inputs.find_map(|input| match input {
@@ -601,7 +660,7 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                                                        // counterparty's latest commitment don't have any HTLCs present.
                                                        let conf_target = ConfirmationTarget::HighPriority;
                                                        let package_target_feerate_sat_per_1000_weight = cached_request
-                                                               .compute_package_feerate(fee_estimator, conf_target);
+                                                               .compute_package_feerate(fee_estimator, conf_target, force_feerate_bump);
                                                        Some((
                                                                new_timer,
                                                                package_target_feerate_sat_per_1000_weight as u64,
@@ -616,7 +675,7 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                                                // attempt to broadcast the transaction with its current fee rate and hope
                                                // it confirms. This is essentially the same behavior as a commitment
                                                // transaction without anchor outputs.
-                                               None => Some((None, 0, OnchainClaim::Tx(tx.clone()))),
+                                               None => Some((new_timer, 0, OnchainClaim::Tx(tx.clone()))),
                                        }
                                },
                                _ => {
@@ -700,7 +759,9 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                // Generate claim transactions and track them to bump if necessary at
                // height timer expiration (i.e in how many blocks we're going to take action).
                for mut req in preprocessed_requests {
-                       if let Some((new_timer, new_feerate, claim)) = self.generate_claim(cur_height, &req, &*fee_estimator, &*logger) {
+                       if let Some((new_timer, new_feerate, claim)) = self.generate_claim(
+                               cur_height, &req, true /* force_feerate_bump */, &*fee_estimator, &*logger,
+                       ) {
                                req.set_timer(new_timer);
                                req.set_feerate(new_feerate);
                                let package_id = match claim {
@@ -885,17 +946,17 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
 
                // Check if any pending claim request must be rescheduled
                for (package_id, request) in self.pending_claim_requests.iter() {
-                       if let Some(h) = request.timer() {
-                               if cur_height >= h {
-                                       bump_candidates.insert(*package_id, request.clone());
-                               }
+                       if cur_height >= request.timer() {
+                               bump_candidates.insert(*package_id, request.clone());
                        }
                }
 
                // Build, bump and rebroadcast tx accordingly
                log_trace!(logger, "Bumping {} candidates", bump_candidates.len());
                for (package_id, request) in bump_candidates.iter() {
-                       if let Some((new_timer, new_feerate, bump_claim)) = self.generate_claim(cur_height, &request, &*fee_estimator, &*logger) {
+                       if let Some((new_timer, new_feerate, bump_claim)) = self.generate_claim(
+                               cur_height, &request, true /* force_feerate_bump */, &*fee_estimator, &*logger,
+                       ) {
                                match bump_claim {
                                        OnchainClaim::Tx(bump_tx) => {
                                                log_info!(logger, "Broadcasting RBF-bumped onchain {}", log_tx!(bump_tx));
@@ -975,7 +1036,9 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                        }
                }
                for ((_package_id, _), ref mut request) in bump_candidates.iter_mut() {
-                       if let Some((new_timer, new_feerate, bump_claim)) = self.generate_claim(height, &request, fee_estimator, &&*logger) {
+                       if let Some((new_timer, new_feerate, bump_claim)) = self.generate_claim(
+                               height, &request, true /* force_feerate_bump */, fee_estimator, &&*logger
+                       ) {
                                request.set_timer(new_timer);
                                request.set_feerate(new_feerate);
                                match bump_claim {
index e8886e5a2aa49958fcac012529a3ae4160348230..b3f6f50ed1d17fb390c04a2995eb98e9ef3fd558 100644 (file)
@@ -538,7 +538,7 @@ pub struct PackageTemplate {
        feerate_previous: u64,
        // Cache of next height at which fee-bumping and rebroadcast will be attempted. In
        // the future, we might abstract it to an observed mempool fluctuation.
-       height_timer: Option<u32>,
+       height_timer: u32,
        // Confirmation height of the claimed outputs set transaction. In case of reorg reaching
        // it, we wipe out and forget the package.
        height_original: u32,
@@ -554,16 +554,16 @@ impl PackageTemplate {
        pub(crate) fn aggregable(&self) -> bool {
                self.aggregable
        }
+       pub(crate) fn previous_feerate(&self) -> u64 {
+               self.feerate_previous
+       }
        pub(crate) fn set_feerate(&mut self, new_feerate: u64) {
                self.feerate_previous = new_feerate;
        }
-       pub(crate) fn timer(&self) -> Option<u32> {
-               if let Some(ref timer) = self.height_timer {
-                       return Some(*timer);
-               }
-               None
+       pub(crate) fn timer(&self) -> u32 {
+               self.height_timer
        }
-       pub(crate) fn set_timer(&mut self, new_timer: Option<u32>) {
+       pub(crate) fn set_timer(&mut self, new_timer: u32) {
                self.height_timer = new_timer;
        }
        pub(crate) fn outpoints(&self) -> Vec<&BitcoinOutPoint> {
@@ -765,16 +765,23 @@ impl PackageTemplate {
        /// Returns value in satoshis to be included as package outgoing output amount and feerate
        /// which was used to generate the value. Will not return less than `dust_limit_sats` for the
        /// value.
-       pub(crate) fn compute_package_output<F: Deref, L: Deref>(&self, predicted_weight: usize, dust_limit_sats: u64, fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L) -> Option<(u64, u64)>
-               where F::Target: FeeEstimator,
-                     L::Target: Logger,
+       pub(crate) fn compute_package_output<F: Deref, L: Deref>(
+               &self, predicted_weight: usize, dust_limit_sats: u64, force_feerate_bump: bool,
+               fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L,
+       ) -> Option<(u64, u64)>
+       where
+               F::Target: FeeEstimator,
+               L::Target: Logger,
        {
                debug_assert!(self.malleability == PackageMalleability::Malleable, "The package output is fixed for non-malleable packages");
                let input_amounts = self.package_amount();
                assert!(dust_limit_sats as i64 > 0, "Output script must be broadcastable/have a 'real' dust limit.");
                // If old feerate is 0, first iteration of this claim, use normal fee calculation
                if self.feerate_previous != 0 {
-                       if let Some((new_fee, feerate)) = feerate_bump(predicted_weight, input_amounts, self.feerate_previous, fee_estimator, logger) {
+                       if let Some((new_fee, feerate)) = feerate_bump(
+                               predicted_weight, input_amounts, self.feerate_previous, force_feerate_bump,
+                               fee_estimator, logger,
+                       ) {
                                return Some((cmp::max(input_amounts as i64 - new_fee as i64, dust_limit_sats as i64) as u64, feerate));
                        }
                } else {
@@ -787,16 +794,19 @@ impl PackageTemplate {
 
        #[cfg(anchors)]
        /// Computes a feerate based on the given confirmation target. If a previous feerate was used,
-       /// and the new feerate is below it, we'll use a 25% increase of the previous feerate instead of
-       /// the new one.
+       /// the new feerate is below it, and `force_feerate_bump` is set, we'll use a 25% increase of
+       /// the previous feerate instead of the new feerate.
        pub(crate) fn compute_package_feerate<F: Deref>(
                &self, fee_estimator: &LowerBoundedFeeEstimator<F>, conf_target: ConfirmationTarget,
+               force_feerate_bump: bool,
        ) -> u32 where F::Target: FeeEstimator {
                let feerate_estimate = fee_estimator.bounded_sat_per_1000_weight(conf_target);
                if self.feerate_previous != 0 {
                        // If old feerate inferior to actual one given back by Fee Estimator, use it to compute new fee...
                        if feerate_estimate as u64 > self.feerate_previous {
                                feerate_estimate
+                       } else if !force_feerate_bump {
+                               self.feerate_previous.try_into().unwrap_or(u32::max_value())
                        } else {
                                // ...else just increase the previous feerate by 25% (because that's a nice number)
                                (self.feerate_previous + (self.feerate_previous / 4)).try_into().unwrap_or(u32::max_value())
@@ -837,7 +847,7 @@ impl PackageTemplate {
                        soonest_conf_deadline,
                        aggregable,
                        feerate_previous: 0,
-                       height_timer: None,
+                       height_timer: height_original,
                        height_original,
                }
        }
@@ -854,7 +864,7 @@ impl Writeable for PackageTemplate {
                        (0, self.soonest_conf_deadline, required),
                        (2, self.feerate_previous, required),
                        (4, self.height_original, required),
-                       (6, self.height_timer, option)
+                       (6, self.height_timer, required)
                });
                Ok(())
        }
@@ -893,13 +903,16 @@ impl Readable for PackageTemplate {
                        (4, height_original, required),
                        (6, height_timer, option),
                });
+               if height_timer.is_none() {
+                       height_timer = Some(height_original);
+               }
                Ok(PackageTemplate {
                        inputs,
                        malleability,
                        soonest_conf_deadline,
                        aggregable,
                        feerate_previous,
-                       height_timer,
+                       height_timer: height_timer.unwrap(),
                        height_original,
                })
        }
@@ -945,32 +958,47 @@ fn compute_fee_from_spent_amounts<F: Deref, L: Deref>(input_amounts: u64, predic
 
 /// Attempt to propose a bumping fee for a transaction from its spent output's values and predicted
 /// weight. If feerates proposed by the fee-estimator have been increasing since last fee-bumping
-/// attempt, use them. Otherwise, blindly bump the feerate by 25% of the previous feerate. We also
-/// verify that those bumping heuristics respect BIP125 rules 3) and 4) and if required adjust
-/// the new fee to meet the RBF policy requirement.
-fn feerate_bump<F: Deref, L: Deref>(predicted_weight: usize, input_amounts: u64, previous_feerate: u64, fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L) -> Option<(u64, u64)>
-       where F::Target: FeeEstimator,
-             L::Target: Logger,
+/// attempt, use them. If `force_feerate_bump` is set, we bump the feerate by 25% of the previous
+/// feerate, or just use the previous feerate otherwise. If a feerate bump did happen, we also
+/// verify that those bumping heuristics respect BIP125 rules 3) and 4) and if required adjust the
+/// new fee to meet the RBF policy requirement.
+fn feerate_bump<F: Deref, L: Deref>(
+       predicted_weight: usize, input_amounts: u64, previous_feerate: u64, force_feerate_bump: bool,
+       fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L,
+) -> Option<(u64, u64)>
+where
+       F::Target: FeeEstimator,
+       L::Target: Logger,
 {
        // If old feerate inferior to actual one given back by Fee Estimator, use it to compute new fee...
-       let new_fee = if let Some((new_fee, _)) = compute_fee_from_spent_amounts(input_amounts, predicted_weight, fee_estimator, logger) {
-               let updated_feerate = new_fee / (predicted_weight as u64 * 1000);
-               if updated_feerate > previous_feerate {
-                       new_fee
+       let (new_fee, new_feerate) = if let Some((new_fee, new_feerate)) = compute_fee_from_spent_amounts(input_amounts, predicted_weight, fee_estimator, logger) {
+               if new_feerate > previous_feerate {
+                       (new_fee, new_feerate)
+               } else if !force_feerate_bump {
+                       let previous_fee = previous_feerate * (predicted_weight as u64) / 1000;
+                       (previous_fee, previous_feerate)
                } else {
                        // ...else just increase the previous feerate by 25% (because that's a nice number)
-                       let new_fee = previous_feerate * (predicted_weight as u64) / 750;
-                       if input_amounts <= new_fee {
+                       let bumped_feerate = previous_feerate + (previous_feerate / 4);
+                       let bumped_fee = bumped_feerate * (predicted_weight as u64) / 1000;
+                       if input_amounts <= bumped_fee {
                                log_warn!(logger, "Can't 25% bump new claiming tx, amount {} is too small", input_amounts);
                                return None;
                        }
-                       new_fee
+                       (bumped_fee, bumped_feerate)
                }
        } else {
                log_warn!(logger, "Can't new-estimation bump new claiming tx, amount {} is too small", input_amounts);
                return None;
        };
 
+       // Our feerates should never decrease. If it hasn't changed though, we just need to
+       // rebroadcast/re-sign the previous claim.
+       debug_assert!(new_feerate >= previous_feerate);
+       if new_feerate == previous_feerate {
+               return Some((new_fee, new_feerate));
+       }
+
        let previous_fee = previous_feerate * (predicted_weight as u64) / 1000;
        let min_relay_fee = MIN_RELAY_FEE_SAT_PER_1000_WEIGHT * (predicted_weight as u64) / 1000;
        // BIP 125 Opt-in Full Replace-by-Fee Signaling
@@ -1177,12 +1205,9 @@ mod tests {
                let revk_outp = dumb_revk_output!(secp_ctx);
 
                let mut package = PackageTemplate::build_package(txid, 0, revk_outp, 1000, true, 100);
-               let timer_none = package.timer();
-               assert!(timer_none.is_none());
-               package.set_timer(Some(100));
-               if let Some(timer_some) = package.timer() {
-                       assert_eq!(timer_some, 100);
-               } else { panic!() }
+               assert_eq!(package.timer(), 100);
+               package.set_timer(101);
+               assert_eq!(package.timer(), 101);
        }
 
        #[test]
index 63762dc55d4a95b82e95c8660ae652809641a9c1..1bd1214596de769154fab9cd37e41df1b3748e7d 100644 (file)
@@ -21,7 +21,7 @@ pub mod bump_transaction;
 pub use bump_transaction::BumpTransactionEvent;
 
 use crate::chain::keysinterface::SpendableOutputDescriptor;
-use crate::ln::channelmanager::{InterceptId, PaymentId};
+use crate::ln::channelmanager::{InterceptId, PaymentId, RecipientOnionFields};
 use crate::ln::channel::FUNDING_CONF_DEADLINE_BLOCKS;
 use crate::ln::features::ChannelTypeFeatures;
 use crate::ln::msgs;
@@ -232,8 +232,11 @@ pub enum HTLCDestination {
        ///
        /// Some of the reasons may include:
        /// * HTLC Timeouts
-       /// * Expected MPP amount has already been reached
-       /// * Claimable amount does not match expected amount
+       /// * Excess HTLCs for a payment that we have already fully received, over-paying for the
+       ///   payment,
+       /// * The counterparty node modified the HTLC in transit,
+       /// * A probing attack where an intermediary node is trying to detect if we are the ultimate
+       ///   recipient for a payment.
        FailedPayment {
                /// The payment hash of the payment we attempted to process.
                payment_hash: PaymentHash
@@ -379,6 +382,11 @@ pub enum Event {
                /// The hash for which the preimage should be handed to the ChannelManager. Note that LDK will
                /// not stop you from registering duplicate payment hashes for inbound payments.
                payment_hash: PaymentHash,
+               /// The fields in the onion which were received with each HTLC. Only fields which were
+               /// identical in each HTLC involved in the payment will be included here.
+               ///
+               /// Payments received on LDK versions prior to 0.0.115 will have this field unset.
+               onion_fields: Option<RecipientOnionFields>,
                /// The value, in thousandths of a satoshi, that this payment is for.
                amount_msat: u64,
                /// Information for claiming this received payment, based on whether the purpose of the
@@ -820,7 +828,10 @@ impl Writeable for Event {
                                // We never write out FundingGenerationReady events as, upon disconnection, peers
                                // drop any channels which have not yet exchanged funding_signed.
                        },
-                       &Event::PaymentClaimable { ref payment_hash, ref amount_msat, ref purpose, ref receiver_node_id, ref via_channel_id, ref via_user_channel_id, ref claim_deadline } => {
+                       &Event::PaymentClaimable { ref payment_hash, ref amount_msat, ref purpose,
+                               ref receiver_node_id, ref via_channel_id, ref via_user_channel_id,
+                               ref claim_deadline, ref onion_fields
+                       } => {
                                1u8.write(writer)?;
                                let mut payment_secret = None;
                                let payment_preimage;
@@ -843,6 +854,7 @@ impl Writeable for Event {
                                        (6, 0u64, required), // user_payment_id required for compatibility with 0.0.103 and earlier
                                        (7, claim_deadline, option),
                                        (8, payment_preimage, option),
+                                       (9, onion_fields, option),
                                });
                        },
                        &Event::PaymentSent { ref payment_id, ref payment_preimage, ref payment_hash, ref fee_paid_msat } => {
@@ -1043,6 +1055,7 @@ impl MaybeReadable for Event {
                                        let mut via_channel_id = None;
                                        let mut claim_deadline = None;
                                        let mut via_user_channel_id = None;
+                                       let mut onion_fields = None;
                                        read_tlv_fields!(reader, {
                                                (0, payment_hash, required),
                                                (1, receiver_node_id, option),
@@ -1053,6 +1066,7 @@ impl MaybeReadable for Event {
                                                (6, _user_payment_id, option),
                                                (7, claim_deadline, option),
                                                (8, payment_preimage, option),
+                                               (9, onion_fields, option),
                                        });
                                        let purpose = match payment_secret {
                                                Some(secret) => PaymentPurpose::InvoicePayment {
@@ -1070,6 +1084,7 @@ impl MaybeReadable for Event {
                                                via_channel_id,
                                                via_user_channel_id,
                                                claim_deadline,
+                                               onion_fields,
                                        }))
                                };
                                f()
index 533da9772a2c8db922794546b696e4bae32a7f94..9699cc78a1a3253a7635277d22c133a3517b2804 100644 (file)
@@ -21,6 +21,7 @@ use bitcoin::hashes::sha256::Hash as Sha256;
 use bitcoin::hashes::ripemd160::Hash as Ripemd160;
 use bitcoin::hash_types::{Txid, PubkeyHash};
 
+use crate::chain::keysinterface::EntropySource;
 use crate::ln::{PaymentHash, PaymentPreimage};
 use crate::ln::msgs::DecodeError;
 use crate::util::ser::{Readable, Writeable, Writer};
@@ -39,7 +40,7 @@ use crate::util::transaction_utils::sort_outputs;
 use crate::ln::channel::{INITIAL_COMMITMENT_NUMBER, ANCHOR_OUTPUT_VALUE_SATOSHI};
 use core::ops::Deref;
 use crate::chain;
-use crate::util::crypto::sign;
+use crate::util::crypto::{sign, sign_with_aux_rand};
 
 /// Maximum number of one-way in-flight HTLC (protocol-level value).
 pub const MAX_HTLCS: u16 = 483;
@@ -1081,12 +1082,20 @@ impl BuiltCommitmentTransaction {
                hash_to_message!(sighash)
        }
 
-       /// Sign a transaction, either because we are counter-signing the counterparty's transaction or
-       /// because we are about to broadcast a holder transaction.
-       pub fn sign<T: secp256k1::Signing>(&self, funding_key: &SecretKey, funding_redeemscript: &Script, channel_value_satoshis: u64, secp_ctx: &Secp256k1<T>) -> Signature {
+       /// Signs the counterparty's commitment transaction.
+       pub fn sign_counterparty_commitment<T: secp256k1::Signing>(&self, funding_key: &SecretKey, funding_redeemscript: &Script, channel_value_satoshis: u64, secp_ctx: &Secp256k1<T>) -> Signature {
                let sighash = self.get_sighash_all(funding_redeemscript, channel_value_satoshis);
                sign(secp_ctx, &sighash, funding_key)
        }
+
+       /// Signs the holder commitment transaction because we are about to broadcast it.
+       pub fn sign_holder_commitment<T: secp256k1::Signing, ES: Deref>(
+               &self, funding_key: &SecretKey, funding_redeemscript: &Script, channel_value_satoshis: u64,
+               entropy_source: &ES, secp_ctx: &Secp256k1<T>
+       ) -> Signature where ES::Target: EntropySource {
+               let sighash = self.get_sighash_all(funding_redeemscript, channel_value_satoshis);
+               sign_with_aux_rand(secp_ctx, &sighash, funding_key, entropy_source)
+       }
 }
 
 /// This class tracks the per-transaction information needed to build a closing transaction and will
@@ -1563,7 +1572,10 @@ impl<'a> TrustedCommitmentTransaction<'a> {
        /// The returned Vec has one entry for each HTLC, and in the same order.
        ///
        /// This function is only valid in the holder commitment context, it always uses EcdsaSighashType::All.
-       pub fn get_htlc_sigs<T: secp256k1::Signing>(&self, htlc_base_key: &SecretKey, channel_parameters: &DirectedChannelTransactionParameters, secp_ctx: &Secp256k1<T>) -> Result<Vec<Signature>, ()> {
+       pub fn get_htlc_sigs<T: secp256k1::Signing, ES: Deref>(
+               &self, htlc_base_key: &SecretKey, channel_parameters: &DirectedChannelTransactionParameters,
+               entropy_source: &ES, secp_ctx: &Secp256k1<T>,
+       ) -> Result<Vec<Signature>, ()> where ES::Target: EntropySource {
                let inner = self.inner;
                let keys = &inner.keys;
                let txid = inner.built.txid;
@@ -1577,7 +1589,7 @@ impl<'a> TrustedCommitmentTransaction<'a> {
                        let htlc_redeemscript = get_htlc_redeemscript_with_explicit_keys(&this_htlc, self.opt_anchors(), &keys.broadcaster_htlc_key, &keys.countersignatory_htlc_key, &keys.revocation_key);
 
                        let sighash = hash_to_message!(&sighash::SighashCache::new(&htlc_tx).segwit_signature_hash(0, &htlc_redeemscript, this_htlc.amount_msat / 1000, EcdsaSighashType::All).unwrap()[..]);
-                       ret.push(sign(secp_ctx, &sighash, &holder_htlc_key));
+                       ret.push(sign_with_aux_rand(secp_ctx, &sighash, &holder_htlc_key, entropy_source));
                }
                Ok(ret)
        }
index e533e532f7e6d526c97fd11b2257e70dafa59c07..dd553c1ff8604c75c03063fffb1e073fc64ae7fe 100644 (file)
@@ -312,9 +312,9 @@ pub(super) enum ChannelUpdateStatus {
        /// We've announced the channel as enabled and are connected to our peer.
        Enabled,
        /// Our channel is no longer live, but we haven't announced the channel as disabled yet.
-       DisabledStaged,
+       DisabledStaged(u8),
        /// Our channel is live again, but we haven't announced the channel as enabled yet.
-       EnabledStaged,
+       EnabledStaged(u8),
        /// We've announced the channel as disabled.
        Disabled,
 }
@@ -6193,8 +6193,8 @@ impl Writeable for ChannelUpdateStatus {
                // channel as enabled, so we write 0. For EnabledStaged, we similarly write a 1.
                match self {
                        ChannelUpdateStatus::Enabled => 0u8.write(writer)?,
-                       ChannelUpdateStatus::DisabledStaged => 0u8.write(writer)?,
-                       ChannelUpdateStatus::EnabledStaged => 1u8.write(writer)?,
+                       ChannelUpdateStatus::DisabledStaged(_) => 0u8.write(writer)?,
+                       ChannelUpdateStatus::EnabledStaged(_) => 1u8.write(writer)?,
                        ChannelUpdateStatus::Disabled => 1u8.write(writer)?,
                }
                Ok(())
@@ -7516,7 +7516,7 @@ mod tests {
                }
        }
 
-       #[cfg(not(feature = "grind_signatures"))]
+       #[cfg(feature = "_test_vectors")]
        #[test]
        fn outbound_commitment_test() {
                use bitcoin::util::sighash;
@@ -7549,6 +7549,7 @@ mod tests {
                        [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff],
                        10_000_000,
                        [0; 32],
+                       [0; 32],
                );
 
                assert_eq!(signer.pubkeys().funding_pubkey.serialize()[..],
index 37b22a18428d27fa14cf24b05624dd85f2324cee..830985ac713d1976840ea841def2242dd7b9120f 100644 (file)
@@ -106,11 +106,13 @@ pub(super) enum PendingHTLCRouting {
        },
        Receive {
                payment_data: msgs::FinalOnionHopData,
+               payment_metadata: Option<Vec<u8>>,
                incoming_cltv_expiry: u32, // Used to track when we should expire pending HTLCs that go unclaimed
                phantom_shared_secret: Option<[u8; 32]>,
        },
        ReceiveKeysend {
                payment_preimage: PaymentPreimage,
+               payment_metadata: Option<Vec<u8>>,
                incoming_cltv_expiry: u32, // Used to track when we should expire pending HTLCs that go unclaimed
        },
 }
@@ -470,6 +472,12 @@ impl_writeable_tlv_based!(ClaimingPayment, {
        (4, receiver_node_id, required),
 });
 
+struct ClaimablePayment {
+       purpose: events::PaymentPurpose,
+       onion_fields: Option<RecipientOnionFields>,
+       htlcs: Vec<ClaimableHTLC>,
+}
+
 /// Information about claimable or being-claimed payments
 struct ClaimablePayments {
        /// Map from payment hash to the payment data and any HTLCs which are to us and can be
@@ -480,7 +488,7 @@ struct ClaimablePayments {
        ///
        /// When adding to the map, [`Self::pending_claiming_payments`] must also be checked to ensure
        /// we don't get a duplicate payment.
-       claimable_htlcs: HashMap<PaymentHash, (events::PaymentPurpose, Vec<ClaimableHTLC>)>,
+       claimable_payments: HashMap<PaymentHash, ClaimablePayment>,
 
        /// Map from payment hash to the payment data for HTLCs which we have begun claiming, but which
        /// are waiting on a [`ChannelMonitorUpdate`] to complete in order to be surfaced to the user
@@ -1072,6 +1080,14 @@ pub(crate) const MPP_TIMEOUT_TICKS: u8 = 3;
 /// [`OutboundPayments::remove_stale_resolved_payments`].
 pub(crate) const IDEMPOTENCY_TIMEOUT_TICKS: u8 = 7;
 
+/// The number of ticks of [`ChannelManager::timer_tick_occurred`] where a peer is disconnected
+/// until we mark the channel disabled and gossip the update.
+pub(crate) const DISABLE_GOSSIP_TICKS: u8 = 10;
+
+/// The number of ticks of [`ChannelManager::timer_tick_occurred`] where a peer is connected until
+/// we mark the channel enabled and gossip the update.
+pub(crate) const ENABLE_GOSSIP_TICKS: u8 = 5;
+
 /// The maximum number of unfunded channels we can have per-peer before we start rejecting new
 /// (inbound) ones. The number of peers with unfunded channels is limited separately in
 /// [`MAX_UNFUNDED_CHANNEL_PEERS`].
@@ -1772,7 +1788,7 @@ where
                        pending_inbound_payments: Mutex::new(HashMap::new()),
                        pending_outbound_payments: OutboundPayments::new(),
                        forward_htlcs: Mutex::new(HashMap::new()),
-                       claimable_payments: Mutex::new(ClaimablePayments { claimable_htlcs: HashMap::new(), pending_claiming_payments: HashMap::new() }),
+                       claimable_payments: Mutex::new(ClaimablePayments { claimable_payments: HashMap::new(), pending_claiming_payments: HashMap::new() }),
                        pending_intercepted_htlcs: Mutex::new(HashMap::new()),
                        id_to_peer: Mutex::new(HashMap::new()),
                        short_to_chan_info: FairRwLock::new(HashMap::new()),
@@ -2268,7 +2284,7 @@ where
                                        msg: "Got non final data with an HMAC of 0",
                                });
                        },
-                       msgs::OnionHopDataFormat::FinalNode { payment_data, keysend_preimage } => {
+                       msgs::OnionHopDataFormat::FinalNode { payment_data, keysend_preimage, payment_metadata } => {
                                if payment_data.is_some() && keysend_preimage.is_some() {
                                        return Err(ReceiveError {
                                                err_code: 0x4000|22,
@@ -2278,6 +2294,7 @@ where
                                } else if let Some(data) = payment_data {
                                        PendingHTLCRouting::Receive {
                                                payment_data: data,
+                                               payment_metadata,
                                                incoming_cltv_expiry: hop_data.outgoing_cltv_value,
                                                phantom_shared_secret,
                                        }
@@ -2298,6 +2315,7 @@ where
 
                                        PendingHTLCRouting::ReceiveKeysend {
                                                payment_preimage,
+                                               payment_metadata,
                                                incoming_cltv_expiry: hop_data.outgoing_cltv_value,
                                        }
                                } else {
@@ -2477,7 +2495,14 @@ where
                                                // hopefully an attacker trying to path-trace payments cannot make this occur
                                                // on a small/per-node/per-channel scale.
                                                if !chan.is_live() { // channel_disabled
-                                                       break Some(("Forwarding channel is not in a ready state.", 0x1000 | 20, chan_update_opt));
+                                                       // If the channel_update we're going to return is disabled (i.e. the
+                                                       // peer has been disabled for some time), return `channel_disabled`,
+                                                       // otherwise return `temporary_channel_failure`.
+                                                       if chan_update_opt.as_ref().map(|u| u.contents.flags & 2 == 2).unwrap_or(false) {
+                                                               break Some(("Forwarding channel has been disconnected for some time.", 0x1000 | 20, chan_update_opt));
+                                                       } else {
+                                                               break Some(("Forwarding channel is not in a ready state.", 0x1000 | 7, chan_update_opt));
+                                                       }
                                                }
                                                if *outgoing_amt_msat < chan.get_counterparty_htlc_minimum_msat() { // amount_below_minimum
                                                        break Some(("HTLC amount was below the htlc_minimum_msat", 0x1000 | 11, chan_update_opt));
@@ -2602,11 +2627,18 @@ where
                log_trace!(self.logger, "Generating channel update for channel {}", log_bytes!(chan.channel_id()));
                let were_node_one = self.our_network_pubkey.serialize()[..] < chan.get_counterparty_node_id().serialize()[..];
 
+               let enabled = chan.is_usable() && match chan.channel_update_status() {
+                       ChannelUpdateStatus::Enabled => true,
+                       ChannelUpdateStatus::DisabledStaged(_) => true,
+                       ChannelUpdateStatus::Disabled => false,
+                       ChannelUpdateStatus::EnabledStaged(_) => false,
+               };
+
                let unsigned = msgs::UnsignedChannelUpdate {
                        chain_hash: self.genesis_hash,
                        short_channel_id,
                        timestamp: chan.get_update_time_counter(),
-                       flags: (!were_node_one) as u8 | ((!chan.is_live() as u8) << 1),
+                       flags: (!were_node_one) as u8 | ((!enabled as u8) << 1),
                        cltv_expiry_delta: chan.get_cltv_expiry_delta(),
                        htlc_minimum_msat: chan.get_counterparty_htlc_minimum_msat(),
                        htlc_maximum_msat: chan.get_announced_htlc_max_msat(),
@@ -2797,6 +2829,11 @@ where
                self.pending_outbound_payments.test_add_new_pending_payment(payment_hash, recipient_onion, payment_id, route, None, &self.entropy_source, best_block_height)
        }
 
+       #[cfg(test)]
+       pub(crate) fn test_set_payment_metadata(&self, payment_id: PaymentId, new_payment_metadata: Option<Vec<u8>>) {
+               self.pending_outbound_payments.test_set_payment_metadata(payment_id, new_payment_metadata);
+       }
+
 
        /// Signals that no further retries for the given payment should occur. Useful if you have a
        /// pending outbound payment with retries remaining, but wish to stop retrying the payment before
@@ -3381,7 +3418,7 @@ where
                                                }
                                        }
                                } else {
-                                       for forward_info in pending_forwards.drain(..) {
+                                       'next_forwardable_htlc: for forward_info in pending_forwards.drain(..) {
                                                match forward_info {
                                                        HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
                                                                prev_short_channel_id, prev_htlc_id, prev_funding_outpoint, prev_user_channel_id,
@@ -3389,13 +3426,19 @@ where
                                                                        routing, incoming_shared_secret, payment_hash, incoming_amt_msat, outgoing_amt_msat, ..
                                                                }
                                                        }) => {
-                                                               let (cltv_expiry, onion_payload, payment_data, phantom_shared_secret) = match routing {
-                                                                       PendingHTLCRouting::Receive { payment_data, incoming_cltv_expiry, phantom_shared_secret } => {
+                                                               let (cltv_expiry, onion_payload, payment_data, phantom_shared_secret, mut onion_fields) = match routing {
+                                                                       PendingHTLCRouting::Receive { payment_data, payment_metadata, incoming_cltv_expiry, phantom_shared_secret } => {
                                                                                let _legacy_hop_data = Some(payment_data.clone());
-                                                                               (incoming_cltv_expiry, OnionPayload::Invoice { _legacy_hop_data }, Some(payment_data), phantom_shared_secret)
+                                                                               let onion_fields =
+                                                                                       RecipientOnionFields { payment_secret: Some(payment_data.payment_secret), payment_metadata };
+                                                                               (incoming_cltv_expiry, OnionPayload::Invoice { _legacy_hop_data },
+                                                                                       Some(payment_data), phantom_shared_secret, onion_fields)
+                                                                       },
+                                                                       PendingHTLCRouting::ReceiveKeysend { payment_preimage, payment_metadata, incoming_cltv_expiry } => {
+                                                                               let onion_fields = RecipientOnionFields { payment_secret: None, payment_metadata };
+                                                                               (incoming_cltv_expiry, OnionPayload::Spontaneous(payment_preimage),
+                                                                                       None, None, onion_fields)
                                                                        },
-                                                                       PendingHTLCRouting::ReceiveKeysend { payment_preimage, incoming_cltv_expiry } =>
-                                                                               (incoming_cltv_expiry, OnionPayload::Spontaneous(payment_preimage), None, None),
                                                                        _ => {
                                                                                panic!("short_channel_id == 0 should imply any pending_forward entries are of type Receive");
                                                                        }
@@ -3420,8 +3463,11 @@ where
                                                                        onion_payload,
                                                                };
 
+                                                               let mut committed_to_claimable = false;
+
                                                                macro_rules! fail_htlc {
                                                                        ($htlc: expr, $payment_hash: expr) => {
+                                                                               debug_assert!(!committed_to_claimable);
                                                                                let mut htlc_msat_height_data = $htlc.value.to_be_bytes().to_vec();
                                                                                htlc_msat_height_data.extend_from_slice(
                                                                                        &self.best_block.read().unwrap().height().to_be_bytes(),
@@ -3436,6 +3482,7 @@ where
                                                                                        HTLCFailReason::reason(0x4000 | 15, htlc_msat_height_data),
                                                                                        HTLCDestination::FailedPayment { payment_hash: $payment_hash },
                                                                                ));
+                                                                               continue 'next_forwardable_htlc;
                                                                        }
                                                                }
                                                                let phantom_shared_secret = claimable_htlc.prev_hop.phantom_shared_secret;
@@ -3457,15 +3504,28 @@ where
                                                                                let mut claimable_payments = self.claimable_payments.lock().unwrap();
                                                                                if claimable_payments.pending_claiming_payments.contains_key(&payment_hash) {
                                                                                        fail_htlc!(claimable_htlc, payment_hash);
-                                                                                       continue
                                                                                }
-                                                                               let (_, ref mut htlcs) = claimable_payments.claimable_htlcs.entry(payment_hash)
-                                                                                       .or_insert_with(|| (purpose(), Vec::new()));
+                                                                               let ref mut claimable_payment = claimable_payments.claimable_payments
+                                                                                       .entry(payment_hash)
+                                                                                       // Note that if we insert here we MUST NOT fail_htlc!()
+                                                                                       .or_insert_with(|| {
+                                                                                               committed_to_claimable = true;
+                                                                                               ClaimablePayment {
+                                                                                                       purpose: purpose(), htlcs: Vec::new(), onion_fields: None,
+                                                                                               }
+                                                                                       });
+                                                                               if let Some(earlier_fields) = &mut claimable_payment.onion_fields {
+                                                                                       if earlier_fields.check_merge(&mut onion_fields).is_err() {
+                                                                                               fail_htlc!(claimable_htlc, payment_hash);
+                                                                                       }
+                                                                               } else {
+                                                                                       claimable_payment.onion_fields = Some(onion_fields);
+                                                                               }
+                                                                               let ref mut htlcs = &mut claimable_payment.htlcs;
                                                                                if htlcs.len() == 1 {
                                                                                        if let OnionPayload::Spontaneous(_) = htlcs[0].onion_payload {
                                                                                                log_trace!(self.logger, "Failing new HTLC with payment_hash {} as we already had an existing keysend HTLC with the same payment hash", log_bytes!(payment_hash.0));
                                                                                                fail_htlc!(claimable_htlc, payment_hash);
-                                                                                               continue
                                                                                        }
                                                                                }
                                                                                let mut total_value = claimable_htlc.sender_intended_value;
@@ -3494,6 +3554,9 @@ where
                                                                                                log_bytes!(payment_hash.0));
                                                                                        fail_htlc!(claimable_htlc, payment_hash);
                                                                                } else if total_value >= $payment_data.total_msat {
+                                                                                       #[allow(unused_assignments)] {
+                                                                                               committed_to_claimable = true;
+                                                                                       }
                                                                                        let prev_channel_id = prev_funding_outpoint.to_channel_id();
                                                                                        htlcs.push(claimable_htlc);
                                                                                        let amount_msat = htlcs.iter().map(|htlc| htlc.value).sum();
@@ -3506,6 +3569,7 @@ where
                                                                                                via_channel_id: Some(prev_channel_id),
                                                                                                via_user_channel_id: Some(prev_user_channel_id),
                                                                                                claim_deadline: Some(earliest_expiry - HTLC_FAIL_BACK_BUFFER),
+                                                                                               onion_fields: claimable_payment.onion_fields.clone(),
                                                                                        });
                                                                                        payment_claimable_generated = true;
                                                                                } else {
@@ -3513,6 +3577,9 @@ where
                                                                                        // payment value yet, wait until we receive more
                                                                                        // MPP parts.
                                                                                        htlcs.push(claimable_htlc);
+                                                                                       #[allow(unused_assignments)] {
+                                                                                               committed_to_claimable = true;
+                                                                                       }
                                                                                }
                                                                                payment_claimable_generated
                                                                        }}
@@ -3535,7 +3602,6 @@ where
                                                                                                        Err(()) => {
                                                                                                                log_trace!(self.logger, "Failing new HTLC with payment_hash {} as payment verification failed", log_bytes!(payment_hash.0));
                                                                                                                fail_htlc!(claimable_htlc, payment_hash);
-                                                                                                               continue
                                                                                                        }
                                                                                                };
                                                                                                if let Some(min_final_cltv_expiry_delta) = min_final_cltv_expiry_delta {
@@ -3544,7 +3610,6 @@ where
                                                                                                                log_trace!(self.logger, "Failing new HTLC with payment_hash {} as its CLTV expiry was too soon (had {}, earliest expected {})",
                                                                                                                        log_bytes!(payment_hash.0), cltv_expiry, expected_min_expiry_height);
                                                                                                                fail_htlc!(claimable_htlc, payment_hash);
-                                                                                                               continue;
                                                                                                        }
                                                                                                }
                                                                                                check_total_value!(payment_data, payment_preimage);
@@ -3553,15 +3618,18 @@ where
                                                                                                let mut claimable_payments = self.claimable_payments.lock().unwrap();
                                                                                                if claimable_payments.pending_claiming_payments.contains_key(&payment_hash) {
                                                                                                        fail_htlc!(claimable_htlc, payment_hash);
-                                                                                                       continue
                                                                                                }
-                                                                                               match claimable_payments.claimable_htlcs.entry(payment_hash) {
+                                                                                               match claimable_payments.claimable_payments.entry(payment_hash) {
                                                                                                        hash_map::Entry::Vacant(e) => {
                                                                                                                let amount_msat = claimable_htlc.value;
                                                                                                                claimable_htlc.total_value_received = Some(amount_msat);
                                                                                                                let claim_deadline = Some(claimable_htlc.cltv_expiry - HTLC_FAIL_BACK_BUFFER);
                                                                                                                let purpose = events::PaymentPurpose::SpontaneousPayment(preimage);
-                                                                                                               e.insert((purpose.clone(), vec![claimable_htlc]));
+                                                                                                               e.insert(ClaimablePayment {
+                                                                                                                       purpose: purpose.clone(),
+                                                                                                                       onion_fields: Some(onion_fields.clone()),
+                                                                                                                       htlcs: vec![claimable_htlc],
+                                                                                                               });
                                                                                                                let prev_channel_id = prev_funding_outpoint.to_channel_id();
                                                                                                                new_events.push(events::Event::PaymentClaimable {
                                                                                                                        receiver_node_id: Some(receiver_node_id),
@@ -3571,6 +3639,7 @@ where
                                                                                                                        via_channel_id: Some(prev_channel_id),
                                                                                                                        via_user_channel_id: Some(prev_user_channel_id),
                                                                                                                        claim_deadline,
+                                                                                                                       onion_fields: Some(onion_fields),
                                                                                                                });
                                                                                                        },
                                                                                                        hash_map::Entry::Occupied(_) => {
@@ -3585,7 +3654,6 @@ where
                                                                                if payment_data.is_none() {
                                                                                        log_trace!(self.logger, "Failing new keysend HTLC with payment_hash {} because we already have an inbound payment with the same payment hash", log_bytes!(payment_hash.0));
                                                                                        fail_htlc!(claimable_htlc, payment_hash);
-                                                                                       continue
                                                                                };
                                                                                let payment_data = payment_data.unwrap();
                                                                                if inbound_payment.get().payment_secret != payment_data.payment_secret {
@@ -3756,27 +3824,39 @@ where
                                                }
 
                                                match chan.channel_update_status() {
-                                                       ChannelUpdateStatus::Enabled if !chan.is_live() => chan.set_channel_update_status(ChannelUpdateStatus::DisabledStaged),
-                                                       ChannelUpdateStatus::Disabled if chan.is_live() => chan.set_channel_update_status(ChannelUpdateStatus::EnabledStaged),
-                                                       ChannelUpdateStatus::DisabledStaged if chan.is_live() => chan.set_channel_update_status(ChannelUpdateStatus::Enabled),
-                                                       ChannelUpdateStatus::EnabledStaged if !chan.is_live() => chan.set_channel_update_status(ChannelUpdateStatus::Disabled),
-                                                       ChannelUpdateStatus::DisabledStaged if !chan.is_live() => {
-                                                               if let Ok(update) = self.get_channel_update_for_broadcast(&chan) {
-                                                                       pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
-                                                                               msg: update
-                                                                       });
+                                                       ChannelUpdateStatus::Enabled if !chan.is_live() => chan.set_channel_update_status(ChannelUpdateStatus::DisabledStaged(0)),
+                                                       ChannelUpdateStatus::Disabled if chan.is_live() => chan.set_channel_update_status(ChannelUpdateStatus::EnabledStaged(0)),
+                                                       ChannelUpdateStatus::DisabledStaged(_) if chan.is_live()
+                                                               => chan.set_channel_update_status(ChannelUpdateStatus::Enabled),
+                                                       ChannelUpdateStatus::EnabledStaged(_) if !chan.is_live()
+                                                               => chan.set_channel_update_status(ChannelUpdateStatus::Disabled),
+                                                       ChannelUpdateStatus::DisabledStaged(mut n) if !chan.is_live() => {
+                                                               n += 1;
+                                                               if n >= DISABLE_GOSSIP_TICKS {
+                                                                       chan.set_channel_update_status(ChannelUpdateStatus::Disabled);
+                                                                       if let Ok(update) = self.get_channel_update_for_broadcast(&chan) {
+                                                                               pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
+                                                                                       msg: update
+                                                                               });
+                                                                       }
+                                                                       should_persist = NotifyOption::DoPersist;
+                                                               } else {
+                                                                       chan.set_channel_update_status(ChannelUpdateStatus::DisabledStaged(n));
                                                                }
-                                                               should_persist = NotifyOption::DoPersist;
-                                                               chan.set_channel_update_status(ChannelUpdateStatus::Disabled);
                                                        },
-                                                       ChannelUpdateStatus::EnabledStaged if chan.is_live() => {
-                                                               if let Ok(update) = self.get_channel_update_for_broadcast(&chan) {
-                                                                       pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
-                                                                               msg: update
-                                                                       });
+                                                       ChannelUpdateStatus::EnabledStaged(mut n) if chan.is_live() => {
+                                                               n += 1;
+                                                               if n >= ENABLE_GOSSIP_TICKS {
+                                                                       chan.set_channel_update_status(ChannelUpdateStatus::Enabled);
+                                                                       if let Ok(update) = self.get_channel_update_for_broadcast(&chan) {
+                                                                               pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
+                                                                                       msg: update
+                                                                               });
+                                                                       }
+                                                                       should_persist = NotifyOption::DoPersist;
+                                                               } else {
+                                                                       chan.set_channel_update_status(ChannelUpdateStatus::EnabledStaged(n));
                                                                }
-                                                               should_persist = NotifyOption::DoPersist;
-                                                               chan.set_channel_update_status(ChannelUpdateStatus::Enabled);
                                                        },
                                                        _ => {},
                                                }
@@ -3818,24 +3898,27 @@ where
                                }
                        }
 
-                       self.claimable_payments.lock().unwrap().claimable_htlcs.retain(|payment_hash, (_, htlcs)| {
-                               if htlcs.is_empty() {
+                       self.claimable_payments.lock().unwrap().claimable_payments.retain(|payment_hash, payment| {
+                               if payment.htlcs.is_empty() {
                                        // This should be unreachable
                                        debug_assert!(false);
                                        return false;
                                }
-                               if let OnionPayload::Invoice { .. } = htlcs[0].onion_payload {
+                               if let OnionPayload::Invoice { .. } = payment.htlcs[0].onion_payload {
                                        // Check if we've received all the parts we need for an MPP (the value of the parts adds to total_msat).
                                        // In this case we're not going to handle any timeouts of the parts here.
                                        // This condition determining whether the MPP is complete here must match
                                        // exactly the condition used in `process_pending_htlc_forwards`.
-                                       if htlcs[0].total_msat <= htlcs.iter().fold(0, |total, htlc| total + htlc.sender_intended_value) {
+                                       if payment.htlcs[0].total_msat <= payment.htlcs.iter()
+                                               .fold(0, |total, htlc| total + htlc.sender_intended_value)
+                                       {
                                                return true;
-                                       } else if htlcs.into_iter().any(|htlc| {
+                                       } else if payment.htlcs.iter_mut().any(|htlc| {
                                                htlc.timer_ticks += 1;
                                                return htlc.timer_ticks >= MPP_TIMEOUT_TICKS
                                        }) {
-                                               timed_out_mpp_htlcs.extend(htlcs.drain(..).map(|htlc: ClaimableHTLC| (htlc.prev_hop, *payment_hash)));
+                                               timed_out_mpp_htlcs.extend(payment.htlcs.drain(..)
+                                                       .map(|htlc: ClaimableHTLC| (htlc.prev_hop, *payment_hash)));
                                                return false;
                                        }
                                }
@@ -3890,9 +3973,9 @@ where
        pub fn fail_htlc_backwards_with_reason(&self, payment_hash: &PaymentHash, failure_code: FailureCode) {
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
 
-               let removed_source = self.claimable_payments.lock().unwrap().claimable_htlcs.remove(payment_hash);
-               if let Some((_, mut sources)) = removed_source {
-                       for htlc in sources.drain(..) {
+               let removed_source = self.claimable_payments.lock().unwrap().claimable_payments.remove(payment_hash);
+               if let Some(payment) = removed_source {
+                       for htlc in payment.htlcs {
                                let reason = self.get_htlc_fail_reason_from_failure_code(failure_code, &htlc);
                                let source = HTLCSource::PreviousHopData(htlc.prev_hop);
                                let receiver = HTLCDestination::FailedPayment { payment_hash: *payment_hash };
@@ -4069,9 +4152,9 @@ where
 
                let mut sources = {
                        let mut claimable_payments = self.claimable_payments.lock().unwrap();
-                       if let Some((payment_purpose, sources)) = claimable_payments.claimable_htlcs.remove(&payment_hash) {
+                       if let Some(payment) = claimable_payments.claimable_payments.remove(&payment_hash) {
                                let mut receiver_node_id = self.our_network_pubkey;
-                               for htlc in sources.iter() {
+                               for htlc in payment.htlcs.iter() {
                                        if htlc.prev_hop.phantom_shared_secret.is_some() {
                                                let phantom_pubkey = self.node_signer.get_node_id(Recipient::PhantomNode)
                                                        .expect("Failed to get node_id for phantom node recipient");
@@ -4081,15 +4164,15 @@ where
                                }
 
                                let dup_purpose = claimable_payments.pending_claiming_payments.insert(payment_hash,
-                                       ClaimingPayment { amount_msat: sources.iter().map(|source| source.value).sum(),
-                                       payment_purpose, receiver_node_id,
+                                       ClaimingPayment { amount_msat: payment.htlcs.iter().map(|source| source.value).sum(),
+                                       payment_purpose: payment.purpose, receiver_node_id,
                                });
                                if dup_purpose.is_some() {
                                        debug_assert!(false, "Shouldn't get a duplicate pending claim event ever");
                                        log_error!(self.logger, "Got a duplicate pending claimable event on payment hash {}! Please report this bug",
                                                log_bytes!(payment_hash.0));
                                }
-                               sources
+                               payment.htlcs
                        } else { return; }
                };
                debug_assert!(!sources.is_empty());
@@ -6161,8 +6244,8 @@ where
                }
 
                if let Some(height) = height_opt {
-                       self.claimable_payments.lock().unwrap().claimable_htlcs.retain(|payment_hash, (_, htlcs)| {
-                               htlcs.retain(|htlc| {
+                       self.claimable_payments.lock().unwrap().claimable_payments.retain(|payment_hash, payment| {
+                               payment.htlcs.retain(|htlc| {
                                        // If height is approaching the number of blocks we think it takes us to get
                                        // our commitment transaction confirmed before the HTLC expires, plus the
                                        // number of blocks we generally consider it to take to do a commitment update,
@@ -6177,7 +6260,7 @@ where
                                                false
                                        } else { true }
                                });
-                               !htlcs.is_empty() // Only retain this entry if htlcs has at least one entry.
+                               !payment.htlcs.is_empty() // Only retain this entry if htlcs has at least one entry.
                        });
 
                        let mut intercepted_htlcs = self.pending_intercepted_htlcs.lock().unwrap();
@@ -6762,10 +6845,12 @@ impl_writeable_tlv_based_enum!(PendingHTLCRouting,
                (0, payment_data, required),
                (1, phantom_shared_secret, option),
                (2, incoming_cltv_expiry, required),
+               (3, payment_metadata, option),
        },
        (2, ReceiveKeysend) => {
                (0, payment_preimage, required),
                (2, incoming_cltv_expiry, required),
+               (3, payment_metadata, option),
        },
 ;);
 
@@ -7098,14 +7183,16 @@ where
                let pending_outbound_payments = self.pending_outbound_payments.pending_outbound_payments.lock().unwrap();
 
                let mut htlc_purposes: Vec<&events::PaymentPurpose> = Vec::new();
-               (claimable_payments.claimable_htlcs.len() as u64).write(writer)?;
-               for (payment_hash, (purpose, previous_hops)) in claimable_payments.claimable_htlcs.iter() {
+               let mut htlc_onion_fields: Vec<&_> = Vec::new();
+               (claimable_payments.claimable_payments.len() as u64).write(writer)?;
+               for (payment_hash, payment) in claimable_payments.claimable_payments.iter() {
                        payment_hash.write(writer)?;
-                       (previous_hops.len() as u64).write(writer)?;
-                       for htlc in previous_hops.iter() {
+                       (payment.htlcs.len() as u64).write(writer)?;
+                       for htlc in payment.htlcs.iter() {
                                htlc.write(writer)?;
                        }
-                       htlc_purposes.push(purpose);
+                       htlc_purposes.push(&payment.purpose);
+                       htlc_onion_fields.push(&payment.onion_fields);
                }
 
                let mut monitor_update_blocked_actions_per_peer = None;
@@ -7220,6 +7307,7 @@ where
                        (7, self.fake_scid_rand_bytes, required),
                        (9, htlc_purposes, vec_type),
                        (11, self.probing_cookie_secret, required),
+                       (13, htlc_onion_fields, optional_vec),
                });
 
                Ok(())
@@ -7598,6 +7686,7 @@ where
                let mut fake_scid_rand_bytes: Option<[u8; 32]> = None;
                let mut probing_cookie_secret: Option<[u8; 32]> = None;
                let mut claimable_htlc_purposes = None;
+               let mut claimable_htlc_onion_fields = None;
                let mut pending_claiming_payments = Some(HashMap::new());
                let mut monitor_update_blocked_actions_per_peer = Some(Vec::new());
                read_tlv_fields!(reader, {
@@ -7610,6 +7699,7 @@ where
                        (7, fake_scid_rand_bytes, option),
                        (9, claimable_htlc_purposes, vec_type),
                        (11, probing_cookie_secret, option),
+                       (13, claimable_htlc_onion_fields, optional_vec),
                });
                if fake_scid_rand_bytes.is_none() {
                        fake_scid_rand_bytes = Some(args.entropy_source.get_secure_random_bytes());
@@ -7673,6 +7763,7 @@ where
                                                                                session_privs: [session_priv_bytes].iter().map(|a| *a).collect(),
                                                                                payment_hash: htlc.payment_hash,
                                                                                payment_secret: None, // only used for retries, and we'll never retry on startup
+                                                                               payment_metadata: None, // only used for retries, and we'll never retry on startup
                                                                                keysend_preimage: None, // only used for retries, and we'll never retry on startup
                                                                                pending_amt_msat: path_amt,
                                                                                pending_fee_msat: Some(path_fee),
@@ -7757,22 +7848,39 @@ where
                let inbound_pmt_key_material = args.node_signer.get_inbound_payment_key_material();
                let expanded_inbound_key = inbound_payment::ExpandedKey::new(&inbound_pmt_key_material);
 
-               let mut claimable_htlcs = HashMap::with_capacity(claimable_htlcs_list.len());
-               if let Some(mut purposes) = claimable_htlc_purposes {
+               let mut claimable_payments = HashMap::with_capacity(claimable_htlcs_list.len());
+               if let Some(purposes) = claimable_htlc_purposes {
                        if purposes.len() != claimable_htlcs_list.len() {
                                return Err(DecodeError::InvalidValue);
                        }
-                       for (purpose, (payment_hash, previous_hops)) in purposes.drain(..).zip(claimable_htlcs_list.drain(..)) {
-                               claimable_htlcs.insert(payment_hash, (purpose, previous_hops));
+                       if let Some(onion_fields) = claimable_htlc_onion_fields {
+                               if onion_fields.len() != claimable_htlcs_list.len() {
+                                       return Err(DecodeError::InvalidValue);
+                               }
+                               for (purpose, (onion, (payment_hash, htlcs))) in
+                                       purposes.into_iter().zip(onion_fields.into_iter().zip(claimable_htlcs_list.into_iter()))
+                               {
+                                       let existing_payment = claimable_payments.insert(payment_hash, ClaimablePayment {
+                                               purpose, htlcs, onion_fields: onion,
+                                       });
+                                       if existing_payment.is_some() { return Err(DecodeError::InvalidValue); }
+                               }
+                       } else {
+                               for (purpose, (payment_hash, htlcs)) in purposes.into_iter().zip(claimable_htlcs_list.into_iter()) {
+                                       let existing_payment = claimable_payments.insert(payment_hash, ClaimablePayment {
+                                               purpose, htlcs, onion_fields: None,
+                                       });
+                                       if existing_payment.is_some() { return Err(DecodeError::InvalidValue); }
+                               }
                        }
                } else {
                        // LDK versions prior to 0.0.107 did not write a `pending_htlc_purposes`, but do
                        // include a `_legacy_hop_data` in the `OnionPayload`.
-                       for (payment_hash, previous_hops) in claimable_htlcs_list.drain(..) {
-                               if previous_hops.is_empty() {
+                       for (payment_hash, htlcs) in claimable_htlcs_list.drain(..) {
+                               if htlcs.is_empty() {
                                        return Err(DecodeError::InvalidValue);
                                }
-                               let purpose = match &previous_hops[0].onion_payload {
+                               let purpose = match &htlcs[0].onion_payload {
                                        OnionPayload::Invoice { _legacy_hop_data } => {
                                                if let Some(hop_data) = _legacy_hop_data {
                                                        events::PaymentPurpose::InvoicePayment {
@@ -7793,7 +7901,9 @@ where
                                        OnionPayload::Spontaneous(payment_preimage) =>
                                                events::PaymentPurpose::SpontaneousPayment(*payment_preimage),
                                };
-                               claimable_htlcs.insert(payment_hash, (purpose, previous_hops));
+                               claimable_payments.insert(payment_hash, ClaimablePayment {
+                                       purpose, htlcs, onion_fields: None,
+                               });
                        }
                }
 
@@ -7845,17 +7955,17 @@ where
 
                for (_, monitor) in args.channel_monitors.iter() {
                        for (payment_hash, payment_preimage) in monitor.get_stored_preimages() {
-                               if let Some((payment_purpose, claimable_htlcs)) = claimable_htlcs.remove(&payment_hash) {
+                               if let Some(payment) = claimable_payments.remove(&payment_hash) {
                                        log_info!(args.logger, "Re-claiming HTLCs with payment hash {} as we've released the preimage to a ChannelMonitor!", log_bytes!(payment_hash.0));
                                        let mut claimable_amt_msat = 0;
                                        let mut receiver_node_id = Some(our_network_pubkey);
-                                       let phantom_shared_secret = claimable_htlcs[0].prev_hop.phantom_shared_secret;
+                                       let phantom_shared_secret = payment.htlcs[0].prev_hop.phantom_shared_secret;
                                        if phantom_shared_secret.is_some() {
                                                let phantom_pubkey = args.node_signer.get_node_id(Recipient::PhantomNode)
                                                        .expect("Failed to get node_id for phantom node recipient");
                                                receiver_node_id = Some(phantom_pubkey)
                                        }
-                                       for claimable_htlc in claimable_htlcs {
+                                       for claimable_htlc in payment.htlcs {
                                                claimable_amt_msat += claimable_htlc.value;
 
                                                // Add a holding-cell claim of the payment to the Channel, which should be
@@ -7889,7 +7999,7 @@ where
                                        pending_events_read.push(events::Event::PaymentClaimed {
                                                receiver_node_id,
                                                payment_hash,
-                                               purpose: payment_purpose,
+                                               purpose: payment.purpose,
                                                amount_msat: claimable_amt_msat,
                                        });
                                }
@@ -7920,7 +8030,7 @@ where
                        pending_intercepted_htlcs: Mutex::new(pending_intercepted_htlcs.unwrap()),
 
                        forward_htlcs: Mutex::new(forward_htlcs),
-                       claimable_payments: Mutex::new(ClaimablePayments { claimable_htlcs, pending_claiming_payments: pending_claiming_payments.unwrap() }),
+                       claimable_payments: Mutex::new(ClaimablePayments { claimable_payments, pending_claiming_payments: pending_claiming_payments.unwrap() }),
                        outbound_scid_aliases: Mutex::new(outbound_scid_aliases),
                        id_to_peer: Mutex::new(id_to_peer),
                        short_to_chan_info: FairRwLock::new(short_to_chan_info),
index b4a5c5ae52d5282135b6e664819c2b5962ea0b2c..cf375603b37b5290aaf10250218f4b66cce0678a 100644 (file)
 //!     (see [BOLT-2](https://github.com/lightning/bolts/blob/master/02-peer-protocol.md) for more information).
 //! - `OnionMessages` - requires/supports forwarding onion messages
 //!     (see [BOLT-7](https://github.com/lightning/bolts/pull/759/files) for more information).
-//!     TODO: update link
+//     TODO: update link
 //! - `ChannelType` - node supports the channel_type field in open/accept
 //!     (see [BOLT-2](https://github.com/lightning/bolts/blob/master/02-peer-protocol.md) for more information).
 //! - `SCIDPrivacy` - supply channel aliases for routing
 //!     (see [BOLT-2](https://github.com/lightning/bolts/blob/master/02-peer-protocol.md) for more information).
+//! - `PaymentMetadata` - include additional data in invoices which is passed to recipients in the
+//!      onion.
+//!      (see [BOLT-11](https://github.com/lightning/bolts/blob/master/11-payment-encoding.md) for
+//!      more).
+//! - `ZeroConf` - supports accepting HTLCs and using channels prior to funding confirmation
+//!      (see
+//!      [BOLT-2](https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#the-channel_ready-message)
+//!      for more info).
 //! - `Keysend` - send funds to a node without an invoice
 //!     (see the [`Keysend` feature assignment proposal](https://github.com/lightning/bolts/issues/605#issuecomment-606679798) for more information).
 //! - `AnchorsZeroFeeHtlcTx` - requires/supports that commitment transactions include anchor outputs
-//!   and HTLC transactions are pre-signed with zero fee (see
-//!   [BOLT-3](https://github.com/lightning/bolts/blob/master/03-transactions.md) for more
-//!   information).
+//!     and HTLC transactions are pre-signed with zero fee (see
+//!     [BOLT-3](https://github.com/lightning/bolts/blob/master/03-transactions.md) for more
+//!     information).
 //!
 //! [BOLT #9]: https://github.com/lightning/bolts/blob/master/09-features.md
 //! [messages]: crate::ln::msgs
@@ -160,6 +168,14 @@ mod sealed {
                VariableLengthOnion | PaymentSecret,
                // Byte 2
                BasicMPP,
+               // Byte 3
+               ,
+               // Byte 4
+               ,
+               // Byte 5
+               ,
+               // Byte 6
+               PaymentMetadata,
        ]);
        define_context!(OfferContext, []);
        define_context!(InvoiceRequestContext, []);
@@ -259,6 +275,7 @@ mod sealed {
                                        }
 
                                        flags[Self::BYTE_OFFSET] |= Self::REQUIRED_MASK;
+                                       flags[Self::BYTE_OFFSET] &= !Self::OPTIONAL_MASK;
                                }
 
                                /// Sets the feature's optional (odd) bit in the given flags.
@@ -376,12 +393,16 @@ mod sealed {
        define_feature!(47, SCIDPrivacy, [InitContext, NodeContext, ChannelTypeContext],
                "Feature flags for only forwarding with SCID aliasing. Called `option_scid_alias` in the BOLTs",
                set_scid_privacy_optional, set_scid_privacy_required, supports_scid_privacy, requires_scid_privacy);
+       define_feature!(49, PaymentMetadata, [InvoiceContext],
+               "Feature flags for payment metadata in invoices.", set_payment_metadata_optional,
+               set_payment_metadata_required, supports_payment_metadata, requires_payment_metadata);
        define_feature!(51, ZeroConf, [InitContext, NodeContext, ChannelTypeContext],
                "Feature flags for accepting channels with zero confirmations. Called `option_zeroconf` in the BOLTs",
                set_zero_conf_optional, set_zero_conf_required, supports_zero_conf, requires_zero_conf);
        define_feature!(55, Keysend, [NodeContext],
                "Feature flags for keysend payments.", set_keysend_optional, set_keysend_required,
                supports_keysend, requires_keysend);
+       // Note: update the module-level docs when a new feature bit is added!
 
        #[cfg(test)]
        define_feature!(123456789, UnknownFeature,
@@ -885,13 +906,13 @@ mod tests {
        #[test]
        fn convert_to_context_with_unknown_flags() {
                // Ensure the `from` context has fewer known feature bytes than the `to` context.
-               assert!(<sealed::InvoiceContext as sealed::Context>::KNOWN_FEATURE_MASK.len() <
-                       <sealed::NodeContext as sealed::Context>::KNOWN_FEATURE_MASK.len());
-               let mut invoice_features = InvoiceFeatures::empty();
-               invoice_features.set_unknown_feature_optional();
-               assert!(invoice_features.supports_unknown_bits());
-               let node_features: NodeFeatures = invoice_features.to_context();
-               assert!(!node_features.supports_unknown_bits());
+               assert!(<sealed::ChannelContext as sealed::Context>::KNOWN_FEATURE_MASK.len() <
+                       <sealed::InvoiceContext as sealed::Context>::KNOWN_FEATURE_MASK.len());
+               let mut channel_features = ChannelFeatures::empty();
+               channel_features.set_unknown_feature_optional();
+               assert!(channel_features.supports_unknown_bits());
+               let invoice_features: InvoiceFeatures = channel_features.to_context_internal();
+               assert!(!invoice_features.supports_unknown_bits());
        }
 
        #[test]
index f8ccce05847a7d426a5b05118c25670febb4d6b0..b695e20a8dd22ba589f2e3cf0a0bd8f4dbad0ddc 100644 (file)
@@ -137,6 +137,34 @@ pub enum ConnectStyle {
 }
 
 impl ConnectStyle {
+       pub fn skips_blocks(&self) -> bool {
+               match self {
+                       ConnectStyle::BestBlockFirst => false,
+                       ConnectStyle::BestBlockFirstSkippingBlocks => true,
+                       ConnectStyle::BestBlockFirstReorgsOnlyTip => true,
+                       ConnectStyle::TransactionsFirst => false,
+                       ConnectStyle::TransactionsFirstSkippingBlocks => true,
+                       ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks => true,
+                       ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks => true,
+                       ConnectStyle::TransactionsFirstReorgsOnlyTip => true,
+                       ConnectStyle::FullBlockViaListen => false,
+               }
+       }
+
+       pub fn updates_best_block_first(&self) -> bool {
+               match self {
+                       ConnectStyle::BestBlockFirst => true,
+                       ConnectStyle::BestBlockFirstSkippingBlocks => true,
+                       ConnectStyle::BestBlockFirstReorgsOnlyTip => true,
+                       ConnectStyle::TransactionsFirst => false,
+                       ConnectStyle::TransactionsFirstSkippingBlocks => false,
+                       ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks => false,
+                       ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks => false,
+                       ConnectStyle::TransactionsFirstReorgsOnlyTip => false,
+                       ConnectStyle::FullBlockViaListen => false,
+               }
+       }
+
        fn random_style() -> ConnectStyle {
                #[cfg(feature = "std")] {
                        use core::hash::{BuildHasher, Hasher};
@@ -164,12 +192,7 @@ impl ConnectStyle {
 }
 
 pub fn connect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, depth: u32) -> BlockHash {
-       let skip_intermediaries = match *node.connect_style.borrow() {
-               ConnectStyle::BestBlockFirstSkippingBlocks|ConnectStyle::TransactionsFirstSkippingBlocks|
-                       ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks|ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks|
-                       ConnectStyle::BestBlockFirstReorgsOnlyTip|ConnectStyle::TransactionsFirstReorgsOnlyTip => true,
-               _ => false,
-       };
+       let skip_intermediaries = node.connect_style.borrow().skips_blocks();
 
        let height = node.best_block_info().1 + 1;
        let mut block = Block {
@@ -2034,21 +2057,26 @@ pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_p
                        let events_2 = node.node.get_and_clear_pending_events();
                        if payment_claimable_expected {
                                assert_eq!(events_2.len(), 1);
-                               match events_2[0] {
-                                       Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, ref via_channel_id, ref via_user_channel_id, claim_deadline } => {
+                               match &events_2[0] {
+                                       Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat,
+                                               receiver_node_id, ref via_channel_id, ref via_user_channel_id,
+                                               claim_deadline, onion_fields,
+                                       } => {
                                                assert_eq!(our_payment_hash, *payment_hash);
                                                assert_eq!(node.node.get_our_node_id(), receiver_node_id.unwrap());
+                                               assert!(onion_fields.is_some());
                                                match &purpose {
                                                        PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                                                assert_eq!(expected_preimage, *payment_preimage);
                                                                assert_eq!(our_payment_secret.unwrap(), *payment_secret);
+                                                               assert_eq!(Some(*payment_secret), onion_fields.as_ref().unwrap().payment_secret);
                                                        },
                                                        PaymentPurpose::SpontaneousPayment(payment_preimage) => {
                                                                assert_eq!(expected_preimage.unwrap(), *payment_preimage);
                                                                assert!(our_payment_secret.is_none());
                                                        },
                                                }
-                                               assert_eq!(amount_msat, recv_value);
+                                               assert_eq!(*amount_msat, recv_value);
                                                assert!(node.node.list_channels().iter().any(|details| details.channel_id == via_channel_id.unwrap()));
                                                assert!(node.node.list_channels().iter().any(|details| details.user_channel_id == via_user_channel_id.unwrap()));
                                                assert!(claim_deadline.unwrap() > node.best_block_info().1);
@@ -2530,6 +2558,8 @@ pub enum HTLCType { NONE, TIMEOUT, SUCCESS }
 /// also fail.
 pub fn test_txn_broadcast<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, chan: &(msgs::ChannelUpdate, msgs::ChannelUpdate, [u8; 32], Transaction), commitment_tx: Option<Transaction>, has_htlc_tx: HTLCType) -> Vec<Transaction>  {
        let mut node_txn = node.tx_broadcaster.txn_broadcasted.lock().unwrap();
+       let mut txn_seen = HashSet::new();
+       node_txn.retain(|tx| txn_seen.insert(tx.txid()));
        assert!(node_txn.len() >= if commitment_tx.is_some() { 0 } else { 1 } + if has_htlc_tx == HTLCType::NONE { 0 } else { 1 });
 
        let mut res = Vec::with_capacity(2);
@@ -2593,22 +2623,23 @@ pub fn test_revoked_htlc_claim_txn_broadcast<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>
 
 pub fn check_preimage_claim<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, prev_txn: &Vec<Transaction>) -> Vec<Transaction>  {
        let mut node_txn = node.tx_broadcaster.txn_broadcasted.lock().unwrap();
+       let mut txn_seen = HashSet::new();
+       node_txn.retain(|tx| txn_seen.insert(tx.txid()));
 
-       assert!(node_txn.len() >= 1);
-       assert_eq!(node_txn[0].input.len(), 1);
        let mut found_prev = false;
-
-       for tx in prev_txn {
-               if node_txn[0].input[0].previous_output.txid == tx.txid() {
-                       check_spends!(node_txn[0], tx);
-                       let mut iter = node_txn[0].input[0].witness.iter();
-                       iter.next().expect("expected 3 witness items");
-                       iter.next().expect("expected 3 witness items");
-                       assert!(iter.next().expect("expected 3 witness items").len() > 106); // must spend an htlc output
-                       assert_eq!(tx.input.len(), 1); // must spend a commitment tx
-
-                       found_prev = true;
-                       break;
+       for prev_tx in prev_txn {
+               for tx in &*node_txn {
+                       if tx.input[0].previous_output.txid == prev_tx.txid() {
+                               check_spends!(tx, prev_tx);
+                               let mut iter = tx.input[0].witness.iter();
+                               iter.next().expect("expected 3 witness items");
+                               iter.next().expect("expected 3 witness items");
+                               assert!(iter.next().expect("expected 3 witness items").len() > 106); // must spend an htlc output
+                               assert_eq!(tx.input.len(), 1); // must spend a commitment tx
+
+                               found_prev = true;
+                               break;
+                       }
                }
        }
        assert!(found_prev);
@@ -2724,10 +2755,9 @@ macro_rules! handle_chan_reestablish_msgs {
                        }
 
                        let mut had_channel_update = false; // ChannelUpdate may be now or later, but not both
-                       if let Some(&MessageSendEvent::SendChannelUpdate { ref node_id, ref msg }) = msg_events.get(idx) {
+                       if let Some(&MessageSendEvent::SendChannelUpdate { ref node_id, .. }) = msg_events.get(idx) {
                                assert_eq!(*node_id, $dst_node.node.get_our_node_id());
                                idx += 1;
-                               assert_eq!(msg.contents.flags & 2, 0); // "disabled" flag must not be set as we just reconnected.
                                had_channel_update = true;
                        }
 
@@ -2771,10 +2801,9 @@ macro_rules! handle_chan_reestablish_msgs {
                                }
                        }
 
-                       if let Some(&MessageSendEvent::SendChannelUpdate { ref node_id, ref msg }) = msg_events.get(idx) {
+                       if let Some(&MessageSendEvent::SendChannelUpdate { ref node_id, .. }) = msg_events.get(idx) {
                                assert_eq!(*node_id, $dst_node.node.get_our_node_id());
                                idx += 1;
-                               assert_eq!(msg.contents.flags & 2, 0); // "disabled" flag must not be set as we just reconnected.
                                assert!(!had_channel_update);
                        }
 
index 5d07edc2e5360e53a474b98559832bcb5a5ac2c6..17f8e1597b7c0b2c34f064656c8ebf3de91e0a50 100644 (file)
@@ -21,7 +21,7 @@ use crate::chain::keysinterface::{ChannelSigner, EcdsaChannelSigner, EntropySour
 use crate::events::{Event, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentPurpose, ClosureReason, HTLCDestination, PaymentFailureReason};
 use crate::ln::{PaymentPreimage, PaymentSecret, PaymentHash};
 use crate::ln::channel::{commitment_tx_base_weight, COMMITMENT_TX_WEIGHT_PER_HTLC, CONCURRENT_INBOUND_HTLC_FEE_BUFFER, FEE_SPIKE_BUFFER_FEE_INCREASE_MULTIPLE, MIN_AFFORDABLE_HTLC_COUNT};
-use crate::ln::channelmanager::{self, PaymentId, RAACommitmentOrder, PaymentSendFailure, RecipientOnionFields, BREAKDOWN_TIMEOUT, MIN_CLTV_EXPIRY_DELTA};
+use crate::ln::channelmanager::{self, PaymentId, RAACommitmentOrder, PaymentSendFailure, RecipientOnionFields, BREAKDOWN_TIMEOUT, ENABLE_GOSSIP_TICKS, DISABLE_GOSSIP_TICKS, MIN_CLTV_EXPIRY_DELTA};
 use crate::ln::channel::{Channel, ChannelError};
 use crate::ln::{chan_utils, onion_utils};
 use crate::ln::chan_utils::{OFFERED_HTLC_SCRIPT_WEIGHT, htlc_success_tx_weight, htlc_timeout_tx_weight, HTLCOutputInCommitment};
@@ -2390,8 +2390,8 @@ fn channel_monitor_network_test() {
 }
 
 #[test]
-fn test_justice_tx() {
-       // Test justice txn built on revoked HTLC-Success tx, against both sides
+fn test_justice_tx_htlc_timeout() {
+       // Test justice txn built on revoked HTLC-Timeout tx, against both sides
        let mut alice_config = UserConfig::default();
        alice_config.channel_handshake_config.announced_channel = true;
        alice_config.channel_handshake_limits.force_announced_channel_preference = false;
@@ -2407,7 +2407,6 @@ fn test_justice_tx() {
        let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
        let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &user_cfgs);
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
-       *nodes[0].connect_style.borrow_mut() = ConnectStyle::FullBlockViaListen;
        // Create some new channels:
        let chan_5 = create_announced_chan_between_nodes(&nodes, 0, 1);
 
@@ -2431,7 +2430,6 @@ fn test_justice_tx() {
                        let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
                        assert_eq!(node_txn.len(), 1); // ChannelMonitor: penalty tx
                        assert_eq!(node_txn[0].input.len(), 2); // We should claim the revoked output and the HTLC output
-
                        check_spends!(node_txn[0], revoked_local_txn[0]);
                        node_txn.swap_remove(0);
                }
@@ -2450,17 +2448,30 @@ fn test_justice_tx() {
                test_revoked_htlc_claim_txn_broadcast(&nodes[1], node_txn[1].clone(), revoked_local_txn[0].clone());
        }
        get_announce_close_broadcast_events(&nodes, 0, 1);
-
        assert_eq!(nodes[0].node.list_channels().len(), 0);
        assert_eq!(nodes[1].node.list_channels().len(), 0);
+}
 
-       // We test justice_tx build by A on B's revoked HTLC-Success tx
+#[test]
+fn test_justice_tx_htlc_success() {
+       // Test justice txn built on revoked HTLC-Success tx, against both sides
+       let mut alice_config = UserConfig::default();
+       alice_config.channel_handshake_config.announced_channel = true;
+       alice_config.channel_handshake_limits.force_announced_channel_preference = false;
+       alice_config.channel_handshake_config.our_to_self_delay = 6 * 24 * 5;
+       let mut bob_config = UserConfig::default();
+       bob_config.channel_handshake_config.announced_channel = true;
+       bob_config.channel_handshake_limits.force_announced_channel_preference = false;
+       bob_config.channel_handshake_config.our_to_self_delay = 6 * 24 * 3;
+       let user_cfgs = [Some(alice_config), Some(bob_config)];
+       let mut chanmon_cfgs = create_chanmon_cfgs(2);
+       chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
+       chanmon_cfgs[1].keys_manager.disable_revocation_policy_check = true;
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &user_cfgs);
+       let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
        // Create some new channels:
        let chan_6 = create_announced_chan_between_nodes(&nodes, 0, 1);
-       {
-               let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
-               node_txn.clear();
-       }
 
        // A pending HTLC which will be revoked:
        let payment_preimage_4 = route_payment(&nodes[0], &vec!(&nodes[1])[..], 3000000).0;
@@ -2638,8 +2649,7 @@ fn claim_htlc_outputs_single_tx() {
                connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
                assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 
-               let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
-               assert_eq!(node_txn.len(), 7);
+               let mut node_txn = nodes[1].tx_broadcaster.txn_broadcast();
 
                // Check the pair local commitment and HTLC-timeout broadcast due to HTLC expiration
                assert_eq!(node_txn[0].input.len(), 1);
@@ -2649,19 +2659,22 @@ fn claim_htlc_outputs_single_tx() {
                assert_eq!(witness_script.len(), OFFERED_HTLC_SCRIPT_WEIGHT); //Spending an offered htlc output
                check_spends!(node_txn[1], node_txn[0]);
 
-               // Justice transactions are indices 2-3-4
+               // Filter out any non justice transactions.
+               node_txn.retain(|tx| tx.input[0].previous_output.txid == revoked_local_txn[0].txid());
+               assert!(node_txn.len() > 3);
+
+               assert_eq!(node_txn[0].input.len(), 1);
+               assert_eq!(node_txn[1].input.len(), 1);
                assert_eq!(node_txn[2].input.len(), 1);
-               assert_eq!(node_txn[3].input.len(), 1);
-               assert_eq!(node_txn[4].input.len(), 1);
 
+               check_spends!(node_txn[0], revoked_local_txn[0]);
+               check_spends!(node_txn[1], revoked_local_txn[0]);
                check_spends!(node_txn[2], revoked_local_txn[0]);
-               check_spends!(node_txn[3], revoked_local_txn[0]);
-               check_spends!(node_txn[4], revoked_local_txn[0]);
 
                let mut witness_lens = BTreeSet::new();
+               witness_lens.insert(node_txn[0].input[0].witness.last().unwrap().len());
+               witness_lens.insert(node_txn[1].input[0].witness.last().unwrap().len());
                witness_lens.insert(node_txn[2].input[0].witness.last().unwrap().len());
-               witness_lens.insert(node_txn[3].input[0].witness.last().unwrap().len());
-               witness_lens.insert(node_txn[4].input[0].witness.last().unwrap().len());
                assert_eq!(witness_lens.len(), 3);
                assert_eq!(*witness_lens.iter().skip(0).next().unwrap(), 77); // revoked to_local
                assert_eq!(*witness_lens.iter().skip(1).next().unwrap(), OFFERED_HTLC_SCRIPT_WEIGHT); // revoked offered HTLC
@@ -2669,9 +2682,9 @@ fn claim_htlc_outputs_single_tx() {
 
                // Finally, mine the penalty transactions and check that we get an HTLC failure after
                // ANTI_REORG_DELAY confirmations.
+               mine_transaction(&nodes[1], &node_txn[0]);
+               mine_transaction(&nodes[1], &node_txn[1]);
                mine_transaction(&nodes[1], &node_txn[2]);
-               mine_transaction(&nodes[1], &node_txn[3]);
-               mine_transaction(&nodes[1], &node_txn[4]);
                connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
                expect_payment_failed!(nodes[1], payment_hash_2, false);
        }
@@ -2970,25 +2983,20 @@ fn do_test_htlc_on_chain_timeout(connect_style: ConnectStyle) {
 
        // Broadcast timeout transaction by B on received output from C's commitment tx on B's chain
        // Verify that B's ChannelManager is able to detect that HTLC is timeout by its own tx and react backward in consequence
-       connect_blocks(&nodes[1], 200 - nodes[2].best_block_info().1);
        mine_transaction(&nodes[1], &commitment_tx[0]);
-       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
-       let timeout_tx;
-       {
-               let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
-               assert_eq!(node_txn.len(), 3); // 2 (local commitment tx + HTLC-timeout), 1 timeout tx
-
-               check_spends!(node_txn[2], commitment_tx[0]);
-               assert_eq!(node_txn[2].clone().input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
-
-               check_spends!(node_txn[0], chan_2.3);
-               check_spends!(node_txn[1], node_txn[0]);
-               assert_eq!(node_txn[0].clone().input[0].witness.last().unwrap().len(), 71);
-               assert_eq!(node_txn[1].clone().input[0].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT);
-
-               timeout_tx = node_txn[2].clone();
-               node_txn.clear();
-       }
+       check_closed_event(&nodes[1], 1, ClosureReason::CommitmentTxConfirmed, false);
+       connect_blocks(&nodes[1], 200 - nodes[2].best_block_info().1);
+       let timeout_tx = {
+               let mut txn = nodes[1].tx_broadcaster.txn_broadcast();
+               if nodes[1].connect_style.borrow().skips_blocks() {
+                       assert_eq!(txn.len(), 1);
+               } else {
+                       assert_eq!(txn.len(), 3); // Two extra fee bumps for timeout transaction
+               }
+               txn.iter().for_each(|tx| check_spends!(tx, commitment_tx[0]));
+               assert_eq!(txn[0].clone().input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
+               txn.remove(0)
+       };
 
        mine_transaction(&nodes[1], &timeout_tx);
        check_added_monitors!(nodes[1], 1);
@@ -3453,7 +3461,7 @@ fn test_htlc_ignore_latest_remote_commitment() {
 
        let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(node_txn.len(), 3);
-       assert_eq!(node_txn[0], node_txn[1]);
+       assert_eq!(node_txn[0].txid(), node_txn[1].txid());
 
        let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 };
        connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[0].clone(), node_txn[1].clone()]});
@@ -7100,8 +7108,9 @@ fn test_announce_disable_channels() {
        nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
        nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 
-       nodes[0].node.timer_tick_occurred(); // Enabled -> DisabledStaged
-       nodes[0].node.timer_tick_occurred(); // DisabledStaged -> Disabled
+       for _ in 0..DISABLE_GOSSIP_TICKS + 1 {
+               nodes[0].node.timer_tick_occurred();
+       }
        let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(msg_events.len(), 3);
        let mut chans_disabled = HashMap::new();
@@ -7141,7 +7150,9 @@ fn test_announce_disable_channels() {
        nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[2]);
        handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 
-       nodes[0].node.timer_tick_occurred();
+       for _ in 0..ENABLE_GOSSIP_TICKS {
+               nodes[0].node.timer_tick_occurred();
+       }
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
        nodes[0].node.timer_tick_occurred();
        let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -7309,17 +7320,21 @@ fn test_bump_penalty_txn_on_revoked_htlcs() {
        check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        connect_blocks(&nodes[1], 49); // Confirm blocks until the HTLC expires (note CLTV was explicitly 50 above)
 
-       let revoked_htlc_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
-       assert_eq!(revoked_htlc_txn.len(), 2);
+       let revoked_htlc_txn = {
+               let txn = nodes[1].tx_broadcaster.unique_txn_broadcast();
+               assert_eq!(txn.len(), 2);
 
-       assert_eq!(revoked_htlc_txn[0].input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
-       assert_eq!(revoked_htlc_txn[0].input.len(), 1);
-       check_spends!(revoked_htlc_txn[0], revoked_local_txn[0]);
+               assert_eq!(txn[0].input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
+               assert_eq!(txn[0].input.len(), 1);
+               check_spends!(txn[0], revoked_local_txn[0]);
 
-       assert_eq!(revoked_htlc_txn[1].input.len(), 1);
-       assert_eq!(revoked_htlc_txn[1].input[0].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT);
-       assert_eq!(revoked_htlc_txn[1].output.len(), 1);
-       check_spends!(revoked_htlc_txn[1], revoked_local_txn[0]);
+               assert_eq!(txn[1].input.len(), 1);
+               assert_eq!(txn[1].input[0].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT);
+               assert_eq!(txn[1].output.len(), 1);
+               check_spends!(txn[1], revoked_local_txn[0]);
+
+               txn
+       };
 
        // Broadcast set of revoked txn on A
        let hash_128 = connect_blocks(&nodes[0], 40);
@@ -7495,7 +7510,7 @@ fn test_bump_penalty_txn_on_remote_commitment() {
        assert_ne!(feerate_preimage, 0);
 
        // After exhaustion of height timer, new bumped claim txn should have been broadcast, check it
-       connect_blocks(&nodes[1], 15);
+       connect_blocks(&nodes[1], 1);
        {
                let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
                assert_eq!(node_txn.len(), 1);
@@ -8491,11 +8506,11 @@ fn test_concurrent_monitor_claim() {
        watchtower_alice.chain_monitor.block_connected(&block, CHAN_CONFIRM_DEPTH + 1 + TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS);
 
        // Watchtower Alice should have broadcast a commitment/HTLC-timeout
-       {
-               let mut txn = chanmon_cfgs[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
+       let alice_state = {
+               let mut txn = chanmon_cfgs[0].tx_broadcaster.txn_broadcast();
                assert_eq!(txn.len(), 2);
-               txn.clear();
-       }
+               txn.remove(0)
+       };
 
        // Copy ChainMonitor to simulate watchtower Bob and make it receive a commitment update first.
        let chain_source = test_utils::TestChainSource::new(Network::Testnet);
@@ -8546,19 +8561,21 @@ fn test_concurrent_monitor_claim() {
        // Watchtower Bob should have broadcast a commitment/HTLC-timeout
        let bob_state_y;
        {
-               let mut txn = chanmon_cfgs[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
+               let mut txn = chanmon_cfgs[0].tx_broadcaster.txn_broadcast();
                assert_eq!(txn.len(), 2);
-               bob_state_y = txn[0].clone();
-               txn.clear();
+               bob_state_y = txn.remove(0);
        };
 
        // We confirm Bob's state Y on Alice, she should broadcast a HTLC-timeout
        let header = BlockHeader { version: 0x20000000, prev_blockhash: BlockHash::all_zeros(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 };
        watchtower_alice.chain_monitor.block_connected(&Block { header, txdata: vec![bob_state_y.clone()] }, CHAN_CONFIRM_DEPTH + 2 + TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS);
        {
-               let htlc_txn = chanmon_cfgs[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
-               assert_eq!(htlc_txn.len(), 1);
+               let htlc_txn = chanmon_cfgs[0].tx_broadcaster.txn_broadcast();
+               assert_eq!(htlc_txn.len(), 2);
                check_spends!(htlc_txn[0], bob_state_y);
+               // Alice doesn't clean up the old HTLC claim since it hasn't seen a conflicting spend for
+               // it. However, she should, because it now has an invalid parent.
+               check_spends!(htlc_txn[1], alice_state);
        }
 }
 
@@ -9231,7 +9248,7 @@ fn do_test_tx_confirmed_skipping_blocks_immediate_broadcast(test_height_before_t
                // We should broadcast an HTLC transaction spending our funding transaction first
                let spending_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
                assert_eq!(spending_txn.len(), 2);
-               assert_eq!(spending_txn[0], node_txn[0]);
+               assert_eq!(spending_txn[0].txid(), node_txn[0].txid());
                check_spends!(spending_txn[1], node_txn[0]);
                // We should also generate a SpendableOutputs event with the to_self output (as its
                // timelock is up).
index 0c6d6f2b804bccd8cfb651c24995bf4959998a91..2d15876bf95ccad75fb90233b135ffd0fabd17ad 100644 (file)
@@ -19,14 +19,14 @@ use crate::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
 use crate::ln::msgs;
 use crate::ln::msgs::MAX_VALUE_MSAT;
 use crate::util::chacha20::ChaCha20;
-use crate::util::crypto::hkdf_extract_expand_thrice;
+use crate::util::crypto::hkdf_extract_expand_4x;
 use crate::util::errors::APIError;
 use crate::util::logger::Logger;
 
-use core::convert::TryInto;
+use core::convert::{TryFrom, TryInto};
 use core::ops::Deref;
 
-const IV_LEN: usize = 16;
+pub(crate) const IV_LEN: usize = 16;
 const METADATA_LEN: usize = 16;
 const METADATA_KEY_LEN: usize = 32;
 const AMT_MSAT_LEN: usize = 8;
@@ -48,6 +48,8 @@ pub struct ExpandedKey {
        /// The key used to authenticate a user-provided payment hash and metadata as previously
        /// registered with LDK.
        user_pmt_hash_key: [u8; 32],
+       /// The base key used to derive signing keys and authenticate messages for BOLT 12 Offers.
+       offers_base_key: [u8; 32],
 }
 
 impl ExpandedKey {
@@ -55,14 +57,76 @@ impl ExpandedKey {
        ///
        /// It is recommended to cache this value and not regenerate it for each new inbound payment.
        pub fn new(key_material: &KeyMaterial) -> ExpandedKey {
-               let (metadata_key, ldk_pmt_hash_key, user_pmt_hash_key) =
-                       hkdf_extract_expand_thrice(b"LDK Inbound Payment Key Expansion", &key_material.0);
+               let (metadata_key, ldk_pmt_hash_key, user_pmt_hash_key, offers_base_key) =
+                       hkdf_extract_expand_4x(b"LDK Inbound Payment Key Expansion", &key_material.0);
                Self {
                        metadata_key,
                        ldk_pmt_hash_key,
                        user_pmt_hash_key,
+                       offers_base_key,
                }
        }
+
+       /// Returns an [`HmacEngine`] used to construct [`Offer::metadata`].
+       ///
+       /// [`Offer::metadata`]: crate::offers::offer::Offer::metadata
+       #[allow(unused)]
+       pub(crate) fn hmac_for_offer(
+               &self, nonce: Nonce, iv_bytes: &[u8; IV_LEN]
+       ) -> HmacEngine<Sha256> {
+               let mut hmac = HmacEngine::<Sha256>::new(&self.offers_base_key);
+               hmac.input(iv_bytes);
+               hmac.input(&nonce.0);
+               hmac
+       }
+}
+
+/// A 128-bit number used only once.
+///
+/// Needed when constructing [`Offer::metadata`] and deriving [`Offer::signing_pubkey`] from
+/// [`ExpandedKey`]. Must not be reused for any other derivation without first hashing.
+///
+/// [`Offer::metadata`]: crate::offers::offer::Offer::metadata
+/// [`Offer::signing_pubkey`]: crate::offers::offer::Offer::signing_pubkey
+#[allow(unused)]
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub(crate) struct Nonce(pub(crate) [u8; Self::LENGTH]);
+
+impl Nonce {
+       /// Number of bytes in the nonce.
+       pub const LENGTH: usize = 16;
+
+       /// Creates a `Nonce` from the given [`EntropySource`].
+       pub fn from_entropy_source<ES: Deref>(entropy_source: ES) -> Self
+       where
+               ES::Target: EntropySource,
+       {
+               let mut bytes = [0u8; Self::LENGTH];
+               let rand_bytes = entropy_source.get_secure_random_bytes();
+               bytes.copy_from_slice(&rand_bytes[..Self::LENGTH]);
+
+               Nonce(bytes)
+       }
+
+       /// Returns a slice of the underlying bytes of size [`Nonce::LENGTH`].
+       pub fn as_slice(&self) -> &[u8] {
+               &self.0
+       }
+}
+
+impl TryFrom<&[u8]> for Nonce {
+       type Error = ();
+
+       fn try_from(bytes: &[u8]) -> Result<Self, ()> {
+               if bytes.len() != Self::LENGTH {
+                       return Err(());
+               }
+
+               let mut copied_bytes = [0u8; Self::LENGTH];
+               copied_bytes.copy_from_slice(bytes);
+
+               Ok(Self(copied_bytes))
+       }
 }
 
 enum Method {
index c589d57405b0d52c8b6ebfbe59343d457ae0ce46..5b2b29dbb76f11675a2269cc3ebafb527eda063d 100644 (file)
@@ -30,9 +30,7 @@ use crate::ln::msgs::ChannelMessageHandler;
 use crate::util::config::UserConfig;
 #[cfg(anchors)]
 use crate::util::crypto::sign;
-#[cfg(anchors)]
 use crate::util::ser::Writeable;
-#[cfg(anchors)]
 use crate::util::test_utils;
 
 #[cfg(anchors)]
@@ -1324,20 +1322,29 @@ fn test_revoked_counterparty_htlc_tx_balances() {
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
        check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
-       let revoked_htlc_success_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
-
-       assert_eq!(revoked_htlc_success_txn.len(), 1);
-       assert_eq!(revoked_htlc_success_txn[0].input.len(), 1);
-       assert_eq!(revoked_htlc_success_txn[0].input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
-       check_spends!(revoked_htlc_success_txn[0], revoked_local_txn[0]);
+       let revoked_htlc_success = {
+               let mut txn = nodes[1].tx_broadcaster.txn_broadcast();
+               assert_eq!(txn.len(), 1);
+               assert_eq!(txn[0].input.len(), 1);
+               assert_eq!(txn[0].input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
+               check_spends!(txn[0], revoked_local_txn[0]);
+               txn.pop().unwrap()
+       };
 
        connect_blocks(&nodes[1], TEST_FINAL_CLTV);
-       let revoked_htlc_timeout_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
-       assert_eq!(revoked_htlc_timeout_txn.len(), 1);
-       check_spends!(revoked_htlc_timeout_txn[0], revoked_local_txn[0]);
-       assert_ne!(revoked_htlc_success_txn[0].input[0].previous_output, revoked_htlc_timeout_txn[0].input[0].previous_output);
-       assert_eq!(revoked_htlc_success_txn[0].lock_time.0, 0);
-       assert_ne!(revoked_htlc_timeout_txn[0].lock_time.0, 0);
+       let revoked_htlc_timeout = {
+               let mut txn = nodes[1].tx_broadcaster.unique_txn_broadcast();
+               assert_eq!(txn.len(), 2);
+               if txn[0].input[0].previous_output == revoked_htlc_success.input[0].previous_output {
+                       txn.remove(1)
+               } else {
+                       txn.remove(0)
+               }
+       };
+       check_spends!(revoked_htlc_timeout, revoked_local_txn[0]);
+       assert_ne!(revoked_htlc_success.input[0].previous_output, revoked_htlc_timeout.input[0].previous_output);
+       assert_eq!(revoked_htlc_success.lock_time.0, 0);
+       assert_ne!(revoked_htlc_timeout.lock_time.0, 0);
 
        // A will generate justice tx from B's revoked commitment/HTLC tx
        mine_transaction(&nodes[0], &revoked_local_txn[0]);
@@ -1369,10 +1376,10 @@ fn test_revoked_counterparty_htlc_tx_balances() {
        assert_eq!(as_balances,
                sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances()));
 
-       mine_transaction(&nodes[0], &revoked_htlc_success_txn[0]);
+       mine_transaction(&nodes[0], &revoked_htlc_success);
        let as_htlc_claim_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(as_htlc_claim_tx.len(), 2);
-       check_spends!(as_htlc_claim_tx[0], revoked_htlc_success_txn[0]);
+       check_spends!(as_htlc_claim_tx[0], revoked_htlc_success);
        check_spends!(as_htlc_claim_tx[1], revoked_local_txn[0]); // A has to generate a new claim for the remaining revoked
                                                                  // outputs (which no longer includes the spent HTLC output)
 
@@ -1381,7 +1388,7 @@ fn test_revoked_counterparty_htlc_tx_balances() {
 
        assert_eq!(as_htlc_claim_tx[0].output.len(), 1);
        fuzzy_assert_eq(as_htlc_claim_tx[0].output[0].value,
-               3_000 - chan_feerate * (revoked_htlc_success_txn[0].weight() + as_htlc_claim_tx[0].weight()) as u64 / 1000);
+               3_000 - chan_feerate * (revoked_htlc_success.weight() + as_htlc_claim_tx[0].weight()) as u64 / 1000);
 
        mine_transaction(&nodes[0], &as_htlc_claim_tx[0]);
        assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations {
@@ -1423,7 +1430,7 @@ fn test_revoked_counterparty_htlc_tx_balances() {
                }]),
                sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances()));
 
-       connect_blocks(&nodes[0], revoked_htlc_timeout_txn[0].lock_time.0 - nodes[0].best_block_info().1);
+       connect_blocks(&nodes[0], revoked_htlc_timeout.lock_time.0 - nodes[0].best_block_info().1);
        expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(&nodes[0],
                [HTLCDestination::FailedPayment { payment_hash: failed_payment_hash }]);
        // As time goes on A may split its revocation claim transaction into multiple.
@@ -1440,11 +1447,11 @@ fn test_revoked_counterparty_htlc_tx_balances() {
                check_spends!(tx, revoked_local_txn[0]);
        }
 
-       mine_transaction(&nodes[0], &revoked_htlc_timeout_txn[0]);
+       mine_transaction(&nodes[0], &revoked_htlc_timeout);
        let as_second_htlc_claim_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(as_second_htlc_claim_tx.len(), 2);
 
-       check_spends!(as_second_htlc_claim_tx[0], revoked_htlc_timeout_txn[0]);
+       check_spends!(as_second_htlc_claim_tx[0], revoked_htlc_timeout);
        check_spends!(as_second_htlc_claim_tx[1], revoked_local_txn[0]);
 
        // Connect blocks to finalize the HTLC resolution with the HTLC-Timeout transaction. In a
@@ -1695,6 +1702,233 @@ fn test_revoked_counterparty_aggregated_claims() {
        assert!(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty());
 }
 
+fn do_test_restored_packages_retry(check_old_monitor_retries_after_upgrade: bool) {
+       // Tests that we'll retry packages that were previously timelocked after we've restored them.
+       let persister;
+       let new_chain_monitor;
+       let node_deserialized;
+
+       let chanmon_cfgs = create_chanmon_cfgs(2);
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+       let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+       // Open a channel, lock in an HTLC, and immediately broadcast the commitment transaction. This
+       // ensures that the HTLC timeout package is held until we reach its expiration height.
+       let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 50_000_000);
+       route_payment(&nodes[0], &[&nodes[1]], 10_000_000);
+
+       nodes[0].node.force_close_broadcasting_latest_txn(&chan_id, &nodes[1].node.get_our_node_id()).unwrap();
+       check_added_monitors(&nodes[0], 1);
+       check_closed_broadcast(&nodes[0], 1, true);
+       check_closed_event(&nodes[0], 1, ClosureReason::HolderForceClosed, false);
+
+       let commitment_tx = {
+               let mut txn = nodes[0].tx_broadcaster.txn_broadcast();
+               assert_eq!(txn.len(), 1);
+               assert_eq!(txn[0].output.len(), 3);
+               check_spends!(txn[0], funding_tx);
+               txn.pop().unwrap()
+       };
+
+       mine_transaction(&nodes[0], &commitment_tx);
+
+       // Connect blocks until the HTLC's expiration is met, expecting a transaction broadcast.
+       connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1);
+       let htlc_timeout_tx = {
+               let mut txn = nodes[0].tx_broadcaster.txn_broadcast();
+               assert_eq!(txn.len(), 1);
+               check_spends!(txn[0], commitment_tx);
+               txn.pop().unwrap()
+       };
+
+       // Check that we can still rebroadcast these packages/transactions if we're upgrading from an
+       // old `ChannelMonitor` that did not exercise said rebroadcasting logic.
+       if check_old_monitor_retries_after_upgrade {
+               let serialized_monitor = hex::decode(
+                       "0101fffffffffffffffff9550f22c95100160014d5a9aa98b89acc215fc3d23d6fec0ad59ca3665f00002200204c5f18e5e95b184f34d02ba6de8a2a4e36ae3d4ec87299ad81f3284dc7195c6302d7dde8e10a5a22c9bd0d7ef5494d85683ac050253b917615d4f97af633f0a8e2035f5e9d58b4328566223c107d86cf853e6b9fae1d26ff6d969be0178d1423c4ea0016001467822698d782e8421ebdf96d010de99382b7ec2300160014caf6d80fe2bab80473b021f57588a9c384bf23170000000000000000000000004d49e5da0000000000000000000000000000002a0270b20ad0f2c2bb30a55590fc77778495bc1b38c96476901145dda57491237f0f74c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e00000022002034c0cc0ad0dd5fe61dcf7ef58f995e3d34f8dbd24aa2a6fae68fefe102bf025c21391732ce658e1fe167300bb689a81e7db5399b9ee4095e217b0e997e8dd3d17a0000000000000000004a002103adde8029d3ee281a32e9db929b39f503ff9d7e93cd308eb157955344dc6def84022103205087e2dc1f6b9937e887dfa712c5bdfa950b01dbda3ebac4c85efdde48ee6a04020090004752210307a78def56cba9fc4db22a25928181de538ee59ba1a475ae113af7790acd0db32103c21e841cbc0b48197d060c71e116c185fa0ac281b7d0aa5924f535154437ca3b52ae00000000000186a0ffffffffffff0291e7c0a3232fb8650a6b4089568a81062b48a768780e5a74bb4a4a74e33aec2c029d5760248ec86c4a76d9df8308555785a06a65472fb995f5b392d520bbd000650090c1c94b11625690c9d84c5daa67b6ad19fcc7f9f23e194384140b08fcab9e8e810000ffffffffffff000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000002391732ce658e1fe167300bb689a81e7db5399b9ee4095e217b0e997e8dd3d17a00000000000000010000000000009896800000005166687aadf862bd776c8fc18b8e9f8e20089714856ee233b3902a591d0d5f292505000000009c009900202d704fbfe342a9ff6eaca14d80a24aaed0e680bbbdd36157b6f2798c61d906910120f9fe5e552aa0fc45020f0505efde432a4e373e5d393863973a6899f8c26d33d10208000000000098968004494800210355f8d2238a322d16b602bd0ceaad5b01019fb055971eaadcc9b29226a4da6c23020500030241000408000001000000000006020000080800000000009896800a0400000046167c86cc0e598a6b541f7c9bf9ef17222e4a76f636e2d22185aeadd2b02d029c00000000000000000000000000000000000000000000000166687aadf862bd776c8fc18b8e9f8e20089714856ee233b3902a591d0d5f2925fffffffffffe01e3002004f8eda5676356f539169a8e9a1e86c7f125283328d6f4bded1b939b52a6a7e30108000000000000c299022103a1f98e85886df54add6908b4fc1ff515e44aedefe9eb9c02879c89994298fa79042103a650bf03971df0176c7b412247390ef717853e8bd487b204dccc2fe2078bb75206210390bbbcebe9f70ba5dfd98866a79f72f75e0a6ea550ef73b202dd87cd6477350a08210284152d57908488e666e872716a286eb670b3d06cbeebf3f2e4ad350e01ec5e5b0a2102295e2de39eb3dcc2882f8cc266df7882a8b6d2c32aa08799f49b693aad3be28e0c04000000fd0e00fd01fe002045cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d01080000000000009b5e0221035f5e9d58b4328566223c107d86cf853e6b9fae1d26ff6d969be0178d1423c4ea04210230fde9c031f487db95ff55b7c0acbe0c7c26a8d82615e9184416bd350101616706210225afb4e88eac8b47b67adeaf085f5eb5d37d936f56138f0848de3d104edf113208210208e4687a95c172b86b920c3bc5dbd5f023094ec2cb0abdb74f9b624f45740df90a2102d7dde8e10a5a22c9bd0d7ef5494d85683ac050253b917615d4f97af633f0a8e20c04000000fd0efd01193b00010102080000000000989680040400000051062066687aadf862bd776c8fc18b8e9f8e20089714856ee233b3902a591d0d5f2925080400000000417e2650c201383711eed2a7cb8652c3e77ee6a395e81849c5c222217ed68b333c0ca9f1e900662ae68a7359efa7ef9d90613f2a62f7c3ff90f8c25e2cc974c9d39c009900202d704fbfe342a9ff6eaca14d80a24aaed0e680bbbdd36157b6f2798c61d906910120f9fe5e552aa0fc45020f0505efde432a4e373e5d393863973a6899f8c26d33d10208000000000098968004494800210355f8d2238a322d16b602bd0ceaad5b01019fb055971eaadcc9b29226a4da6c23020500030241000408000001000000000006020000080800000000009896800a0400000046fffffffffffefffffffffffe000000000000000000000000000000000000000000000000ffe099e83ae3761c7f1b781d22613bd1f6977e9ad59fae12b3eba34462ee8a3d000000500000000000000002fd01da002045cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d01fd01840200000000010174c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e00000000000f55f9800310270000000000002200208309b406e3b96e76cde414fbb8f5159f5b25b24075656c6382cec797854d53495e9b0000000000002200204c5f18e5e95b184f34d02ba6de8a2a4e36ae3d4ec87299ad81f3284dc7195c6350c300000000000016001425df8ec4a074f80579fed67d4707d5ec8ed7e8d304004730440220671c9badf26bd3a1ebd2d17020c6be20587d7822530daacc52c28839875eaec602204b575a21729ed27311f6d79fdf6fe8702b0a798f7d842e39ede1b56f249a613401473044022016a0da36f70cbf5d889586af88f238982889dc161462c56557125c7acfcb69e9022036ae10c6cc8cbc3b27d9e9ef6babb556086585bc819f252208bd175286699fdd014752210307a78def56cba9fc4db22a25928181de538ee59ba1a475ae113af7790acd0db32103c21e841cbc0b48197d060c71e116c185fa0ac281b7d0aa5924f535154437ca3b52ae50c9222002040000000b03209452ca8c90d4c78928b80ec41398f2a890324d8ad6e6c81408a0cb9b8d977b070406030400020090fd02a1002045cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d01fd01840200000000010174c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e00000000000f55f9800310270000000000002200208309b406e3b96e76cde414fbb8f5159f5b25b24075656c6382cec797854d53495e9b0000000000002200204c5f18e5e95b184f34d02ba6de8a2a4e36ae3d4ec87299ad81f3284dc7195c6350c300000000000016001425df8ec4a074f80579fed67d4707d5ec8ed7e8d304004730440220671c9badf26bd3a1ebd2d17020c6be20587d7822530daacc52c28839875eaec602204b575a21729ed27311f6d79fdf6fe8702b0a798f7d842e39ede1b56f249a613401473044022016a0da36f70cbf5d889586af88f238982889dc161462c56557125c7acfcb69e9022036ae10c6cc8cbc3b27d9e9ef6babb556086585bc819f252208bd175286699fdd014752210307a78def56cba9fc4db22a25928181de538ee59ba1a475ae113af7790acd0db32103c21e841cbc0b48197d060c71e116c185fa0ac281b7d0aa5924f535154437ca3b52ae50c9222002040000000b03209452ca8c90d4c78928b80ec41398f2a890324d8ad6e6c81408a0cb9b8d977b0704cd01cb00c901c7002245cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d0001022102d7dde8e10a5a22c9bd0d7ef5494d85683ac050253b917615d4f97af633f0a8e204020090062b5e9b0000000000002200204c5f18e5e95b184f34d02ba6de8a2a4e36ae3d4ec87299ad81f3284dc7195c630821035f5e9d58b4328566223c107d86cf853e6b9fae1d26ff6d969be0178d1423c4ea0a200000000000000000000000004d49e5da0000000000000000000000000000002a0c0800000000000186a0000000000000000274c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e0000000000000001000000000022002034c0cc0ad0dd5fe61dcf7ef58f995e3d34f8dbd24aa2a6fae68fefe102bf025c45cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d000000000000000100000000002200208309b406e3b96e76cde414fbb8f5159f5b25b24075656c6382cec797854d5349010100160014d5a9aa98b89acc215fc3d23d6fec0ad59ca3665ffd027100fd01e6fd01e300080000fffffffffffe02080000000000009b5e0408000000000000c3500604000000fd08b0af002102d7dde8e10a5a22c9bd0d7ef5494d85683ac050253b917615d4f97af633f0a8e20221035f5e9d58b4328566223c107d86cf853e6b9fae1d26ff6d969be0178d1423c4ea04210230fde9c031f487db95ff55b7c0acbe0c7c26a8d82615e9184416bd350101616706210225afb4e88eac8b47b67adeaf085f5eb5d37d936f56138f0848de3d104edf113208210208e4687a95c172b86b920c3bc5dbd5f023094ec2cb0abdb74f9b624f45740df90acdcc00a8020000000174c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e00000000000f55f9800310270000000000002200208309b406e3b96e76cde414fbb8f5159f5b25b24075656c6382cec797854d53495e9b0000000000002200204c5f18e5e95b184f34d02ba6de8a2a4e36ae3d4ec87299ad81f3284dc7195c6350c300000000000016001425df8ec4a074f80579fed67d4707d5ec8ed7e8d350c92220022045cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d0c3c3b00010102080000000000989680040400000051062066687aadf862bd776c8fc18b8e9f8e20089714856ee233b3902a591d0d5f29250804000000000240671c9badf26bd3a1ebd2d17020c6be20587d7822530daacc52c28839875eaec64b575a21729ed27311f6d79fdf6fe8702b0a798f7d842e39ede1b56f249a613404010006407e2650c201383711eed2a7cb8652c3e77ee6a395e81849c5c222217ed68b333c0ca9f1e900662ae68a7359efa7ef9d90613f2a62f7c3ff90f8c25e2cc974c9d3010000000000000001010000000000000000090b2a953d93a124c600ecb1a0ccfed420169cdd37f538ad94a3e4e6318c93c14adf59cdfbb40bdd40950c9f8dd547d29d75a173e1376a7850743394c46dea2dfd01cefd01ca00fd017ffd017c00080000ffffffffffff0208000000000000c2990408000000000000c3500604000000fd08b0af002102295e2de39eb3dcc2882f8cc266df7882a8b6d2c32aa08799f49b693aad3be28e022103a1f98e85886df54add6908b4fc1ff515e44aedefe9eb9c02879c89994298fa79042103a650bf03971df0176c7b412247390ef717853e8bd487b204dccc2fe2078bb75206210390bbbcebe9f70ba5dfd98866a79f72f75e0a6ea550ef73b202dd87cd6477350a08210284152d57908488e666e872716a286eb670b3d06cbeebf3f2e4ad350e01ec5e5b0aa2a1007d020000000174c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e00000000000f55f9800299c2000000000000220020740e108cfbc93967b6ab242a351ebee7de51814cf78d366adefd78b10281f17e50c300000000000016001425df8ec4a074f80579fed67d4707d5ec8ed7e8d351c92220022004f8eda5676356f539169a8e9a1e86c7f125283328d6f4bded1b939b52a6a7e30c00024045cb2485594bb1ec08e7bb6af4f89c912bd53f006d7876ea956773e04a4aad4a40e2b8d4fc612102f0b54061b3c1239fb78783053e8e6f9d92b1b99f81ae9ec2040100060000fd019600b0af002103c21e841cbc0b48197d060c71e116c185fa0ac281b7d0aa5924f535154437ca3b02210270b20ad0f2c2bb30a55590fc77778495bc1b38c96476901145dda57491237f0f042103b4e59df102747edc3a3e2283b42b88a8c8218ffd0dcfb52f2524b371d64cadaa062103d902b7b8b3434076d2b210e912c76645048b71e28995aad227a465a65ccd817608210301e9a52f923c157941de4a7692e601f758660969dcf5abdb67817efe84cce2ef0202009004010106b7b600b0af00210307a78def56cba9fc4db22a25928181de538ee59ba1a475ae113af7790acd0db30221034d0f817cb19b4a3bd144b615459bd06cbab3b4bdc96d73e18549a992cee80e8104210380542b59a9679890cba529fe155a9508ef57dac7416d035b23666e3fb98c3814062103adde8029d3ee281a32e9db929b39f503ff9d7e93cd308eb157955344dc6def84082103205087e2dc1f6b9937e887dfa712c5bdfa950b01dbda3ebac4c85efdde48ee6a02020090082274c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e000000000287010108d30df34e3a1e00ecdd03a2c843db062479a81752c4dfd0cc4baef0f81e7bc7ef8820990daf8d8e8d30a3b4b08af12c9f5cd71e45c7238103e0c80ca13850862e4fd2c56b69b7195312518de1bfe9aed63c80bb7760d70b2a870d542d815895fd12423d11e2adb0cdf55d776dac8f487c9b3b7ea12f1b150eb15889cf41333ade465692bf1cdc360b9c2a19bf8c1ca4fed7639d8bc953d36c10d8c6c9a8c0a57608788979bcf145e61b308006896e21d03e92084f93bd78740c20639134a7a8fd019afd019600b0af002103c21e841cbc0b48197d060c71e116c185fa0ac281b7d0aa5924f535154437ca3b02210270b20ad0f2c2bb30a55590fc77778495bc1b38c96476901145dda57491237f0f042103b4e59df102747edc3a3e2283b42b88a8c8218ffd0dcfb52f2524b371d64cadaa062103d902b7b8b3434076d2b210e912c76645048b71e28995aad227a465a65ccd817608210301e9a52f923c157941de4a7692e601f758660969dcf5abdb67817efe84cce2ef0202009004010106b7b600b0af00210307a78def56cba9fc4db22a25928181de538ee59ba1a475ae113af7790acd0db30221034d0f817cb19b4a3bd144b615459bd06cbab3b4bdc96d73e18549a992cee80e8104210380542b59a9679890cba529fe155a9508ef57dac7416d035b23666e3fb98c3814062103adde8029d3ee281a32e9db929b39f503ff9d7e93cd308eb157955344dc6def84082103205087e2dc1f6b9937e887dfa712c5bdfa950b01dbda3ebac4c85efdde48ee6a02020090082274c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e000000000000000186a00000000000000000000000004d49e5da0000000000000000000000000000002a000000000000000001b77b61346a2a408afdb01743a2230cb36e55771a0790f67a0910e207fd223fc8000000000000000145cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d00000000041000080000000000989680020400000051160004000000510208000000000000000004040000000b000000000000000145cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d00000000b77b61346a2a408afdb01743a2230cb36e55771a0790f67a0910e207fd223fc80000005000000000000000000000000000000000000101300300050007010109210355f8d2238a322d16b602bd0ceaad5b01019fb055971eaadcc9b29226a4da6c230d000f020000",
+               ).unwrap();
+               reload_node!(nodes[0], &nodes[0].node.encode(), &[&serialized_monitor], persister, new_chain_monitor, node_deserialized);
+       }
+
+       // Connecting more blocks should result in the HTLC transactions being rebroadcast.
+       connect_blocks(&nodes[0], 6);
+       if check_old_monitor_retries_after_upgrade {
+               check_added_monitors(&nodes[0], 1);
+       }
+       {
+               let txn = nodes[0].tx_broadcaster.txn_broadcast();
+               if !nodes[0].connect_style.borrow().skips_blocks() {
+                       assert_eq!(txn.len(), 6);
+               } else {
+                       assert!(txn.len() < 6);
+               }
+               for tx in txn {
+                       assert_eq!(tx.input.len(), htlc_timeout_tx.input.len());
+                       assert_eq!(tx.output.len(), htlc_timeout_tx.output.len());
+                       assert_eq!(tx.input[0].previous_output, htlc_timeout_tx.input[0].previous_output);
+                       assert_eq!(tx.output[0], htlc_timeout_tx.output[0]);
+               }
+       }
+}
+
+#[test]
+fn test_restored_packages_retry() {
+       do_test_restored_packages_retry(false);
+       do_test_restored_packages_retry(true);
+}
+
+fn do_test_monitor_rebroadcast_pending_claims(anchors: bool) {
+       // Test that we will retry broadcasting pending claims for a force-closed channel on every
+       // `ChainMonitor::rebroadcast_pending_claims` call.
+       if anchors {
+               assert!(cfg!(anchors));
+       }
+       let secp = Secp256k1::new();
+       let mut chanmon_cfgs = create_chanmon_cfgs(2);
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let mut config = test_default_channel_config();
+       if anchors {
+               #[cfg(anchors)] {
+                       config.channel_handshake_config.negotiate_anchors_zero_fee_htlc_tx = true;
+               }
+       }
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[Some(config), Some(config)]);
+       let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+       let (_, _, _, chan_id, funding_tx) = create_chan_between_nodes_with_value(
+               &nodes[0], &nodes[1], 1_000_000, 500_000_000
+       );
+       const HTLC_AMT_MSAT: u64 = 1_000_000;
+       const HTLC_AMT_SAT: u64 = HTLC_AMT_MSAT / 1000;
+       route_payment(&nodes[0], &[&nodes[1]], HTLC_AMT_MSAT);
+
+       let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 1;
+
+       let commitment_txn = get_local_commitment_txn!(&nodes[0], &chan_id);
+       assert_eq!(commitment_txn.len(), if anchors { 1 /* commitment tx only */} else { 2 /* commitment and htlc timeout tx */ });
+       check_spends!(&commitment_txn[0], &funding_tx);
+       mine_transaction(&nodes[0], &commitment_txn[0]);
+       check_closed_broadcast!(&nodes[0], true);
+       check_closed_event(&nodes[0], 1, ClosureReason::CommitmentTxConfirmed, false);
+       check_added_monitors(&nodes[0], 1);
+
+       // Set up a helper closure we'll use throughout our test. We should only expect retries without
+       // bumps if fees have not increased after a block has been connected (assuming the height timer
+       // re-evaluates at every block) or after `ChainMonitor::rebroadcast_pending_claims` is called.
+       let mut prev_htlc_tx_feerate = None;
+       let mut check_htlc_retry = |should_retry: bool, should_bump: bool| -> Option<Transaction> {
+               let (htlc_tx, htlc_tx_feerate) = if anchors {
+                       assert!(nodes[0].tx_broadcaster.txn_broadcast().is_empty());
+                       let mut events = nodes[0].chain_monitor.chain_monitor.get_and_clear_pending_events();
+                       assert_eq!(events.len(), if should_retry { 1 } else { 0 });
+                       if !should_retry {
+                               return None;
+                       }
+                       #[allow(unused_assignments)]
+                       let mut tx = Transaction {
+                               version: 2,
+                               lock_time: bitcoin::PackedLockTime::ZERO,
+                               input: vec![],
+                               output: vec![],
+                       };
+                       #[allow(unused_assignments)]
+                       let mut feerate = 0;
+                       #[cfg(anchors)] {
+                               feerate = if let Event::BumpTransaction(BumpTransactionEvent::HTLCResolution {
+                                       target_feerate_sat_per_1000_weight, mut htlc_descriptors, tx_lock_time,
+                               }) = events.pop().unwrap() {
+                                       assert_eq!(htlc_descriptors.len(), 1);
+                                       let descriptor = htlc_descriptors.pop().unwrap();
+                                       assert_eq!(descriptor.commitment_txid, commitment_txn[0].txid());
+                                       let htlc_output_idx = descriptor.htlc.transaction_output_index.unwrap() as usize;
+                                       assert!(htlc_output_idx < commitment_txn[0].output.len());
+                                       tx.lock_time = tx_lock_time;
+                                       // Note that we don't care about actually making the HTLC transaction meet the
+                                       // feerate for the test, we just want to make sure the feerates we receive from
+                                       // the events never decrease.
+                                       tx.input.push(descriptor.unsigned_tx_input());
+                                       let signer = nodes[0].keys_manager.derive_channel_keys(
+                                               descriptor.channel_value_satoshis, &descriptor.channel_keys_id,
+                                       );
+                                       let per_commitment_point = signer.get_per_commitment_point(
+                                               descriptor.per_commitment_number, &secp
+                                       );
+                                       tx.output.push(descriptor.tx_output(&per_commitment_point, &secp));
+                                       let our_sig = signer.sign_holder_htlc_transaction(&mut tx, 0, &descriptor, &secp).unwrap();
+                                       let witness_script = descriptor.witness_script(&per_commitment_point, &secp);
+                                       tx.input[0].witness = descriptor.tx_input_witness(&our_sig, &witness_script);
+                                       target_feerate_sat_per_1000_weight as u64
+                               } else { panic!("unexpected event"); };
+                       }
+                       (tx, feerate)
+               } else {
+                       assert!(nodes[0].chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+                       let mut txn = nodes[0].tx_broadcaster.txn_broadcast();
+                       assert_eq!(txn.len(), if should_retry { 1 } else { 0 });
+                       if !should_retry {
+                               return None;
+                       }
+                       let htlc_tx = txn.pop().unwrap();
+                       check_spends!(htlc_tx, commitment_txn[0]);
+                       let htlc_tx_fee = HTLC_AMT_SAT - htlc_tx.output[0].value;
+                       let htlc_tx_feerate = htlc_tx_fee * 1000 / htlc_tx.weight() as u64;
+                       (htlc_tx, htlc_tx_feerate)
+               };
+               if should_bump {
+                       assert!(htlc_tx_feerate > prev_htlc_tx_feerate.take().unwrap());
+               } else if let Some(prev_feerate) = prev_htlc_tx_feerate.take() {
+                       assert_eq!(htlc_tx_feerate, prev_feerate);
+               }
+               prev_htlc_tx_feerate = Some(htlc_tx_feerate);
+               Some(htlc_tx)
+       };
+
+       // Connect blocks up to one before the HTLC expires. This should not result in a claim/retry.
+       connect_blocks(&nodes[0], htlc_expiry - nodes[0].best_block_info().1 - 2);
+       check_htlc_retry(false, false);
+
+       // Connect one more block, producing our first claim.
+       connect_blocks(&nodes[0], 1);
+       check_htlc_retry(true, false);
+
+       // Connect one more block, expecting a retry with a fee bump. Unfortunately, we cannot bump HTLC
+       // transactions pre-anchors.
+       connect_blocks(&nodes[0], 1);
+       check_htlc_retry(true, anchors);
+
+       // Trigger a call and we should have another retry, but without a bump.
+       nodes[0].chain_monitor.chain_monitor.rebroadcast_pending_claims();
+       check_htlc_retry(true, false);
+
+       // Double the feerate and trigger a call, expecting a fee-bumped retry.
+       *nodes[0].fee_estimator.sat_per_kw.lock().unwrap() *= 2;
+       nodes[0].chain_monitor.chain_monitor.rebroadcast_pending_claims();
+       check_htlc_retry(true, anchors);
+
+       // Connect one more block, expecting a retry with a fee bump. Unfortunately, we cannot bump HTLC
+       // transactions pre-anchors.
+       connect_blocks(&nodes[0], 1);
+       let htlc_tx = check_htlc_retry(true, anchors).unwrap();
+
+       // Mine the HTLC transaction to ensure we don't retry claims while they're confirmed.
+       mine_transaction(&nodes[0], &htlc_tx);
+       // If we have a `ConnectStyle` that advertises the new block first without the transasctions,
+       // we'll receive an extra bumped claim.
+       if nodes[0].connect_style.borrow().updates_best_block_first() {
+               check_htlc_retry(true, anchors);
+       }
+       nodes[0].chain_monitor.chain_monitor.rebroadcast_pending_claims();
+       check_htlc_retry(false, false);
+}
+
+#[test]
+fn test_monitor_timer_based_claim() {
+       do_test_monitor_rebroadcast_pending_claims(false);
+       #[cfg(anchors)]
+       do_test_monitor_rebroadcast_pending_claims(true);
+}
+
 #[cfg(anchors)]
 #[test]
 fn test_yield_anchors_events() {
index 501451c40fc34081e6f6378074b71ebf5e6880cd..4b2eb9674fa8acefa0afba97245ca4674d49e577 100644 (file)
@@ -42,7 +42,7 @@ use crate::io_extras::read_to_end;
 
 use crate::events::{MessageSendEventsProvider, OnionMessageProvider};
 use crate::util::logger;
-use crate::util::ser::{LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname};
+use crate::util::ser::{LengthReadable, Readable, ReadableArgs, Writeable, Writer, WithoutLength, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname};
 
 use crate::ln::{PaymentPreimage, PaymentHash, PaymentSecret};
 
@@ -1168,6 +1168,7 @@ mod fuzzy_internal_msgs {
                },
                FinalNode {
                        payment_data: Option<FinalOnionHopData>,
+                       payment_metadata: Option<Vec<u8>>,
                        keysend_preimage: Option<PaymentPreimage>,
                },
        }
@@ -1661,11 +1662,12 @@ impl Writeable for OnionHopData {
                                        (6, short_channel_id, required)
                                });
                        },
-                       OnionHopDataFormat::FinalNode { ref payment_data, ref keysend_preimage } => {
+                       OnionHopDataFormat::FinalNode { ref payment_data, ref payment_metadata, ref keysend_preimage } => {
                                _encode_varint_length_prefixed_tlv!(w, {
                                        (2, HighZeroBytesDroppedBigSize(self.amt_to_forward), required),
                                        (4, HighZeroBytesDroppedBigSize(self.outgoing_cltv_value), required),
                                        (8, payment_data, option),
+                                       (16, payment_metadata.as_ref().map(|m| WithoutLength(m)), option),
                                        (5482373484, keysend_preimage, option)
                                });
                        },
@@ -1680,29 +1682,33 @@ impl Readable for OnionHopData {
                let mut cltv_value = HighZeroBytesDroppedBigSize(0u32);
                let mut short_id: Option<u64> = None;
                let mut payment_data: Option<FinalOnionHopData> = None;
+               let mut payment_metadata: Option<WithoutLength<Vec<u8>>> = None;
                let mut keysend_preimage: Option<PaymentPreimage> = None;
                read_tlv_fields!(r, {
                        (2, amt, required),
                        (4, cltv_value, required),
                        (6, short_id, option),
                        (8, payment_data, option),
+                       (16, payment_metadata, option),
                        // See https://github.com/lightning/blips/blob/master/blip-0003.md
                        (5482373484, keysend_preimage, option)
                });
 
                let format = if let Some(short_channel_id) = short_id {
                        if payment_data.is_some() { return Err(DecodeError::InvalidValue); }
+                       if payment_metadata.is_some() { return Err(DecodeError::InvalidValue); }
                        OnionHopDataFormat::NonFinalNode {
                                short_channel_id,
                        }
                } else {
-                       if let &Some(ref data) = &payment_data {
+                       if let Some(data) = &payment_data {
                                if data.total_msat > MAX_VALUE_MSAT {
                                        return Err(DecodeError::InvalidValue);
                                }
                        }
                        OnionHopDataFormat::FinalNode {
                                payment_data,
+                               payment_metadata: payment_metadata.map(|w| w.0),
                                keysend_preimage,
                        }
                };
@@ -2880,6 +2886,7 @@ mod tests {
                let mut msg = msgs::OnionHopData {
                        format: OnionHopDataFormat::FinalNode {
                                payment_data: None,
+                               payment_metadata: None,
                                keysend_preimage: None,
                        },
                        amt_to_forward: 0x0badf00d01020304,
@@ -2903,6 +2910,7 @@ mod tests {
                                        payment_secret: expected_payment_secret,
                                        total_msat: 0x1badca1f
                                }),
+                               payment_metadata: None,
                                keysend_preimage: None,
                        },
                        amt_to_forward: 0x0badf00d01020304,
@@ -2917,6 +2925,7 @@ mod tests {
                                payment_secret,
                                total_msat: 0x1badca1f
                        }),
+                       payment_metadata: None,
                        keysend_preimage: None,
                } = msg.format {
                        assert_eq!(payment_secret, expected_payment_secret);
index 743d41eea1ba731ca96ac0114a443660fe1f1738..aa3b3b7e8f69dac9ae897152ece49ebf813bab50 100644 (file)
@@ -16,7 +16,7 @@ use crate::chain::keysinterface::{EntropySource, NodeSigner, Recipient};
 use crate::events::{Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentFailureReason};
 use crate::ln::{PaymentHash, PaymentSecret};
 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
-use crate::ln::channelmanager::{HTLCForwardInfo, FailureCode, CLTV_FAR_FAR_AWAY, MIN_CLTV_EXPIRY_DELTA, PendingAddHTLCInfo, PendingHTLCInfo, PendingHTLCRouting, PaymentId, RecipientOnionFields};
+use crate::ln::channelmanager::{HTLCForwardInfo, FailureCode, CLTV_FAR_FAR_AWAY, DISABLE_GOSSIP_TICKS, MIN_CLTV_EXPIRY_DELTA, PendingAddHTLCInfo, PendingHTLCInfo, PendingHTLCRouting, PaymentId, RecipientOnionFields};
 use crate::ln::onion_utils;
 use crate::routing::gossip::{NetworkUpdate, RoutingFees};
 use crate::routing::router::{get_route, PaymentParameters, Route, RouteHint, RouteHintHop};
@@ -587,6 +587,15 @@ fn test_onion_failure() {
                // disconnect event to the channel between nodes[1] ~ nodes[2]
                nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id());
                nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
+       }, true, Some(UPDATE|7), Some(NetworkUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy(short_channel_id)}), Some(short_channel_id));
+       run_onion_failure_test("channel_disabled", 0, &nodes, &route, &payment_hash, &payment_secret, |_| {}, || {
+               // disconnect event to the channel between nodes[1] ~ nodes[2]
+               for _ in 0..DISABLE_GOSSIP_TICKS + 1 {
+                       nodes[1].node.timer_tick_occurred();
+                       nodes[2].node.timer_tick_occurred();
+               }
+               nodes[1].node.get_and_clear_pending_msg_events();
+               nodes[2].node.get_and_clear_pending_msg_events();
        }, true, Some(UPDATE|20), Some(NetworkUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy(short_channel_id)}), Some(short_channel_id));
        reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 
index 2c20fb17734b74a3394ca724ad732a8ec5f3525b..b9f36bbd8dadf94592c9082a303ef378879a4819 100644 (file)
@@ -170,6 +170,7 @@ pub(super) fn build_onion_payloads(path: &Vec<RouteHop>, total_msat: u64, mut re
                                                        total_msat,
                                                })
                                        } else { None },
+                                       payment_metadata: recipient_onion.payment_metadata.take(),
                                        keysend_preimage: *keysend_preimage,
                                }
                        } else {
index 8dfdf7611b3b653a069f4f90c332208596e72c17..33f4762bbfa3fc8295cfd56f81d7eae7084859a0 100644 (file)
@@ -46,6 +46,7 @@ pub(crate) enum PendingOutboundPayment {
                session_privs: HashSet<[u8; 32]>,
                payment_hash: PaymentHash,
                payment_secret: Option<PaymentSecret>,
+               payment_metadata: Option<Vec<u8>>,
                keysend_preimage: Option<PaymentPreimage>,
                pending_amt_msat: u64,
                /// Used to track the fee paid. Only present if the payment was serialized on 0.0.103+.
@@ -405,7 +406,7 @@ pub enum PaymentSendFailure {
 ///
 /// This should generally be constructed with data communicated to us from the recipient (via a
 /// BOLT11 or BOLT12 invoice).
-#[derive(Clone)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct RecipientOnionFields {
        /// The [`PaymentSecret`] is an arbitrary 32 bytes provided by the recipient for us to repeat
        /// in the onion. It is unrelated to `payment_hash` (or [`PaymentPreimage`]) and exists to
@@ -419,14 +420,32 @@ pub struct RecipientOnionFields {
        /// receives, thus you should generally never be providing a secret here for spontaneous
        /// payments.
        pub payment_secret: Option<PaymentSecret>,
+       /// The payment metadata serves a similar purpose as [`Self::payment_secret`] but is of
+       /// arbitrary length. This gives recipients substantially more flexibility to receive
+       /// additional data.
+       ///
+       /// In LDK, while the [`Self::payment_secret`] is fixed based on an internal authentication
+       /// scheme to authenticate received payments against expected payments and invoices, this field
+       /// is not used in LDK for received payments, and can be used to store arbitrary data in
+       /// invoices which will be received with the payment.
+       ///
+       /// Note that this field was added to the lightning specification more recently than
+       /// [`Self::payment_secret`] and while nearly all lightning senders support secrets, metadata
+       /// may not be supported as universally.
+       pub payment_metadata: Option<Vec<u8>>,
 }
 
+impl_writeable_tlv_based!(RecipientOnionFields, {
+       (0, payment_secret, option),
+       (2, payment_metadata, option),
+});
+
 impl RecipientOnionFields {
        /// Creates a [`RecipientOnionFields`] from only a [`PaymentSecret`]. This is the most common
        /// set of onion fields for today's BOLT11 invoices - most nodes require a [`PaymentSecret`]
        /// but do not require or provide any further data.
        pub fn secret_only(payment_secret: PaymentSecret) -> Self {
-               Self { payment_secret: Some(payment_secret) }
+               Self { payment_secret: Some(payment_secret), payment_metadata: None }
        }
 
        /// Creates a new [`RecipientOnionFields`] with no fields. This generally does not create
@@ -435,7 +454,21 @@ impl RecipientOnionFields {
        ///
        /// [`ChannelManager::send_spontaneous_payment`]: super::channelmanager::ChannelManager::send_spontaneous_payment
        pub fn spontaneous_empty() -> Self {
-               Self { payment_secret: None }
+               Self { payment_secret: None, payment_metadata: None }
+       }
+
+       /// When we have received some HTLC(s) towards an MPP payment, as we receive further HTLC(s) we
+       /// have to make sure that some fields match exactly across the parts. For those that aren't
+       /// required to match, if they don't match we should remove them so as to not expose data
+       /// that's dependent on the HTLC receive order to users.
+       ///
+       /// Here we implement this, first checking compatibility then mutating two objects and then
+       /// dropping any remaining non-matching fields from both.
+       pub(super) fn check_merge(&mut self, further_htlc_fields: &mut Self) -> Result<(), ()> {
+               if self.payment_secret != further_htlc_fields.payment_secret { return Err(()); }
+               if self.payment_metadata != further_htlc_fields.payment_metadata { return Err(()); }
+               // For custom TLVs we should just drop non-matching ones, but not reject the payment.
+               Ok(())
        }
 }
 
@@ -722,7 +755,7 @@ impl OutboundPayments {
                                hash_map::Entry::Occupied(mut payment) => {
                                        let res = match payment.get() {
                                                PendingOutboundPayment::Retryable {
-                                                       total_msat, keysend_preimage, payment_secret, pending_amt_msat, ..
+                                                       total_msat, keysend_preimage, payment_secret, payment_metadata, pending_amt_msat, ..
                                                } => {
                                                        let retry_amt_msat: u64 = route.paths.iter().map(|path| path.last().unwrap().fee_msat).sum();
                                                        if retry_amt_msat + *pending_amt_msat > *total_msat * (100 + RETRY_OVERFLOW_PERCENTAGE) / 100 {
@@ -732,6 +765,7 @@ impl OutboundPayments {
                                                        }
                                                        (*total_msat, RecipientOnionFields {
                                                                        payment_secret: *payment_secret,
+                                                                       payment_metadata: payment_metadata.clone(),
                                                                }, *keysend_preimage)
                                                },
                                                PendingOutboundPayment::Legacy { .. } => {
@@ -886,6 +920,18 @@ impl OutboundPayments {
                }
        }
 
+       #[cfg(test)]
+       pub(super) fn test_set_payment_metadata(
+               &self, payment_id: PaymentId, new_payment_metadata: Option<Vec<u8>>
+       ) {
+               match self.pending_outbound_payments.lock().unwrap().get_mut(&payment_id).unwrap() {
+                       PendingOutboundPayment::Retryable { payment_metadata, .. } => {
+                               *payment_metadata = new_payment_metadata;
+                       },
+                       _ => panic!("Need a retryable payment to update metadata on"),
+               }
+       }
+
        #[cfg(test)]
        pub(super) fn test_add_new_pending_payment<ES: Deref>(
                &self, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields, payment_id: PaymentId,
@@ -917,6 +963,7 @@ impl OutboundPayments {
                                        pending_fee_msat: Some(0),
                                        payment_hash,
                                        payment_secret: recipient_onion.payment_secret,
+                                       payment_metadata: recipient_onion.payment_metadata,
                                        keysend_preimage,
                                        starting_block_height: best_block_height,
                                        total_msat: route.get_total_amount(),
@@ -1358,6 +1405,7 @@ impl_writeable_tlv_based_enum_upgradable!(PendingOutboundPayment,
                (4, payment_secret, option),
                (5, keysend_preimage, option),
                (6, total_msat, required),
+               (7, payment_metadata, option),
                (8, pending_amt_msat, required),
                (10, starting_block_height, required),
                (not_written, retry_strategy, (static_value, None)),
index bcdb42f9382c6b100043759443ecd4c39f376c89..51811ec632943efb373031bc87eacab3feecfb69 100644 (file)
@@ -343,7 +343,7 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
        if !confirm_before_reload {
                let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
                assert_eq!(as_broadcasted_txn.len(), 1);
-               assert_eq!(as_broadcasted_txn[0], as_commitment_tx);
+               assert_eq!(as_broadcasted_txn[0].txid(), as_commitment_tx.txid());
        } else {
                assert!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
        }
@@ -406,9 +406,11 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
        mine_transaction(&nodes[0], &bs_htlc_claim_txn[0]);
        expect_payment_sent!(nodes[0], payment_preimage_1);
        connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
-       let as_htlc_timeout_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
-       assert_eq!(as_htlc_timeout_txn.len(), 2);
-       let (first_htlc_timeout_tx, second_htlc_timeout_tx) = (&as_htlc_timeout_txn[0], &as_htlc_timeout_txn[1]);
+       let (first_htlc_timeout_tx, second_htlc_timeout_tx) = {
+               let mut txn = nodes[0].tx_broadcaster.unique_txn_broadcast();
+               assert_eq!(txn.len(), 2);
+               (txn.remove(0), txn.remove(0))
+       };
        check_spends!(first_htlc_timeout_tx, as_commitment_tx);
        check_spends!(second_htlc_timeout_tx, as_commitment_tx);
        if first_htlc_timeout_tx.input[0].previous_output == bs_htlc_claim_txn[0].input[0].previous_output {
@@ -682,7 +684,7 @@ fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, co
        connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
        let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(node_txn.len(), 3);
-       assert_eq!(node_txn[0], node_txn[1]);
+       assert_eq!(node_txn[0].txid(), node_txn[1].txid());
        check_spends!(node_txn[1], funding_tx);
        check_spends!(node_txn[2], node_txn[1]);
        let timeout_txn = vec![node_txn[2].clone()];
@@ -3017,3 +3019,151 @@ fn claim_from_closed_chan() {
        do_claim_from_closed_chan(true);
        do_claim_from_closed_chan(false);
 }
+
+fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) {
+       // Check that a payment metadata received on one HTLC that doesn't match the one received on
+       // another results in the HTLC being rejected.
+       //
+       // We first set up a diamond shaped network, allowing us to split a payment into two HTLCs, the
+       // first of which we'll deliver and the second of which we'll fail and then re-send with
+       // modified payment metadata, which will in turn result in it being failed by the recipient.
+       let chanmon_cfgs = create_chanmon_cfgs(4);
+       let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
+       let mut config = test_default_channel_config();
+       config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 50;
+       let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, Some(config), Some(config), Some(config)]);
+
+       let persister;
+       let new_chain_monitor;
+       let nodes_0_deserialized;
+
+       let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
+
+       create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
+       let chan_id_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
+       create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
+       let chan_id_cd = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
+
+       // Pay more than half of each channel's max, requiring MPP
+       let amt_msat = 750_000_000;
+       let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3], Some(amt_msat));
+       let payment_id = PaymentId(payment_hash.0);
+       let payment_metadata = vec![44, 49, 52, 142];
+
+       let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
+               .with_features(nodes[1].node.invoice_features());
+       let mut route_params = RouteParameters {
+               payment_params,
+               final_value_msat: amt_msat,
+       };
+
+       // Send the MPP payment, delivering the updated commitment state to nodes[1].
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields {
+                       payment_secret: Some(payment_secret), payment_metadata: Some(payment_metadata),
+               }, payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
+       check_added_monitors!(nodes[0], 2);
+
+       let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
+       assert_eq!(send_events.len(), 2);
+       let first_send = SendEvent::from_event(send_events.pop().unwrap());
+       let second_send = SendEvent::from_event(send_events.pop().unwrap());
+
+       let (b_recv_ev, c_recv_ev) = if first_send.node_id == nodes[1].node.get_our_node_id() {
+               (&first_send, &second_send)
+       } else {
+               (&second_send, &first_send)
+       };
+       nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &b_recv_ev.msgs[0]);
+       commitment_signed_dance!(nodes[1], nodes[0], b_recv_ev.commitment_msg, false, true);
+
+       expect_pending_htlcs_forwardable!(nodes[1]);
+       check_added_monitors(&nodes[1], 1);
+       let b_forward_ev = SendEvent::from_node(&nodes[1]);
+       nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &b_forward_ev.msgs[0]);
+       commitment_signed_dance!(nodes[3], nodes[1], b_forward_ev.commitment_msg, false, true);
+
+       expect_pending_htlcs_forwardable!(nodes[3]);
+
+       // Before delivering the second MPP HTLC to nodes[2], disconnect nodes[2] and nodes[3], which
+       // will result in nodes[2] failing the HTLC back.
+       nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
+       nodes[3].node.peer_disconnected(&nodes[2].node.get_our_node_id());
+
+       nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &c_recv_ev.msgs[0]);
+       commitment_signed_dance!(nodes[2], nodes[0], c_recv_ev.commitment_msg, false, true);
+
+       let cs_fail = get_htlc_update_msgs(&nodes[2], &nodes[0].node.get_our_node_id());
+       nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail.update_fail_htlcs[0]);
+       commitment_signed_dance!(nodes[0], nodes[2], cs_fail.commitment_signed, false, true);
+
+       let payment_fail_retryable_evs = nodes[0].node.get_and_clear_pending_events();
+       assert_eq!(payment_fail_retryable_evs.len(), 2);
+       if let Event::PaymentPathFailed { .. } = payment_fail_retryable_evs[0] {} else { panic!(); }
+       if let Event::PendingHTLCsForwardable { .. } = payment_fail_retryable_evs[1] {} else { panic!(); }
+
+       // Before we allow the HTLC to be retried, optionally change the payment_metadata we have
+       // stored for our payment.
+       if do_modify {
+               nodes[0].node.test_set_payment_metadata(payment_id, Some(Vec::new()));
+       }
+
+       // Optionally reload nodes[3] to check that the payment_metadata is properly serialized with
+       // the payment state.
+       if do_reload {
+               let mon_bd = get_monitor!(nodes[3], chan_id_bd).encode();
+               let mon_cd = get_monitor!(nodes[3], chan_id_cd).encode();
+               reload_node!(nodes[3], config, &nodes[3].node.encode(), &[&mon_bd, &mon_cd],
+                       persister, new_chain_monitor, nodes_0_deserialized);
+               nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
+               reconnect_nodes(&nodes[1], &nodes[3], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+       }
+       reconnect_nodes(&nodes[2], &nodes[3], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+
+       // Create a new channel between C and D as A will refuse to retry on the existing one because
+       // it just failed.
+       let chan_id_cd_2 = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
+
+       // Now retry the failed HTLC.
+       nodes[0].node.process_pending_htlc_forwards();
+       check_added_monitors(&nodes[0], 1);
+       let as_resend = SendEvent::from_node(&nodes[0]);
+       nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_resend.msgs[0]);
+       commitment_signed_dance!(nodes[2], nodes[0], as_resend.commitment_msg, false, true);
+
+       expect_pending_htlcs_forwardable!(nodes[2]);
+       check_added_monitors(&nodes[2], 1);
+       let cs_forward = SendEvent::from_node(&nodes[2]);
+       nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &cs_forward.msgs[0]);
+       commitment_signed_dance!(nodes[3], nodes[2], cs_forward.commitment_msg, false, true);
+
+       // Finally, check that nodes[3] does the correct thing - either accepting the payment or, if
+       // the payment metadata was modified, failing only the one modified HTLC and retaining the
+       // other.
+       if do_modify {
+               expect_pending_htlcs_forwardable_ignore!(nodes[3]);
+               nodes[3].node.process_pending_htlc_forwards();
+               expect_pending_htlcs_forwardable_conditions(nodes[3].node.get_and_clear_pending_events(),
+                       &[HTLCDestination::FailedPayment {payment_hash}]);
+               nodes[3].node.process_pending_htlc_forwards();
+
+               check_added_monitors(&nodes[3], 1);
+               let ds_fail = get_htlc_update_msgs(&nodes[3], &nodes[2].node.get_our_node_id());
+
+               nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &ds_fail.update_fail_htlcs[0]);
+               commitment_signed_dance!(nodes[2], nodes[3], ds_fail.commitment_signed, false, true);
+               expect_pending_htlcs_forwardable_conditions(nodes[2].node.get_and_clear_pending_events(),
+                       &[HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_id_cd_2 }]);
+       } else {
+               expect_pending_htlcs_forwardable!(nodes[3]);
+               expect_payment_claimable!(nodes[3], payment_hash, payment_secret, amt_msat);
+               claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
+       }
+}
+
+#[test]
+fn test_payment_metadata_consistency() {
+       do_test_payment_metadata_consistency(true, true);
+       do_test_payment_metadata_consistency(true, false);
+       do_test_payment_metadata_consistency(false, true);
+       do_test_payment_metadata_consistency(false, false);
+}
index 6d89268cdc34357808178ec5a99d2330dcff61dd..14b06457fc3f66b497ba86c09ca8409fb00dcd0e 100644 (file)
@@ -9,7 +9,7 @@
 
 //! Further functional tests which test blockchain reorganizations.
 
-use crate::chain::channelmonitor::ANTI_REORG_DELAY;
+use crate::chain::channelmonitor::{ANTI_REORG_DELAY, LATENCY_GRACE_PERIOD_BLOCKS};
 use crate::chain::transaction::OutPoint;
 use crate::chain::Confirm;
 use crate::events::{Event, MessageSendEventsProvider, ClosureReason, HTLCDestination};
@@ -467,19 +467,21 @@ fn test_set_outpoints_partial_claiming() {
        }
 
        // Connect blocks on node B
-       connect_blocks(&nodes[1], 135);
+       connect_blocks(&nodes[1], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
        check_closed_broadcast!(nodes[1], true);
        check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        check_added_monitors!(nodes[1], 1);
        // Verify node B broadcast 2 HTLC-timeout txn
        let partial_claim_tx = {
-               let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
+               let mut node_txn = nodes[1].tx_broadcaster.unique_txn_broadcast();
                assert_eq!(node_txn.len(), 3);
+               check_spends!(node_txn[0], chan.3);
                check_spends!(node_txn[1], node_txn[0]);
                check_spends!(node_txn[2], node_txn[0]);
                assert_eq!(node_txn[1].input.len(), 1);
                assert_eq!(node_txn[2].input.len(), 1);
-               node_txn[1].clone()
+               assert_ne!(node_txn[1].input[0].previous_output, node_txn[2].input[0].previous_output);
+               node_txn.remove(1)
        };
 
        // Broadcast partial claim on node A, should regenerate a claiming tx with HTLC dropped
index 48b8cec3536b2ab0576feca6826ed960f46d7a60..6bf155a230324a48c29c5b5c5d0b9cb06ed04a97 100644 (file)
@@ -97,24 +97,27 @@ use bitcoin::blockdata::constants::ChainHash;
 use bitcoin::hash_types::{WPubkeyHash, WScriptHash};
 use bitcoin::hashes::Hash;
 use bitcoin::network::constants::Network;
-use bitcoin::secp256k1::{Message, PublicKey};
+use bitcoin::secp256k1::{KeyPair, Message, PublicKey, Secp256k1, self};
 use bitcoin::secp256k1::schnorr::Signature;
 use bitcoin::util::address::{Address, Payload, WitnessVersion};
 use bitcoin::util::schnorr::TweakedPublicKey;
-use core::convert::TryFrom;
+use core::convert::{Infallible, TryFrom};
 use core::time::Duration;
 use crate::io;
 use crate::ln::PaymentHash;
 use crate::ln::features::{BlindedHopFeatures, Bolt12InvoiceFeatures};
+use crate::ln::inbound_payment::ExpandedKey;
 use crate::ln::msgs::DecodeError;
-use crate::offers::invoice_request::{InvoiceRequest, InvoiceRequestContents, InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef};
-use crate::offers::merkle::{SignError, SignatureTlvStream, SignatureTlvStreamRef, WithoutSignatures, self};
-use crate::offers::offer::{Amount, OfferTlvStream, OfferTlvStreamRef};
+use crate::offers::invoice_request::{INVOICE_REQUEST_PAYER_ID_TYPE, INVOICE_REQUEST_TYPES, IV_BYTES as INVOICE_REQUEST_IV_BYTES, InvoiceRequest, InvoiceRequestContents, InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef};
+use crate::offers::merkle::{SignError, SignatureTlvStream, SignatureTlvStreamRef, TlvStream, WithoutSignatures, self};
+use crate::offers::offer::{Amount, OFFER_TYPES, OfferTlvStream, OfferTlvStreamRef};
 use crate::offers::parse::{ParseError, ParsedMessage, SemanticError};
-use crate::offers::payer::{PayerTlvStream, PayerTlvStreamRef};
-use crate::offers::refund::{Refund, RefundContents};
+use crate::offers::payer::{PAYER_METADATA_TYPE, PayerTlvStream, PayerTlvStreamRef};
+use crate::offers::refund::{IV_BYTES as REFUND_IV_BYTES, Refund, RefundContents};
+use crate::offers::signer;
 use crate::onion_message::BlindedPath;
 use crate::util::ser::{HighZeroBytesDroppedBigSize, Iterable, SeekReadable, WithoutLength, Writeable, Writer};
+use crate::util::string::PrintableString;
 
 use crate::prelude::*;
 
@@ -123,7 +126,7 @@ use std::time::SystemTime;
 
 const DEFAULT_RELATIVE_EXPIRY: Duration = Duration::from_secs(7200);
 
-const SIGNATURE_TAG: &'static str = concat!("lightning", "invoice", "signature");
+pub(super) const SIGNATURE_TAG: &'static str = concat!("lightning", "invoice", "signature");
 
 /// Builds an [`Invoice`] from either:
 /// - an [`InvoiceRequest`] for the "offer to be paid" flow or
@@ -134,62 +137,130 @@ const SIGNATURE_TAG: &'static str = concat!("lightning", "invoice", "signature")
 /// [`InvoiceRequest`]: crate::offers::invoice_request::InvoiceRequest
 /// [`Refund`]: crate::offers::refund::Refund
 /// [module-level documentation]: self
-pub struct InvoiceBuilder<'a> {
+pub struct InvoiceBuilder<'a, S: SigningPubkeyStrategy> {
        invreq_bytes: &'a Vec<u8>,
        invoice: InvoiceContents,
+       keys: Option<KeyPair>,
+       signing_pubkey_strategy: core::marker::PhantomData<S>,
 }
 
-impl<'a> InvoiceBuilder<'a> {
+/// Indicates how [`Invoice::signing_pubkey`] was set.
+pub trait SigningPubkeyStrategy {}
+
+/// [`Invoice::signing_pubkey`] was explicitly set.
+pub struct ExplicitSigningPubkey {}
+
+/// [`Invoice::signing_pubkey`] was derived.
+pub struct DerivedSigningPubkey {}
+
+impl SigningPubkeyStrategy for ExplicitSigningPubkey {}
+impl SigningPubkeyStrategy for DerivedSigningPubkey {}
+
+impl<'a> InvoiceBuilder<'a, ExplicitSigningPubkey> {
        pub(super) fn for_offer(
                invoice_request: &'a InvoiceRequest, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>,
                created_at: Duration, payment_hash: PaymentHash
        ) -> Result<Self, SemanticError> {
-               let amount_msats = match invoice_request.amount_msats() {
-                       Some(amount_msats) => amount_msats,
-                       None => match invoice_request.contents.offer.amount() {
-                               Some(Amount::Bitcoin { amount_msats }) => {
-                                       amount_msats.checked_mul(invoice_request.quantity().unwrap_or(1))
-                                               .ok_or(SemanticError::InvalidAmount)?
-                               },
-                               Some(Amount::Currency { .. }) => return Err(SemanticError::UnsupportedCurrency),
-                               None => return Err(SemanticError::MissingAmount),
-                       },
-               };
-
+               let amount_msats = Self::check_amount_msats(invoice_request)?;
+               let signing_pubkey = invoice_request.contents.inner.offer.signing_pubkey();
                let contents = InvoiceContents::ForOffer {
                        invoice_request: invoice_request.contents.clone(),
-                       fields: InvoiceFields {
-                               payment_paths, created_at, relative_expiry: None, payment_hash, amount_msats,
-                               fallbacks: None, features: Bolt12InvoiceFeatures::empty(),
-                               signing_pubkey: invoice_request.contents.offer.signing_pubkey(),
-                       },
+                       fields: Self::fields(
+                               payment_paths, created_at, payment_hash, amount_msats, signing_pubkey
+                       ),
                };
 
-               Self::new(&invoice_request.bytes, contents)
+               Self::new(&invoice_request.bytes, contents, None)
        }
 
        pub(super) fn for_refund(
                refund: &'a Refund, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, created_at: Duration,
                payment_hash: PaymentHash, signing_pubkey: PublicKey
        ) -> Result<Self, SemanticError> {
+               let amount_msats = refund.amount_msats();
                let contents = InvoiceContents::ForRefund {
                        refund: refund.contents.clone(),
-                       fields: InvoiceFields {
-                               payment_paths, created_at, relative_expiry: None, payment_hash,
-                               amount_msats: refund.amount_msats(), fallbacks: None,
-                               features: Bolt12InvoiceFeatures::empty(), signing_pubkey,
-                       },
+                       fields: Self::fields(
+                               payment_paths, created_at, payment_hash, amount_msats, signing_pubkey
+                       ),
+               };
+
+               Self::new(&refund.bytes, contents, None)
+       }
+}
+
+impl<'a> InvoiceBuilder<'a, DerivedSigningPubkey> {
+       pub(super) fn for_offer_using_keys(
+               invoice_request: &'a InvoiceRequest, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>,
+               created_at: Duration, payment_hash: PaymentHash, keys: KeyPair
+       ) -> Result<Self, SemanticError> {
+               let amount_msats = Self::check_amount_msats(invoice_request)?;
+               let signing_pubkey = invoice_request.contents.inner.offer.signing_pubkey();
+               let contents = InvoiceContents::ForOffer {
+                       invoice_request: invoice_request.contents.clone(),
+                       fields: Self::fields(
+                               payment_paths, created_at, payment_hash, amount_msats, signing_pubkey
+                       ),
                };
 
-               Self::new(&refund.bytes, contents)
+               Self::new(&invoice_request.bytes, contents, Some(keys))
+       }
+
+       pub(super) fn for_refund_using_keys(
+               refund: &'a Refund, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, created_at: Duration,
+               payment_hash: PaymentHash, keys: KeyPair,
+       ) -> Result<Self, SemanticError> {
+               let amount_msats = refund.amount_msats();
+               let signing_pubkey = keys.public_key();
+               let contents = InvoiceContents::ForRefund {
+                       refund: refund.contents.clone(),
+                       fields: Self::fields(
+                               payment_paths, created_at, payment_hash, amount_msats, signing_pubkey
+                       ),
+               };
+
+               Self::new(&refund.bytes, contents, Some(keys))
+       }
+}
+
+impl<'a, S: SigningPubkeyStrategy> InvoiceBuilder<'a, S> {
+       fn check_amount_msats(invoice_request: &InvoiceRequest) -> Result<u64, SemanticError> {
+               match invoice_request.amount_msats() {
+                       Some(amount_msats) => Ok(amount_msats),
+                       None => match invoice_request.contents.inner.offer.amount() {
+                               Some(Amount::Bitcoin { amount_msats }) => {
+                                       amount_msats.checked_mul(invoice_request.quantity().unwrap_or(1))
+                                               .ok_or(SemanticError::InvalidAmount)
+                               },
+                               Some(Amount::Currency { .. }) => Err(SemanticError::UnsupportedCurrency),
+                               None => Err(SemanticError::MissingAmount),
+                       },
+               }
+       }
+
+       fn fields(
+               payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, created_at: Duration,
+               payment_hash: PaymentHash, amount_msats: u64, signing_pubkey: PublicKey
+       ) -> InvoiceFields {
+               InvoiceFields {
+                       payment_paths, created_at, relative_expiry: None, payment_hash, amount_msats,
+                       fallbacks: None, features: Bolt12InvoiceFeatures::empty(), signing_pubkey,
+               }
        }
 
-       fn new(invreq_bytes: &'a Vec<u8>, contents: InvoiceContents) -> Result<Self, SemanticError> {
+       fn new(
+               invreq_bytes: &'a Vec<u8>, contents: InvoiceContents, keys: Option<KeyPair>
+       ) -> Result<Self, SemanticError> {
                if contents.fields().payment_paths.is_empty() {
                        return Err(SemanticError::MissingPaths);
                }
 
-               Ok(Self { invreq_bytes, invoice: contents })
+               Ok(Self {
+                       invreq_bytes,
+                       invoice: contents,
+                       keys,
+                       signing_pubkey_strategy: core::marker::PhantomData,
+               })
        }
 
        /// Sets the [`Invoice::relative_expiry`] as seconds since [`Invoice::created_at`]. Any expiry
@@ -246,7 +317,9 @@ impl<'a> InvoiceBuilder<'a> {
                self.invoice.fields_mut().features.set_basic_mpp_optional();
                self
        }
+}
 
+impl<'a> InvoiceBuilder<'a, ExplicitSigningPubkey> {
        /// Builds an unsigned [`Invoice`] after checking for valid semantics. It can be signed by
        /// [`UnsignedInvoice::sign`].
        pub fn build(self) -> Result<UnsignedInvoice<'a>, SemanticError> {
@@ -256,11 +329,33 @@ impl<'a> InvoiceBuilder<'a> {
                        }
                }
 
-               let InvoiceBuilder { invreq_bytes, invoice } = self;
+               let InvoiceBuilder { invreq_bytes, invoice, .. } = self;
                Ok(UnsignedInvoice { invreq_bytes, invoice })
        }
 }
 
+impl<'a> InvoiceBuilder<'a, DerivedSigningPubkey> {
+       /// Builds a signed [`Invoice`] after checking for valid semantics.
+       pub fn build_and_sign<T: secp256k1::Signing>(
+               self, secp_ctx: &Secp256k1<T>
+       ) -> Result<Invoice, SemanticError> {
+               #[cfg(feature = "std")] {
+                       if self.invoice.is_offer_or_refund_expired() {
+                               return Err(SemanticError::AlreadyExpired);
+                       }
+               }
+
+               let InvoiceBuilder { invreq_bytes, invoice, keys, .. } = self;
+               let unsigned_invoice = UnsignedInvoice { invreq_bytes, invoice };
+
+               let keys = keys.unwrap();
+               let invoice = unsigned_invoice
+                       .sign::<_, Infallible>(|digest| Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys)))
+                       .unwrap();
+               Ok(invoice)
+       }
+}
+
 /// A semantically valid [`Invoice`] that hasn't been signed.
 pub struct UnsignedInvoice<'a> {
        invreq_bytes: &'a Vec<u8>,
@@ -313,7 +408,8 @@ impl<'a> UnsignedInvoice<'a> {
 /// [`Offer`]: crate::offers::offer::Offer
 /// [`Refund`]: crate::offers::refund::Refund
 /// [`InvoiceRequest`]: crate::offers::invoice_request::InvoiceRequest
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub struct Invoice {
        bytes: Vec<u8>,
        contents: InvoiceContents,
@@ -324,7 +420,8 @@ pub struct Invoice {
 ///
 /// [`Offer`]: crate::offers::offer::Offer
 /// [`Refund`]: crate::offers::refund::Refund
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 enum InvoiceContents {
        /// Contents for an [`Invoice`] corresponding to an [`Offer`].
        ///
@@ -356,6 +453,12 @@ struct InvoiceFields {
 }
 
 impl Invoice {
+       /// A complete description of the purpose of the originating offer or refund. Intended to be
+       /// displayed to the user but with the caveat that it has not been verified in any way.
+       pub fn description(&self) -> PrintableString {
+               self.contents.description()
+       }
+
        /// Paths to the recipient originating from publicly reachable nodes, including information
        /// needed for routing payments across them.
        ///
@@ -474,8 +577,15 @@ impl Invoice {
                merkle::message_digest(SIGNATURE_TAG, &self.bytes).as_ref().clone()
        }
 
+       /// Verifies that the invoice was for a request or refund created using the given key.
+       pub fn verify<T: secp256k1::Signing>(
+               &self, key: &ExpandedKey, secp_ctx: &Secp256k1<T>
+       ) -> bool {
+               self.contents.verify(TlvStream::new(&self.bytes), key, secp_ctx)
+       }
+
        #[cfg(test)]
-       fn as_tlv_stream(&self) -> FullInvoiceTlvStreamRef {
+       pub(super) fn as_tlv_stream(&self) -> FullInvoiceTlvStreamRef {
                let (payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream) =
                        self.contents.as_tlv_stream();
                let signature_tlv_stream = SignatureTlvStreamRef {
@@ -491,7 +601,8 @@ impl InvoiceContents {
        #[cfg(feature = "std")]
        fn is_offer_or_refund_expired(&self) -> bool {
                match self {
-                       InvoiceContents::ForOffer { invoice_request, .. } => invoice_request.offer.is_expired(),
+                       InvoiceContents::ForOffer { invoice_request, .. } =>
+                               invoice_request.inner.offer.is_expired(),
                        InvoiceContents::ForRefund { refund, .. } => refund.is_expired(),
                }
        }
@@ -503,6 +614,15 @@ impl InvoiceContents {
                }
        }
 
+       fn description(&self) -> PrintableString {
+               match self {
+                       InvoiceContents::ForOffer { invoice_request, .. } => {
+                               invoice_request.inner.offer.description()
+                       },
+                       InvoiceContents::ForRefund { refund, .. } => refund.description(),
+               }
+       }
+
        fn fields(&self) -> &InvoiceFields {
                match self {
                        InvoiceContents::ForOffer { fields, .. } => fields,
@@ -517,6 +637,41 @@ impl InvoiceContents {
                }
        }
 
+       fn verify<T: secp256k1::Signing>(
+               &self, tlv_stream: TlvStream<'_>, key: &ExpandedKey, secp_ctx: &Secp256k1<T>
+       ) -> bool {
+               let offer_records = tlv_stream.clone().range(OFFER_TYPES);
+               let invreq_records = tlv_stream.range(INVOICE_REQUEST_TYPES).filter(|record| {
+                       match record.r#type {
+                               PAYER_METADATA_TYPE => false, // Should be outside range
+                               INVOICE_REQUEST_PAYER_ID_TYPE => !self.derives_keys(),
+                               _ => true,
+                       }
+               });
+               let tlv_stream = offer_records.chain(invreq_records);
+
+               let (metadata, payer_id, iv_bytes) = match self {
+                       InvoiceContents::ForOffer { invoice_request, .. } => {
+                               (invoice_request.metadata(), invoice_request.payer_id(), INVOICE_REQUEST_IV_BYTES)
+                       },
+                       InvoiceContents::ForRefund { refund, .. } => {
+                               (refund.metadata(), refund.payer_id(), REFUND_IV_BYTES)
+                       },
+               };
+
+               match signer::verify_metadata(metadata, key, iv_bytes, payer_id, tlv_stream, secp_ctx) {
+                       Ok(_) => true,
+                       Err(()) => false,
+               }
+       }
+
+       fn derives_keys(&self) -> bool {
+               match self {
+                       InvoiceContents::ForOffer { invoice_request, .. } => invoice_request.derives_keys(),
+                       InvoiceContents::ForRefund { refund, .. } => refund.derives_keys(),
+               }
+       }
+
        fn as_tlv_stream(&self) -> PartialInvoiceTlvStreamRef {
                let (payer, offer, invoice_request) = match self {
                        InvoiceContents::ForOffer { invoice_request, .. } => invoice_request.as_tlv_stream(),
@@ -777,67 +932,30 @@ impl TryFrom<PartialInvoiceTlvStream> for InvoiceContents {
 
 #[cfg(test)]
 mod tests {
-       use super::{DEFAULT_RELATIVE_EXPIRY, BlindedPayInfo, FallbackAddress, FullInvoiceTlvStreamRef, Invoice, InvoiceTlvStreamRef, SIGNATURE_TAG};
+       use super::{DEFAULT_RELATIVE_EXPIRY, FallbackAddress, FullInvoiceTlvStreamRef, Invoice, InvoiceTlvStreamRef, SIGNATURE_TAG};
 
        use bitcoin::blockdata::script::Script;
        use bitcoin::hashes::Hash;
        use bitcoin::network::constants::Network;
-       use bitcoin::secp256k1::{KeyPair, Message, PublicKey, Secp256k1, SecretKey, XOnlyPublicKey, self};
-       use bitcoin::secp256k1::schnorr::Signature;
+       use bitcoin::secp256k1::{Message, Secp256k1, XOnlyPublicKey, self};
        use bitcoin::util::address::{Address, Payload, WitnessVersion};
        use bitcoin::util::schnorr::TweakedPublicKey;
-       use core::convert::{Infallible, TryFrom};
+       use core::convert::TryFrom;
        use core::time::Duration;
-       use crate::ln::PaymentHash;
+       use crate::chain::keysinterface::KeyMaterial;
+       use crate::ln::features::Bolt12InvoiceFeatures;
+       use crate::ln::inbound_payment::ExpandedKey;
        use crate::ln::msgs::DecodeError;
-       use crate::ln::features::{BlindedHopFeatures, Bolt12InvoiceFeatures};
        use crate::offers::invoice_request::InvoiceRequestTlvStreamRef;
        use crate::offers::merkle::{SignError, SignatureTlvStreamRef, self};
        use crate::offers::offer::{OfferBuilder, OfferTlvStreamRef, Quantity};
        use crate::offers::parse::{ParseError, SemanticError};
        use crate::offers::payer::PayerTlvStreamRef;
        use crate::offers::refund::RefundBuilder;
+       use crate::offers::test_utils::*;
        use crate::onion_message::{BlindedHop, BlindedPath};
        use crate::util::ser::{BigSize, Iterable, Writeable};
-
-       fn payer_keys() -> KeyPair {
-               let secp_ctx = Secp256k1::new();
-               KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap())
-       }
-
-       fn payer_sign(digest: &Message) -> Result<Signature, Infallible> {
-               let secp_ctx = Secp256k1::new();
-               let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap());
-               Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys))
-       }
-
-       fn payer_pubkey() -> PublicKey {
-               payer_keys().public_key()
-       }
-
-       fn recipient_keys() -> KeyPair {
-               let secp_ctx = Secp256k1::new();
-               KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap())
-       }
-
-       fn recipient_sign(digest: &Message) -> Result<Signature, Infallible> {
-               let secp_ctx = Secp256k1::new();
-               let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap());
-               Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys))
-       }
-
-       fn recipient_pubkey() -> PublicKey {
-               recipient_keys().public_key()
-       }
-
-       fn pubkey(byte: u8) -> PublicKey {
-               let secp_ctx = Secp256k1::new();
-               PublicKey::from_secret_key(&secp_ctx, &privkey(byte))
-       }
-
-       fn privkey(byte: u8) -> SecretKey {
-               SecretKey::from_slice(&[byte; 32]).unwrap()
-       }
+       use crate::util::string::PrintableString;
 
        trait ToBytes {
                fn to_bytes(&self) -> Vec<u8>;
@@ -855,58 +973,6 @@ mod tests {
                }
        }
 
-       fn payment_paths() -> Vec<(BlindedPath, BlindedPayInfo)> {
-               let paths = vec![
-                       BlindedPath {
-                               introduction_node_id: pubkey(40),
-                               blinding_point: pubkey(41),
-                               blinded_hops: vec![
-                                       BlindedHop { blinded_node_id: pubkey(43), encrypted_payload: vec![0; 43] },
-                                       BlindedHop { blinded_node_id: pubkey(44), encrypted_payload: vec![0; 44] },
-                               ],
-                       },
-                       BlindedPath {
-                               introduction_node_id: pubkey(40),
-                               blinding_point: pubkey(41),
-                               blinded_hops: vec![
-                                       BlindedHop { blinded_node_id: pubkey(45), encrypted_payload: vec![0; 45] },
-                                       BlindedHop { blinded_node_id: pubkey(46), encrypted_payload: vec![0; 46] },
-                               ],
-                       },
-               ];
-
-               let payinfo = vec![
-                       BlindedPayInfo {
-                               fee_base_msat: 1,
-                               fee_proportional_millionths: 1_000,
-                               cltv_expiry_delta: 42,
-                               htlc_minimum_msat: 100,
-                               htlc_maximum_msat: 1_000_000_000_000,
-                               features: BlindedHopFeatures::empty(),
-                       },
-                       BlindedPayInfo {
-                               fee_base_msat: 1,
-                               fee_proportional_millionths: 1_000,
-                               cltv_expiry_delta: 42,
-                               htlc_minimum_msat: 100,
-                               htlc_maximum_msat: 1_000_000_000_000,
-                               features: BlindedHopFeatures::empty(),
-                       },
-               ];
-
-               paths.into_iter().zip(payinfo.into_iter()).collect()
-       }
-
-       fn payment_hash() -> PaymentHash {
-               PaymentHash([42; 32])
-       }
-
-       fn now() -> Duration {
-               std::time::SystemTime::now()
-                       .duration_since(std::time::SystemTime::UNIX_EPOCH)
-                       .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH")
-       }
-
        #[test]
        fn builds_invoice_for_offer_with_defaults() {
                let payment_paths = payment_paths();
@@ -926,6 +992,7 @@ mod tests {
                invoice.write(&mut buffer).unwrap();
 
                assert_eq!(invoice.bytes, buffer.as_slice());
+               assert_eq!(invoice.description(), PrintableString("foo"));
                assert_eq!(invoice.payment_paths(), payment_paths.as_slice());
                assert_eq!(invoice.created_at(), now);
                assert_eq!(invoice.relative_expiry(), DEFAULT_RELATIVE_EXPIRY);
@@ -1008,6 +1075,7 @@ mod tests {
                invoice.write(&mut buffer).unwrap();
 
                assert_eq!(invoice.bytes, buffer.as_slice());
+               assert_eq!(invoice.description(), PrintableString("foo"));
                assert_eq!(invoice.payment_paths(), payment_paths.as_slice());
                assert_eq!(invoice.created_at(), now);
                assert_eq!(invoice.relative_expiry(), DEFAULT_RELATIVE_EXPIRY);
@@ -1133,6 +1201,87 @@ mod tests {
                }
        }
 
+       #[test]
+       fn builds_invoice_from_offer_using_derived_keys() {
+               let desc = "foo".to_string();
+               let node_id = recipient_pubkey();
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let blinded_path = BlindedPath {
+                       introduction_node_id: pubkey(40),
+                       blinding_point: pubkey(41),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: pubkey(42), encrypted_payload: vec![0; 43] },
+                               BlindedHop { blinded_node_id: node_id, encrypted_payload: vec![0; 44] },
+                       ],
+               };
+
+               let offer = OfferBuilder
+                       ::deriving_signing_pubkey(desc, node_id, &expanded_key, &entropy, &secp_ctx)
+                       .amount_msats(1000)
+                       .path(blinded_path)
+                       .build().unwrap();
+               let invoice_request = offer.request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+
+               if let Err(e) = invoice_request
+                       .verify_and_respond_using_derived_keys_no_std(
+                               payment_paths(), payment_hash(), now(), &expanded_key, &secp_ctx
+                       )
+                       .unwrap()
+                       .build_and_sign(&secp_ctx)
+               {
+                       panic!("error building invoice: {:?}", e);
+               }
+
+               let expanded_key = ExpandedKey::new(&KeyMaterial([41; 32]));
+               match invoice_request.verify_and_respond_using_derived_keys_no_std(
+                       payment_paths(), payment_hash(), now(), &expanded_key, &secp_ctx
+               ) {
+                       Ok(_) => panic!("expected error"),
+                       Err(e) => assert_eq!(e, SemanticError::InvalidMetadata),
+               }
+
+               let desc = "foo".to_string();
+               let offer = OfferBuilder
+                       ::deriving_signing_pubkey(desc, node_id, &expanded_key, &entropy, &secp_ctx)
+                       .amount_msats(1000)
+                       .build().unwrap();
+               let invoice_request = offer.request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+
+               match invoice_request.verify_and_respond_using_derived_keys_no_std(
+                       payment_paths(), payment_hash(), now(), &expanded_key, &secp_ctx
+               ) {
+                       Ok(_) => panic!("expected error"),
+                       Err(e) => assert_eq!(e, SemanticError::InvalidMetadata),
+               }
+       }
+
+       #[test]
+       fn builds_invoice_from_refund_using_derived_keys() {
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let refund = RefundBuilder::new("foo".into(), vec![1; 32], payer_pubkey(), 1000).unwrap()
+                       .build().unwrap();
+
+               if let Err(e) = refund
+                       .respond_using_derived_keys_no_std(
+                               payment_paths(), payment_hash(), now(), &expanded_key, &entropy
+                       )
+                       .unwrap()
+                       .build_and_sign(&secp_ctx)
+               {
+                       panic!("error building invoice: {:?}", e);
+               }
+       }
+
        #[test]
        fn builds_invoice_with_relative_expiry() {
                let now = now();
index a1a0520c62259409b4cd516ed9607eb17bad3296..92fabd6fdf0c7ab5f49c46cfd0793e02a8886774 100644 (file)
 
 use bitcoin::blockdata::constants::ChainHash;
 use bitcoin::network::constants::Network;
-use bitcoin::secp256k1::{Message, PublicKey};
+use bitcoin::secp256k1::{KeyPair, Message, PublicKey, Secp256k1, self};
 use bitcoin::secp256k1::schnorr::Signature;
-use core::convert::TryFrom;
+use core::convert::{Infallible, TryFrom};
+use core::ops::Deref;
+use crate::chain::keysinterface::EntropySource;
 use crate::io;
 use crate::ln::PaymentHash;
 use crate::ln::features::InvoiceRequestFeatures;
+use crate::ln::inbound_payment::{ExpandedKey, IV_LEN, Nonce};
 use crate::ln::msgs::DecodeError;
-use crate::offers::invoice::{BlindedPayInfo, InvoiceBuilder};
+use crate::offers::invoice::{BlindedPayInfo, DerivedSigningPubkey, ExplicitSigningPubkey, InvoiceBuilder};
 use crate::offers::merkle::{SignError, SignatureTlvStream, SignatureTlvStreamRef, self};
 use crate::offers::offer::{Offer, OfferContents, OfferTlvStream, OfferTlvStreamRef};
 use crate::offers::parse::{ParseError, ParsedMessage, SemanticError};
 use crate::offers::payer::{PayerContents, PayerTlvStream, PayerTlvStreamRef};
+use crate::offers::signer::{Metadata, MetadataMaterial};
 use crate::onion_message::BlindedPath;
 use crate::util::ser::{HighZeroBytesDroppedBigSize, SeekReadable, WithoutLength, Writeable, Writer};
 use crate::util::string::PrintableString;
@@ -74,25 +78,83 @@ use crate::prelude::*;
 
 const SIGNATURE_TAG: &'static str = concat!("lightning", "invoice_request", "signature");
 
+pub(super) const IV_BYTES: &[u8; IV_LEN] = b"LDK Invreq ~~~~~";
+
 /// Builds an [`InvoiceRequest`] from an [`Offer`] for the "offer to be paid" flow.
 ///
 /// See [module-level documentation] for usage.
 ///
 /// [module-level documentation]: self
-pub struct InvoiceRequestBuilder<'a> {
+pub struct InvoiceRequestBuilder<'a, 'b, P: PayerIdStrategy, T: secp256k1::Signing> {
        offer: &'a Offer,
-       invoice_request: InvoiceRequestContents,
+       invoice_request: InvoiceRequestContentsWithoutPayerId,
+       payer_id: Option<PublicKey>,
+       payer_id_strategy: core::marker::PhantomData<P>,
+       secp_ctx: Option<&'b Secp256k1<T>>,
 }
 
-impl<'a> InvoiceRequestBuilder<'a> {
+/// Indicates how [`InvoiceRequest::payer_id`] will be set.
+pub trait PayerIdStrategy {}
+
+/// [`InvoiceRequest::payer_id`] will be explicitly set.
+pub struct ExplicitPayerId {}
+
+/// [`InvoiceRequest::payer_id`] will be derived.
+pub struct DerivedPayerId {}
+
+impl PayerIdStrategy for ExplicitPayerId {}
+impl PayerIdStrategy for DerivedPayerId {}
+
+impl<'a, 'b, T: secp256k1::Signing> InvoiceRequestBuilder<'a, 'b, ExplicitPayerId, T> {
        pub(super) fn new(offer: &'a Offer, metadata: Vec<u8>, payer_id: PublicKey) -> Self {
                Self {
                        offer,
-                       invoice_request: InvoiceRequestContents {
-                               payer: PayerContents(metadata), offer: offer.contents.clone(), chain: None,
-                               amount_msats: None, features: InvoiceRequestFeatures::empty(), quantity: None,
-                               payer_id, payer_note: None,
-                       },
+                       invoice_request: Self::create_contents(offer, Metadata::Bytes(metadata)),
+                       payer_id: Some(payer_id),
+                       payer_id_strategy: core::marker::PhantomData,
+                       secp_ctx: None,
+               }
+       }
+
+       pub(super) fn deriving_metadata<ES: Deref>(
+               offer: &'a Offer, payer_id: PublicKey, expanded_key: &ExpandedKey, entropy_source: ES
+       ) -> Self where ES::Target: EntropySource {
+               let nonce = Nonce::from_entropy_source(entropy_source);
+               let derivation_material = MetadataMaterial::new(nonce, expanded_key, IV_BYTES);
+               let metadata = Metadata::Derived(derivation_material);
+               Self {
+                       offer,
+                       invoice_request: Self::create_contents(offer, metadata),
+                       payer_id: Some(payer_id),
+                       payer_id_strategy: core::marker::PhantomData,
+                       secp_ctx: None,
+               }
+       }
+}
+
+impl<'a, 'b, T: secp256k1::Signing> InvoiceRequestBuilder<'a, 'b, DerivedPayerId, T> {
+       pub(super) fn deriving_payer_id<ES: Deref>(
+               offer: &'a Offer, expanded_key: &ExpandedKey, entropy_source: ES, secp_ctx: &'b Secp256k1<T>
+       ) -> Self where ES::Target: EntropySource {
+               let nonce = Nonce::from_entropy_source(entropy_source);
+               let derivation_material = MetadataMaterial::new(nonce, expanded_key, IV_BYTES);
+               let metadata = Metadata::DerivedSigningPubkey(derivation_material);
+               Self {
+                       offer,
+                       invoice_request: Self::create_contents(offer, metadata),
+                       payer_id: None,
+                       payer_id_strategy: core::marker::PhantomData,
+                       secp_ctx: Some(secp_ctx),
+               }
+       }
+}
+
+impl<'a, 'b, P: PayerIdStrategy, T: secp256k1::Signing> InvoiceRequestBuilder<'a, 'b, P, T> {
+       fn create_contents(offer: &Offer, metadata: Metadata) -> InvoiceRequestContentsWithoutPayerId {
+               let offer = offer.contents.clone();
+               InvoiceRequestContentsWithoutPayerId {
+                       payer: PayerContents(metadata), offer, chain: None, amount_msats: None,
+                       features: InvoiceRequestFeatures::empty(), quantity: None, payer_note: None,
                }
        }
 
@@ -143,9 +205,10 @@ impl<'a> InvoiceRequestBuilder<'a> {
                self
        }
 
-       /// Builds an unsigned [`InvoiceRequest`] after checking for valid semantics. It can be signed
-       /// by [`UnsignedInvoiceRequest::sign`].
-       pub fn build(mut self) -> Result<UnsignedInvoiceRequest<'a>, SemanticError> {
+       fn build_with_checks(mut self) -> Result<
+               (UnsignedInvoiceRequest<'a>, Option<KeyPair>, Option<&'b Secp256k1<T>>),
+               SemanticError
+       > {
                #[cfg(feature = "std")] {
                        if self.offer.is_expired() {
                                return Err(SemanticError::AlreadyExpired);
@@ -170,13 +233,79 @@ impl<'a> InvoiceRequestBuilder<'a> {
                        self.invoice_request.amount_msats, self.invoice_request.quantity
                )?;
 
-               let InvoiceRequestBuilder { offer, invoice_request } = self;
-               Ok(UnsignedInvoiceRequest { offer, invoice_request })
+               Ok(self.build_without_checks())
+       }
+
+       fn build_without_checks(mut self) ->
+               (UnsignedInvoiceRequest<'a>, Option<KeyPair>, Option<&'b Secp256k1<T>>)
+       {
+               // Create the metadata for stateless verification of an Invoice.
+               let mut keys = None;
+               let secp_ctx = self.secp_ctx.clone();
+               if self.invoice_request.payer.0.has_derivation_material() {
+                       let mut metadata = core::mem::take(&mut self.invoice_request.payer.0);
+
+                       let mut tlv_stream = self.invoice_request.as_tlv_stream();
+                       debug_assert!(tlv_stream.2.payer_id.is_none());
+                       tlv_stream.0.metadata = None;
+                       if !metadata.derives_keys() {
+                               tlv_stream.2.payer_id = self.payer_id.as_ref();
+                       }
+
+                       let (derived_metadata, derived_keys) = metadata.derive_from(tlv_stream, self.secp_ctx);
+                       metadata = derived_metadata;
+                       keys = derived_keys;
+                       if let Some(keys) = keys {
+                               debug_assert!(self.payer_id.is_none());
+                               self.payer_id = Some(keys.public_key());
+                       }
+
+                       self.invoice_request.payer.0 = metadata;
+               }
+
+               debug_assert!(self.invoice_request.payer.0.as_bytes().is_some());
+               debug_assert!(self.payer_id.is_some());
+               let payer_id = self.payer_id.unwrap();
+
+               let unsigned_invoice = UnsignedInvoiceRequest {
+                       offer: self.offer,
+                       invoice_request: InvoiceRequestContents {
+                               inner: self.invoice_request,
+                               payer_id,
+                       },
+               };
+
+               (unsigned_invoice, keys, secp_ctx)
+       }
+}
+
+impl<'a, 'b, T: secp256k1::Signing> InvoiceRequestBuilder<'a, 'b, ExplicitPayerId, T> {
+       /// Builds an unsigned [`InvoiceRequest`] after checking for valid semantics. It can be signed
+       /// by [`UnsignedInvoiceRequest::sign`].
+       pub fn build(self) -> Result<UnsignedInvoiceRequest<'a>, SemanticError> {
+               let (unsigned_invoice_request, keys, _) = self.build_with_checks()?;
+               debug_assert!(keys.is_none());
+               Ok(unsigned_invoice_request)
+       }
+}
+
+impl<'a, 'b, T: secp256k1::Signing> InvoiceRequestBuilder<'a, 'b, DerivedPayerId, T> {
+       /// Builds a signed [`InvoiceRequest`] after checking for valid semantics.
+       pub fn build_and_sign(self) -> Result<InvoiceRequest, SemanticError> {
+               let (unsigned_invoice_request, keys, secp_ctx) = self.build_with_checks()?;
+               debug_assert!(keys.is_some());
+
+               let secp_ctx = secp_ctx.unwrap();
+               let keys = keys.unwrap();
+               let invoice_request = unsigned_invoice_request
+                       .sign::<_, Infallible>(|digest| Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys)))
+                       .unwrap();
+               Ok(invoice_request)
        }
 }
 
 #[cfg(test)]
-impl<'a> InvoiceRequestBuilder<'a> {
+impl<'a, 'b, P: PayerIdStrategy, T: secp256k1::Signing> InvoiceRequestBuilder<'a, 'b, P, T> {
        fn chain_unchecked(mut self, network: Network) -> Self {
                let chain = ChainHash::using_genesis_block(network);
                self.invoice_request.chain = Some(chain);
@@ -199,8 +328,7 @@ impl<'a> InvoiceRequestBuilder<'a> {
        }
 
        pub(super) fn build_unchecked(self) -> UnsignedInvoiceRequest<'a> {
-               let InvoiceRequestBuilder { offer, invoice_request } = self;
-               UnsignedInvoiceRequest { offer, invoice_request }
+               self.build_without_checks().0
        }
 }
 
@@ -250,7 +378,8 @@ impl<'a> UnsignedInvoiceRequest<'a> {
 ///
 /// [`Invoice`]: crate::offers::invoice::Invoice
 /// [`Offer`]: crate::offers::offer::Offer
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub struct InvoiceRequest {
        pub(super) bytes: Vec<u8>,
        pub(super) contents: InvoiceRequestContents,
@@ -260,15 +389,22 @@ pub struct InvoiceRequest {
 /// The contents of an [`InvoiceRequest`], which may be shared with an [`Invoice`].
 ///
 /// [`Invoice`]: crate::offers::invoice::Invoice
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub(super) struct InvoiceRequestContents {
+       pub(super) inner: InvoiceRequestContentsWithoutPayerId,
+       payer_id: PublicKey,
+}
+
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub(super) struct InvoiceRequestContentsWithoutPayerId {
        payer: PayerContents,
        pub(super) offer: OfferContents,
        chain: Option<ChainHash>,
        amount_msats: Option<u64>,
        features: InvoiceRequestFeatures,
        quantity: Option<u64>,
-       payer_id: PublicKey,
        payer_note: Option<String>,
 }
 
@@ -278,7 +414,7 @@ impl InvoiceRequest {
        ///
        /// [`payer_id`]: Self::payer_id
        pub fn metadata(&self) -> &[u8] {
-               &self.contents.payer.0[..]
+               self.contents.metadata()
        }
 
        /// A chain from [`Offer::chains`] that the offer is valid for.
@@ -291,17 +427,17 @@ impl InvoiceRequest {
        ///
        /// [`chain`]: Self::chain
        pub fn amount_msats(&self) -> Option<u64> {
-               self.contents.amount_msats
+               self.contents.inner.amount_msats
        }
 
        /// Features pertaining to requesting an invoice.
        pub fn features(&self) -> &InvoiceRequestFeatures {
-               &self.contents.features
+               &self.contents.inner.features
        }
 
        /// The quantity of the offer's item conforming to [`Offer::is_valid_quantity`].
        pub fn quantity(&self) -> Option<u64> {
-               self.contents.quantity
+               self.contents.inner.quantity
        }
 
        /// A possibly transient pubkey used to sign the invoice request.
@@ -312,7 +448,8 @@ impl InvoiceRequest {
        /// A payer-provided note which will be seen by the recipient and reflected back in the invoice
        /// response.
        pub fn payer_note(&self) -> Option<PrintableString> {
-               self.contents.payer_note.as_ref().map(|payer_note| PrintableString(payer_note.as_str()))
+               self.contents.inner.payer_note.as_ref()
+                       .map(|payer_note| PrintableString(payer_note.as_str()))
        }
 
        /// Signature of the invoice request using [`payer_id`].
@@ -322,18 +459,17 @@ impl InvoiceRequest {
                self.signature
        }
 
-       /// Creates an [`Invoice`] for the request with the given required fields and using the
+       /// Creates an [`InvoiceBuilder`] for the request with the given required fields and using the
        /// [`Duration`] since [`std::time::SystemTime::UNIX_EPOCH`] as the creation time.
        ///
        /// See [`InvoiceRequest::respond_with_no_std`] for further details where the aforementioned
        /// creation time is used for the `created_at` parameter.
        ///
-       /// [`Invoice`]: crate::offers::invoice::Invoice
        /// [`Duration`]: core::time::Duration
        #[cfg(feature = "std")]
        pub fn respond_with(
                &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash
-       ) -> Result<InvoiceBuilder, SemanticError> {
+       ) -> Result<InvoiceBuilder<ExplicitSigningPubkey>, SemanticError> {
                let created_at = std::time::SystemTime::now()
                        .duration_since(std::time::SystemTime::UNIX_EPOCH)
                        .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH");
@@ -341,7 +477,7 @@ impl InvoiceRequest {
                self.respond_with_no_std(payment_paths, payment_hash, created_at)
        }
 
-       /// Creates an [`Invoice`] for the request with the given required fields.
+       /// Creates an [`InvoiceBuilder`] for the request with the given required fields.
        ///
        /// Unless [`InvoiceBuilder::relative_expiry`] is set, the invoice will expire two hours after
        /// `created_at`, which is used to set [`Invoice::created_at`]. Useful for `no-std` builds where
@@ -357,12 +493,11 @@ impl InvoiceRequest {
        ///
        /// Errors if the request contains unknown required features.
        ///
-       /// [`Invoice`]: crate::offers::invoice::Invoice
        /// [`Invoice::created_at`]: crate::offers::invoice::Invoice::created_at
        pub fn respond_with_no_std(
                &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
                created_at: core::time::Duration
-       ) -> Result<InvoiceBuilder, SemanticError> {
+       ) -> Result<InvoiceBuilder<ExplicitSigningPubkey>, SemanticError> {
                if self.features().requires_unknown_bits() {
                        return Err(SemanticError::UnknownRequiredFeatures);
                }
@@ -370,6 +505,62 @@ impl InvoiceRequest {
                InvoiceBuilder::for_offer(self, payment_paths, created_at, payment_hash)
        }
 
+       /// Creates an [`InvoiceBuilder`] for the request using the given required fields and that uses
+       /// derived signing keys from the originating [`Offer`] to sign the [`Invoice`]. Must use the
+       /// same [`ExpandedKey`] as the one used to create the offer.
+       ///
+       /// See [`InvoiceRequest::respond_with`] for further details.
+       ///
+       /// [`Invoice`]: crate::offers::invoice::Invoice
+       #[cfg(feature = "std")]
+       pub fn verify_and_respond_using_derived_keys<T: secp256k1::Signing>(
+               &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
+               expanded_key: &ExpandedKey, secp_ctx: &Secp256k1<T>
+       ) -> Result<InvoiceBuilder<DerivedSigningPubkey>, SemanticError> {
+               let created_at = std::time::SystemTime::now()
+                       .duration_since(std::time::SystemTime::UNIX_EPOCH)
+                       .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH");
+
+               self.verify_and_respond_using_derived_keys_no_std(
+                       payment_paths, payment_hash, created_at, expanded_key, secp_ctx
+               )
+       }
+
+       /// Creates an [`InvoiceBuilder`] for the request using the given required fields and that uses
+       /// derived signing keys from the originating [`Offer`] to sign the [`Invoice`]. Must use the
+       /// same [`ExpandedKey`] as the one used to create the offer.
+       ///
+       /// See [`InvoiceRequest::respond_with_no_std`] for further details.
+       ///
+       /// [`Invoice`]: crate::offers::invoice::Invoice
+       pub fn verify_and_respond_using_derived_keys_no_std<T: secp256k1::Signing>(
+               &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
+               created_at: core::time::Duration, expanded_key: &ExpandedKey, secp_ctx: &Secp256k1<T>
+       ) -> Result<InvoiceBuilder<DerivedSigningPubkey>, SemanticError> {
+               if self.features().requires_unknown_bits() {
+                       return Err(SemanticError::UnknownRequiredFeatures);
+               }
+
+               let keys = match self.verify(expanded_key, secp_ctx) {
+                       Err(()) => return Err(SemanticError::InvalidMetadata),
+                       Ok(None) => return Err(SemanticError::InvalidMetadata),
+                       Ok(Some(keys)) => keys,
+               };
+
+               InvoiceBuilder::for_offer_using_keys(self, payment_paths, created_at, payment_hash, keys)
+       }
+
+       /// Verifies that the request was for an offer created using the given key. Returns the derived
+       /// keys need to sign an [`Invoice`] for the request if they could be extracted from the
+       /// metadata.
+       ///
+       /// [`Invoice`]: crate::offers::invoice::Invoice
+       pub fn verify<T: secp256k1::Signing>(
+               &self, key: &ExpandedKey, secp_ctx: &Secp256k1<T>
+       ) -> Result<Option<KeyPair>, ()> {
+               self.contents.inner.offer.verify(&self.bytes, key, secp_ctx)
+       }
+
        #[cfg(test)]
        fn as_tlv_stream(&self) -> FullInvoiceRequestTlvStreamRef {
                let (payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream) =
@@ -382,13 +573,41 @@ impl InvoiceRequest {
 }
 
 impl InvoiceRequestContents {
+       pub fn metadata(&self) -> &[u8] {
+               self.inner.metadata()
+       }
+
+       pub(super) fn derives_keys(&self) -> bool {
+               self.inner.payer.0.derives_keys()
+       }
+
+       pub(super) fn chain(&self) -> ChainHash {
+               self.inner.chain()
+       }
+
+       pub(super) fn payer_id(&self) -> PublicKey {
+               self.payer_id
+       }
+
+       pub(super) fn as_tlv_stream(&self) -> PartialInvoiceRequestTlvStreamRef {
+               let (payer, offer, mut invoice_request) = self.inner.as_tlv_stream();
+               invoice_request.payer_id = Some(&self.payer_id);
+               (payer, offer, invoice_request)
+       }
+}
+
+impl InvoiceRequestContentsWithoutPayerId {
+       pub(super) fn metadata(&self) -> &[u8] {
+               self.payer.0.as_bytes().map(|bytes| bytes.as_slice()).unwrap_or(&[])
+       }
+
        pub(super) fn chain(&self) -> ChainHash {
                self.chain.unwrap_or_else(|| self.offer.implied_chain())
        }
 
        pub(super) fn as_tlv_stream(&self) -> PartialInvoiceRequestTlvStreamRef {
                let payer = PayerTlvStreamRef {
-                       metadata: Some(&self.payer.0),
+                       metadata: self.payer.0.as_bytes(),
                };
 
                let offer = self.offer.as_tlv_stream();
@@ -403,7 +622,7 @@ impl InvoiceRequestContents {
                        amount: self.amount_msats,
                        features,
                        quantity: self.quantity,
-                       payer_id: Some(&self.payer_id),
+                       payer_id: None,
                        payer_note: self.payer_note.as_ref(),
                };
 
@@ -423,12 +642,20 @@ impl Writeable for InvoiceRequestContents {
        }
 }
 
-tlv_stream!(InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef, 80..160, {
+/// Valid type range for invoice_request TLV records.
+pub(super) const INVOICE_REQUEST_TYPES: core::ops::Range<u64> = 80..160;
+
+/// TLV record type for [`InvoiceRequest::payer_id`] and [`Refund::payer_id`].
+///
+/// [`Refund::payer_id`]: crate::offers::refund::Refund::payer_id
+pub(super) const INVOICE_REQUEST_PAYER_ID_TYPE: u64 = 88;
+
+tlv_stream!(InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef, INVOICE_REQUEST_TYPES, {
        (80, chain: ChainHash),
        (82, amount: (u64, HighZeroBytesDroppedBigSize)),
        (84, features: (InvoiceRequestFeatures, WithoutLength)),
        (86, quantity: (u64, HighZeroBytesDroppedBigSize)),
-       (88, payer_id: PublicKey),
+       (INVOICE_REQUEST_PAYER_ID_TYPE, payer_id: PublicKey),
        (89, payer_note: (String, WithoutLength)),
 });
 
@@ -498,7 +725,7 @@ impl TryFrom<PartialInvoiceRequestTlvStream> for InvoiceRequestContents {
 
                let payer = match metadata {
                        None => return Err(SemanticError::MissingPayerMetadata),
-                       Some(metadata) => PayerContents(metadata),
+                       Some(metadata) => PayerContents(Metadata::Bytes(metadata)),
                };
                let offer = OfferContents::try_from(offer_tlv_stream)?;
 
@@ -521,7 +748,10 @@ impl TryFrom<PartialInvoiceRequestTlvStream> for InvoiceRequestContents {
                };
 
                Ok(InvoiceRequestContents {
-                       payer, offer, chain, amount_msats: amount, features, quantity, payer_id, payer_note,
+                       inner: InvoiceRequestContentsWithoutPayerId {
+                               payer, offer, chain, amount_msats: amount, features, quantity, payer_note,
+                       },
+                       payer_id,
                })
        }
 }
@@ -532,47 +762,24 @@ mod tests {
 
        use bitcoin::blockdata::constants::ChainHash;
        use bitcoin::network::constants::Network;
-       use bitcoin::secp256k1::{KeyPair, Message, PublicKey, Secp256k1, SecretKey, self};
-       use bitcoin::secp256k1::schnorr::Signature;
+       use bitcoin::secp256k1::{KeyPair, Secp256k1, SecretKey, self};
        use core::convert::{Infallible, TryFrom};
        use core::num::NonZeroU64;
        #[cfg(feature = "std")]
        use core::time::Duration;
+       use crate::chain::keysinterface::KeyMaterial;
        use crate::ln::features::InvoiceRequestFeatures;
+       use crate::ln::inbound_payment::ExpandedKey;
        use crate::ln::msgs::{DecodeError, MAX_VALUE_MSAT};
+       use crate::offers::invoice::{Invoice, SIGNATURE_TAG as INVOICE_SIGNATURE_TAG};
        use crate::offers::merkle::{SignError, SignatureTlvStreamRef, self};
        use crate::offers::offer::{Amount, OfferBuilder, OfferTlvStreamRef, Quantity};
        use crate::offers::parse::{ParseError, SemanticError};
        use crate::offers::payer::PayerTlvStreamRef;
+       use crate::offers::test_utils::*;
        use crate::util::ser::{BigSize, Writeable};
        use crate::util::string::PrintableString;
 
-       fn payer_keys() -> KeyPair {
-               let secp_ctx = Secp256k1::new();
-               KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap())
-       }
-
-       fn payer_sign(digest: &Message) -> Result<Signature, Infallible> {
-               let secp_ctx = Secp256k1::new();
-               let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap());
-               Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys))
-       }
-
-       fn payer_pubkey() -> PublicKey {
-               payer_keys().public_key()
-       }
-
-       fn recipient_sign(digest: &Message) -> Result<Signature, Infallible> {
-               let secp_ctx = Secp256k1::new();
-               let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap());
-               Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys))
-       }
-
-       fn recipient_pubkey() -> PublicKey {
-               let secp_ctx = Secp256k1::new();
-               KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap()).public_key()
-       }
-
        #[test]
        fn builds_invoice_request_with_defaults() {
                let invoice_request = OfferBuilder::new("foo".into(), recipient_pubkey())
@@ -661,6 +868,148 @@ mod tests {
                }
        }
 
+       #[test]
+       fn builds_invoice_request_with_derived_metadata() {
+               let payer_id = payer_pubkey();
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let offer = OfferBuilder::new("foo".into(), recipient_pubkey())
+                       .amount_msats(1000)
+                       .build().unwrap();
+               let invoice_request = offer
+                       .request_invoice_deriving_metadata(payer_id, &expanded_key, &entropy)
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert_eq!(invoice_request.payer_id(), payer_pubkey());
+
+               let invoice = invoice_request.respond_with_no_std(payment_paths(), payment_hash(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered fields
+               let (
+                       payer_tlv_stream, offer_tlv_stream, mut invoice_request_tlv_stream,
+                       mut invoice_tlv_stream, mut signature_tlv_stream
+               ) = invoice.as_tlv_stream();
+               invoice_request_tlv_stream.amount = Some(2000);
+               invoice_tlv_stream.amount = Some(2000);
+
+               let tlv_stream =
+                       (payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream);
+               let mut bytes = Vec::new();
+               tlv_stream.write(&mut bytes).unwrap();
+
+               let signature = merkle::sign_message(
+                       recipient_sign, INVOICE_SIGNATURE_TAG, &bytes, recipient_pubkey()
+               ).unwrap();
+               signature_tlv_stream.signature = Some(&signature);
+
+               let mut encoded_invoice = bytes;
+               signature_tlv_stream.write(&mut encoded_invoice).unwrap();
+
+               let invoice = Invoice::try_from(encoded_invoice).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered metadata
+               let (
+                       mut payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream,
+                       mut signature_tlv_stream
+               ) = invoice.as_tlv_stream();
+               let metadata = payer_tlv_stream.metadata.unwrap().iter().copied().rev().collect();
+               payer_tlv_stream.metadata = Some(&metadata);
+
+               let tlv_stream =
+                       (payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream);
+               let mut bytes = Vec::new();
+               tlv_stream.write(&mut bytes).unwrap();
+
+               let signature = merkle::sign_message(
+                       recipient_sign, INVOICE_SIGNATURE_TAG, &bytes, recipient_pubkey()
+               ).unwrap();
+               signature_tlv_stream.signature = Some(&signature);
+
+               let mut encoded_invoice = bytes;
+               signature_tlv_stream.write(&mut encoded_invoice).unwrap();
+
+               let invoice = Invoice::try_from(encoded_invoice).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+       }
+
+       #[test]
+       fn builds_invoice_request_with_derived_payer_id() {
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let offer = OfferBuilder::new("foo".into(), recipient_pubkey())
+                       .amount_msats(1000)
+                       .build().unwrap();
+               let invoice_request = offer
+                       .request_invoice_deriving_payer_id(&expanded_key, &entropy, &secp_ctx)
+                       .unwrap()
+                       .build_and_sign()
+                       .unwrap();
+
+               let invoice = invoice_request.respond_with_no_std(payment_paths(), payment_hash(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered fields
+               let (
+                       payer_tlv_stream, offer_tlv_stream, mut invoice_request_tlv_stream,
+                       mut invoice_tlv_stream, mut signature_tlv_stream
+               ) = invoice.as_tlv_stream();
+               invoice_request_tlv_stream.amount = Some(2000);
+               invoice_tlv_stream.amount = Some(2000);
+
+               let tlv_stream =
+                       (payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream);
+               let mut bytes = Vec::new();
+               tlv_stream.write(&mut bytes).unwrap();
+
+               let signature = merkle::sign_message(
+                       recipient_sign, INVOICE_SIGNATURE_TAG, &bytes, recipient_pubkey()
+               ).unwrap();
+               signature_tlv_stream.signature = Some(&signature);
+
+               let mut encoded_invoice = bytes;
+               signature_tlv_stream.write(&mut encoded_invoice).unwrap();
+
+               let invoice = Invoice::try_from(encoded_invoice).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered payer id
+               let (
+                       payer_tlv_stream, offer_tlv_stream, mut invoice_request_tlv_stream, invoice_tlv_stream,
+                       mut signature_tlv_stream
+               ) = invoice.as_tlv_stream();
+               let payer_id = pubkey(1);
+               invoice_request_tlv_stream.payer_id = Some(&payer_id);
+
+               let tlv_stream =
+                       (payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream);
+               let mut bytes = Vec::new();
+               tlv_stream.write(&mut bytes).unwrap();
+
+               let signature = merkle::sign_message(
+                       recipient_sign, INVOICE_SIGNATURE_TAG, &bytes, recipient_pubkey()
+               ).unwrap();
+               signature_tlv_stream.signature = Some(&signature);
+
+               let mut encoded_invoice = bytes;
+               signature_tlv_stream.write(&mut encoded_invoice).unwrap();
+
+               let invoice = Invoice::try_from(encoded_invoice).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+       }
+
        #[test]
        fn builds_invoice_request_with_chain() {
                let mainnet = ChainHash::using_genesis_block(Network::Bitcoin);
@@ -1008,12 +1357,28 @@ mod tests {
                }
        }
 
+       #[test]
+       fn fails_responding_with_unknown_required_features() {
+               match OfferBuilder::new("foo".into(), recipient_pubkey())
+                       .amount_msats(1000)
+                       .build().unwrap()
+                       .request_invoice(vec![42; 32], payer_pubkey()).unwrap()
+                       .features_unchecked(InvoiceRequestFeatures::unknown())
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap()
+                       .respond_with_no_std(payment_paths(), payment_hash(), now())
+               {
+                       Ok(_) => panic!("expected error"),
+                       Err(e) => assert_eq!(e, SemanticError::UnknownRequiredFeatures),
+               }
+       }
+
        #[test]
        fn parses_invoice_request_with_metadata() {
                let invoice_request = OfferBuilder::new("foo".into(), recipient_pubkey())
                        .amount_msats(1000)
                        .build().unwrap()
-                       .request_invoice(vec![42; 32], payer_pubkey()).unwrap()
+                       .request_invoice(vec![1; 32], payer_pubkey()).unwrap()
                        .build().unwrap()
                        .sign(payer_sign).unwrap();
 
index 94a1eac0ca416bac20ad7a456c224d84706a73d0..3b05899a8f59214872a1075179fc9428b48412fa 100644 (file)
@@ -143,28 +143,38 @@ fn tagged_branch_hash_from_engine(
 
 /// [`Iterator`] over a sequence of bytes yielding [`TlvRecord`]s. The input is assumed to be a
 /// well-formed TLV stream.
-struct TlvStream<'a> {
+#[derive(Clone)]
+pub(super) struct TlvStream<'a> {
        data: io::Cursor<&'a [u8]>,
 }
 
 impl<'a> TlvStream<'a> {
-       fn new(data: &'a [u8]) -> Self {
+       pub fn new(data: &'a [u8]) -> Self {
                Self {
                        data: io::Cursor::new(data),
                }
        }
 
+       pub fn range<T>(self, types: T) -> impl core::iter::Iterator<Item = TlvRecord<'a>>
+       where
+               T: core::ops::RangeBounds<u64> + Clone,
+       {
+               let take_range = types.clone();
+               self.skip_while(move |record| !types.contains(&record.r#type))
+                       .take_while(move |record| take_range.contains(&record.r#type))
+       }
+
        fn skip_signatures(self) -> core::iter::Filter<TlvStream<'a>, fn(&TlvRecord) -> bool> {
                self.filter(|record| !SIGNATURE_TYPES.contains(&record.r#type))
        }
 }
 
 /// A slice into a [`TlvStream`] for a record.
-struct TlvRecord<'a> {
-       r#type: u64,
+pub(super) struct TlvRecord<'a> {
+       pub(super) r#type: u64,
        type_bytes: &'a [u8],
        // The entire TLV record.
-       record_bytes: &'a [u8],
+       pub(super) record_bytes: &'a [u8],
 }
 
 impl<'a> Iterator for TlvStream<'a> {
@@ -212,7 +222,7 @@ impl<'a> Writeable for WithoutSignatures<'a> {
 
 #[cfg(test)]
 mod tests {
-       use super::{TlvStream, WithoutSignatures};
+       use super::{SIGNATURE_TYPES, TlvStream, WithoutSignatures};
 
        use bitcoin::hashes::{Hash, sha256};
        use bitcoin::secp256k1::{KeyPair, Secp256k1, SecretKey};
@@ -302,6 +312,38 @@ mod tests {
                );
        }
 
+       #[test]
+       fn iterates_over_tlv_stream_range() {
+               let secp_ctx = Secp256k1::new();
+               let recipient_pubkey = {
+                       let secret_key = SecretKey::from_slice(&[41; 32]).unwrap();
+                       KeyPair::from_secret_key(&secp_ctx, &secret_key).public_key()
+               };
+               let payer_keys = {
+                       let secret_key = SecretKey::from_slice(&[42; 32]).unwrap();
+                       KeyPair::from_secret_key(&secp_ctx, &secret_key)
+               };
+
+               let invoice_request = OfferBuilder::new("foo".into(), recipient_pubkey)
+                       .amount_msats(100)
+                       .build_unchecked()
+                       .request_invoice(vec![0; 8], payer_keys.public_key()).unwrap()
+                       .build_unchecked()
+                       .sign::<_, Infallible>(|digest| Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &payer_keys)))
+                       .unwrap();
+
+               let tlv_stream = TlvStream::new(&invoice_request.bytes).range(0..1)
+                       .chain(TlvStream::new(&invoice_request.bytes).range(1..80))
+                       .chain(TlvStream::new(&invoice_request.bytes).range(80..160))
+                       .chain(TlvStream::new(&invoice_request.bytes).range(160..240))
+                       .chain(TlvStream::new(&invoice_request.bytes).range(SIGNATURE_TYPES))
+                       .map(|r| r.record_bytes.to_vec())
+                       .flatten()
+                       .collect::<Vec<u8>>();
+
+               assert_eq!(tlv_stream, invoice_request.bytes);
+       }
+
        impl AsRef<[u8]> for InvoiceRequest {
                fn as_ref(&self) -> &[u8] {
                        &self.bytes
index 2da6fac08ff929e788af18ad1b8970ad71388730..0fb20f42d79e61b394cf46b59e6794e7b42a76fe 100644 (file)
@@ -19,3 +19,7 @@ pub mod offer;
 pub mod parse;
 mod payer;
 pub mod refund;
+#[allow(unused)]
+pub(crate) mod signer;
+#[cfg(test)]
+mod test_utils;
index 405e2e278d8073eac345a801f88906422989f173..192317240f94df2e61ffbe7269f97cd3d4d22701 100644 (file)
 
 use bitcoin::blockdata::constants::ChainHash;
 use bitcoin::network::constants::Network;
-use bitcoin::secp256k1::PublicKey;
+use bitcoin::secp256k1::{KeyPair, PublicKey, Secp256k1, self};
 use core::convert::TryFrom;
 use core::num::NonZeroU64;
+use core::ops::Deref;
 use core::str::FromStr;
 use core::time::Duration;
+use crate::chain::keysinterface::EntropySource;
 use crate::io;
 use crate::ln::features::OfferFeatures;
+use crate::ln::inbound_payment::{ExpandedKey, IV_LEN, Nonce};
 use crate::ln::msgs::MAX_VALUE_MSAT;
-use crate::offers::invoice_request::InvoiceRequestBuilder;
+use crate::offers::invoice_request::{DerivedPayerId, ExplicitPayerId, InvoiceRequestBuilder};
+use crate::offers::merkle::TlvStream;
 use crate::offers::parse::{Bech32Encode, ParseError, ParsedMessage, SemanticError};
+use crate::offers::signer::{Metadata, MetadataMaterial, self};
 use crate::onion_message::BlindedPath;
 use crate::util::ser::{HighZeroBytesDroppedBigSize, WithoutLength, Writeable, Writer};
 use crate::util::string::PrintableString;
@@ -87,30 +92,90 @@ use crate::prelude::*;
 #[cfg(feature = "std")]
 use std::time::SystemTime;
 
+pub(super) const IV_BYTES: &[u8; IV_LEN] = b"LDK Offer ~~~~~~";
+
 /// Builds an [`Offer`] for the "offer to be paid" flow.
 ///
 /// See [module-level documentation] for usage.
 ///
 /// [module-level documentation]: self
-pub struct OfferBuilder {
+pub struct OfferBuilder<'a, M: MetadataStrategy, T: secp256k1::Signing> {
        offer: OfferContents,
+       metadata_strategy: core::marker::PhantomData<M>,
+       secp_ctx: Option<&'a Secp256k1<T>>,
 }
 
-impl OfferBuilder {
+/// Indicates how [`Offer::metadata`] may be set.
+pub trait MetadataStrategy {}
+
+/// [`Offer::metadata`] may be explicitly set or left empty.
+pub struct ExplicitMetadata {}
+
+/// [`Offer::metadata`] will be derived.
+pub struct DerivedMetadata {}
+
+impl MetadataStrategy for ExplicitMetadata {}
+impl MetadataStrategy for DerivedMetadata {}
+
+impl<'a> OfferBuilder<'a, ExplicitMetadata, secp256k1::SignOnly> {
        /// Creates a new builder for an offer setting the [`Offer::description`] and using the
        /// [`Offer::signing_pubkey`] for signing invoices. The associated secret key must be remembered
        /// while the offer is valid.
        ///
        /// Use a different pubkey per offer to avoid correlating offers.
        pub fn new(description: String, signing_pubkey: PublicKey) -> Self {
-               let offer = OfferContents {
-                       chains: None, metadata: None, amount: None, description,
-                       features: OfferFeatures::empty(), absolute_expiry: None, issuer: None, paths: None,
-                       supported_quantity: Quantity::One, signing_pubkey,
-               };
-               OfferBuilder { offer }
+               OfferBuilder {
+                       offer: OfferContents {
+                               chains: None, metadata: None, amount: None, description,
+                               features: OfferFeatures::empty(), absolute_expiry: None, issuer: None, paths: None,
+                               supported_quantity: Quantity::One, signing_pubkey,
+                       },
+                       metadata_strategy: core::marker::PhantomData,
+                       secp_ctx: None,
+               }
+       }
+
+       /// Sets the [`Offer::metadata`] to the given bytes.
+       ///
+       /// Successive calls to this method will override the previous setting.
+       pub fn metadata(mut self, metadata: Vec<u8>) -> Result<Self, SemanticError> {
+               self.offer.metadata = Some(Metadata::Bytes(metadata));
+               Ok(self)
+       }
+}
+
+impl<'a, T: secp256k1::Signing> OfferBuilder<'a, DerivedMetadata, T> {
+       /// Similar to [`OfferBuilder::new`] except, if [`OfferBuilder::path`] is called, the signing
+       /// pubkey is derived from the given [`ExpandedKey`] and [`EntropySource`]. This provides
+       /// recipient privacy by using a different signing pubkey for each offer. Otherwise, the
+       /// provided `node_id` is used for the signing pubkey.
+       ///
+       /// Also, sets the metadata when [`OfferBuilder::build`] is called such that it can be used by
+       /// [`InvoiceRequest::verify`] to determine if the request was produced for the offer given an
+       /// [`ExpandedKey`].
+       ///
+       /// [`InvoiceRequest::verify`]: crate::offers::invoice_request::InvoiceRequest::verify
+       /// [`ExpandedKey`]: crate::ln::inbound_payment::ExpandedKey
+       pub fn deriving_signing_pubkey<ES: Deref>(
+               description: String, node_id: PublicKey, expanded_key: &ExpandedKey, entropy_source: ES,
+               secp_ctx: &'a Secp256k1<T>
+       ) -> Self where ES::Target: EntropySource {
+               let nonce = Nonce::from_entropy_source(entropy_source);
+               let derivation_material = MetadataMaterial::new(nonce, expanded_key, IV_BYTES);
+               let metadata = Metadata::DerivedSigningPubkey(derivation_material);
+               OfferBuilder {
+                       offer: OfferContents {
+                               chains: None, metadata: Some(metadata), amount: None, description,
+                               features: OfferFeatures::empty(), absolute_expiry: None, issuer: None, paths: None,
+                               supported_quantity: Quantity::One, signing_pubkey: node_id,
+                       },
+                       metadata_strategy: core::marker::PhantomData,
+                       secp_ctx: Some(secp_ctx),
+               }
        }
+}
 
+impl<'a, M: MetadataStrategy, T: secp256k1::Signing> OfferBuilder<'a, M, T> {
        /// Adds the chain hash of the given [`Network`] to [`Offer::chains`]. If not called,
        /// the chain hash of [`Network::Bitcoin`] is assumed to be the only one supported.
        ///
@@ -127,14 +192,6 @@ impl OfferBuilder {
                self
        }
 
-       /// Sets the [`Offer::metadata`].
-       ///
-       /// Successive calls to this method will override the previous setting.
-       pub fn metadata(mut self, metadata: Vec<u8>) -> Self {
-               self.offer.metadata = Some(metadata);
-               self
-       }
-
        /// Sets the [`Offer::amount`] as an [`Amount::Bitcoin`].
        ///
        /// Successive calls to this method will override the previous setting.
@@ -204,28 +261,50 @@ impl OfferBuilder {
                        }
                }
 
+               Ok(self.build_without_checks())
+       }
+
+       fn build_without_checks(mut self) -> Offer {
+               // Create the metadata for stateless verification of an InvoiceRequest.
+               if let Some(mut metadata) = self.offer.metadata.take() {
+                       if metadata.has_derivation_material() {
+                               if self.offer.paths.is_none() {
+                                       metadata = metadata.without_keys();
+                               }
+
+                               let mut tlv_stream = self.offer.as_tlv_stream();
+                               debug_assert_eq!(tlv_stream.metadata, None);
+                               tlv_stream.metadata = None;
+                               if metadata.derives_keys() {
+                                       tlv_stream.node_id = None;
+                               }
+
+                               let (derived_metadata, keys) = metadata.derive_from(tlv_stream, self.secp_ctx);
+                               metadata = derived_metadata;
+                               if let Some(keys) = keys {
+                                       self.offer.signing_pubkey = keys.public_key();
+                               }
+                       }
+
+                       self.offer.metadata = Some(metadata);
+               }
+
                let mut bytes = Vec::new();
                self.offer.write(&mut bytes).unwrap();
 
-               Ok(Offer {
-                       bytes,
-                       contents: self.offer,
-               })
+               Offer { bytes, contents: self.offer }
        }
 }
 
 #[cfg(test)]
-impl OfferBuilder {
+impl<'a, M: MetadataStrategy, T: secp256k1::Signing> OfferBuilder<'a, M, T> {
        fn features_unchecked(mut self, features: OfferFeatures) -> Self {
                self.offer.features = features;
                self
        }
 
        pub(super) fn build_unchecked(self) -> Offer {
-               let mut bytes = Vec::new();
-               self.offer.write(&mut bytes).unwrap();
-
-               Offer { bytes, contents: self.offer }
+               self.build_without_checks()
        }
 }
 
@@ -242,7 +321,8 @@ impl OfferBuilder {
 ///
 /// [`InvoiceRequest`]: crate::offers::invoice_request::InvoiceRequest
 /// [`Invoice`]: crate::offers::invoice::Invoice
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub struct Offer {
        // The serialized offer. Needed when creating an `InvoiceRequest` if the offer contains unknown
        // fields.
@@ -254,10 +334,11 @@ pub struct Offer {
 ///
 /// [`InvoiceRequest`]: crate::offers::invoice_request::InvoiceRequest
 /// [`Invoice`]: crate::offers::invoice::Invoice
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub(super) struct OfferContents {
        chains: Option<Vec<ChainHash>>,
-       metadata: Option<Vec<u8>>,
+       metadata: Option<Metadata>,
        amount: Option<Amount>,
        description: String,
        features: OfferFeatures,
@@ -292,7 +373,7 @@ impl Offer {
        /// Opaque bytes set by the originator. Useful for authentication and validating fields since it
        /// is reflected in `invoice_request` messages along with all the other fields from the `offer`.
        pub fn metadata(&self) -> Option<&Vec<u8>> {
-               self.contents.metadata.as_ref()
+               self.contents.metadata()
        }
 
        /// The minimum amount required for a successful payment of a single item.
@@ -303,7 +384,7 @@ impl Offer {
        /// A complete description of the purpose of the payment. Intended to be displayed to the user
        /// but with the caveat that it has not been verified in any way.
        pub fn description(&self) -> PrintableString {
-               PrintableString(&self.contents.description)
+               self.contents.description()
        }
 
        /// Features pertaining to the offer.
@@ -358,8 +439,53 @@ impl Offer {
                self.contents.signing_pubkey()
        }
 
-       /// Creates an [`InvoiceRequest`] for the offer with the given `metadata` and `payer_id`, which
-       /// will be reflected in the `Invoice` response.
+       /// Similar to [`Offer::request_invoice`] except it:
+       /// - derives the [`InvoiceRequest::payer_id`] such that a different key can be used for each
+       ///   request, and
+       /// - sets the [`InvoiceRequest::metadata`] when [`InvoiceRequestBuilder::build`] is called such
+       ///   that it can be used by [`Invoice::verify`] to determine if the invoice was requested using
+       ///   a base [`ExpandedKey`] from which the payer id was derived.
+       ///
+       /// Useful to protect the sender's privacy.
+       ///
+       /// [`InvoiceRequest::payer_id`]: crate::offers::invoice_request::InvoiceRequest::payer_id
+       /// [`InvoiceRequest::metadata`]: crate::offers::invoice_request::InvoiceRequest::metadata
+       /// [`Invoice::verify`]: crate::offers::invoice::Invoice::verify
+       /// [`ExpandedKey`]: crate::ln::inbound_payment::ExpandedKey
+       pub fn request_invoice_deriving_payer_id<'a, 'b, ES: Deref, T: secp256k1::Signing>(
+               &'a self, expanded_key: &ExpandedKey, entropy_source: ES, secp_ctx: &'b Secp256k1<T>
+       ) -> Result<InvoiceRequestBuilder<'a, 'b, DerivedPayerId, T>, SemanticError>
+       where
+               ES::Target: EntropySource,
+       {
+               if self.features().requires_unknown_bits() {
+                       return Err(SemanticError::UnknownRequiredFeatures);
+               }
+
+               Ok(InvoiceRequestBuilder::deriving_payer_id(self, expanded_key, entropy_source, secp_ctx))
+       }
+
+       /// Similar to [`Offer::request_invoice_deriving_payer_id`] except uses `payer_id` for the
+       /// [`InvoiceRequest::payer_id`] instead of deriving a different key for each request.
+       ///
+       /// Useful for recurring payments using the same `payer_id` with different invoices.
+       ///
+       /// [`InvoiceRequest::payer_id`]: crate::offers::invoice_request::InvoiceRequest::payer_id
+       pub fn request_invoice_deriving_metadata<ES: Deref>(
+               &self, payer_id: PublicKey, expanded_key: &ExpandedKey, entropy_source: ES
+       ) -> Result<InvoiceRequestBuilder<ExplicitPayerId, secp256k1::SignOnly>, SemanticError>
+       where
+               ES::Target: EntropySource,
+       {
+               if self.features().requires_unknown_bits() {
+                       return Err(SemanticError::UnknownRequiredFeatures);
+               }
+
+               Ok(InvoiceRequestBuilder::deriving_metadata(self, payer_id, expanded_key, entropy_source))
+       }
+
+       /// Creates an [`InvoiceRequestBuilder`] for the offer with the given `metadata` and `payer_id`,
+       /// which will be reflected in the `Invoice` response.
        ///
        /// The `metadata` is useful for including information about the derivation of `payer_id` such
        /// that invoice response handling can be stateless. Also serves as payer-provided entropy while
@@ -373,7 +499,7 @@ impl Offer {
        /// [`InvoiceRequest`]: crate::offers::invoice_request::InvoiceRequest
        pub fn request_invoice(
                &self, metadata: Vec<u8>, payer_id: PublicKey
-       ) -> Result<InvoiceRequestBuilder, SemanticError> {
+       ) -> Result<InvoiceRequestBuilder<ExplicitPayerId, secp256k1::SignOnly>, SemanticError> {
                if self.features().requires_unknown_bits() {
                        return Err(SemanticError::UnknownRequiredFeatures);
                }
@@ -406,6 +532,14 @@ impl OfferContents {
                self.chains().contains(&chain)
        }
 
+       pub fn metadata(&self) -> Option<&Vec<u8>> {
+               self.metadata.as_ref().and_then(|metadata| metadata.as_bytes())
+       }
+
+       pub fn description(&self) -> PrintableString {
+               PrintableString(&self.description)
+       }
+
        #[cfg(feature = "std")]
        pub(super) fn is_expired(&self) -> bool {
                match self.absolute_expiry {
@@ -483,6 +617,27 @@ impl OfferContents {
                self.signing_pubkey
        }
 
+       /// Verifies that the offer metadata was produced from the offer in the TLV stream.
+       pub(super) fn verify<T: secp256k1::Signing>(
+               &self, bytes: &[u8], key: &ExpandedKey, secp_ctx: &Secp256k1<T>
+       ) -> Result<Option<KeyPair>, ()> {
+               match self.metadata() {
+                       Some(metadata) => {
+                               let tlv_stream = TlvStream::new(bytes).range(OFFER_TYPES).filter(|record| {
+                                       match record.r#type {
+                                               OFFER_METADATA_TYPE => false,
+                                               OFFER_NODE_ID_TYPE => !self.metadata.as_ref().unwrap().derives_keys(),
+                                               _ => true,
+                                       }
+                               });
+                               signer::verify_metadata(
+                                       metadata, key, IV_BYTES, self.signing_pubkey(), tlv_stream, secp_ctx
+                               )
+                       },
+                       None => Err(()),
+               }
+       }
+
        pub(super) fn as_tlv_stream(&self) -> OfferTlvStreamRef {
                let (currency, amount) = match &self.amount {
                        None => (None, None),
@@ -498,7 +653,7 @@ impl OfferContents {
 
                OfferTlvStreamRef {
                        chains: self.chains.as_ref(),
-                       metadata: self.metadata.as_ref(),
+                       metadata: self.metadata(),
                        currency,
                        amount,
                        description: Some(&self.description),
@@ -570,9 +725,18 @@ impl Quantity {
        }
 }
 
-tlv_stream!(OfferTlvStream, OfferTlvStreamRef, 1..80, {
+/// Valid type range for offer TLV records.
+pub(super) const OFFER_TYPES: core::ops::Range<u64> = 1..80;
+
+/// TLV record type for [`Offer::metadata`].
+const OFFER_METADATA_TYPE: u64 = 4;
+
+/// TLV record type for [`Offer::signing_pubkey`].
+const OFFER_NODE_ID_TYPE: u64 = 22;
+
+tlv_stream!(OfferTlvStream, OfferTlvStreamRef, OFFER_TYPES, {
        (2, chains: (Vec<ChainHash>, WithoutLength)),
-       (4, metadata: (Vec<u8>, WithoutLength)),
+       (OFFER_METADATA_TYPE, metadata: (Vec<u8>, WithoutLength)),
        (6, currency: CurrencyCode),
        (8, amount: (u64, HighZeroBytesDroppedBigSize)),
        (10, description: (String, WithoutLength)),
@@ -581,7 +745,7 @@ tlv_stream!(OfferTlvStream, OfferTlvStreamRef, 1..80, {
        (16, paths: (Vec<BlindedPath>, WithoutLength)),
        (18, issuer: (String, WithoutLength)),
        (20, quantity_max: (u64, HighZeroBytesDroppedBigSize)),
-       (22, node_id: PublicKey),
+       (OFFER_NODE_ID_TYPE, node_id: PublicKey),
 });
 
 impl Bech32Encode for Offer {
@@ -616,6 +780,8 @@ impl TryFrom<OfferTlvStream> for OfferContents {
                        issuer, quantity_max, node_id,
                } = tlv_stream;
 
+               let metadata = metadata.map(|metadata| Metadata::Bytes(metadata));
+
                let amount = match (currency, amount) {
                        (None, None) => None,
                        (None, Some(amount_msats)) if amount_msats > MAX_VALUE_MSAT => {
@@ -666,26 +832,20 @@ mod tests {
 
        use bitcoin::blockdata::constants::ChainHash;
        use bitcoin::network::constants::Network;
-       use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey};
+       use bitcoin::secp256k1::Secp256k1;
        use core::convert::TryFrom;
        use core::num::NonZeroU64;
        use core::time::Duration;
+       use crate::chain::keysinterface::KeyMaterial;
        use crate::ln::features::OfferFeatures;
+       use crate::ln::inbound_payment::ExpandedKey;
        use crate::ln::msgs::{DecodeError, MAX_VALUE_MSAT};
        use crate::offers::parse::{ParseError, SemanticError};
+       use crate::offers::test_utils::*;
        use crate::onion_message::{BlindedHop, BlindedPath};
        use crate::util::ser::{BigSize, Writeable};
        use crate::util::string::PrintableString;
 
-       fn pubkey(byte: u8) -> PublicKey {
-               let secp_ctx = Secp256k1::new();
-               PublicKey::from_secret_key(&secp_ctx, &privkey(byte))
-       }
-
-       fn privkey(byte: u8) -> SecretKey {
-               SecretKey::from_slice(&[byte; 32]).unwrap()
-       }
-
        #[test]
        fn builds_offer_with_defaults() {
                let offer = OfferBuilder::new("foo".into(), pubkey(42)).build().unwrap();
@@ -774,21 +934,125 @@ mod tests {
        #[test]
        fn builds_offer_with_metadata() {
                let offer = OfferBuilder::new("foo".into(), pubkey(42))
-                       .metadata(vec![42; 32])
+                       .metadata(vec![42; 32]).unwrap()
                        .build()
                        .unwrap();
                assert_eq!(offer.metadata(), Some(&vec![42; 32]));
                assert_eq!(offer.as_tlv_stream().metadata, Some(&vec![42; 32]));
 
                let offer = OfferBuilder::new("foo".into(), pubkey(42))
-                       .metadata(vec![42; 32])
-                       .metadata(vec![43; 32])
+                       .metadata(vec![42; 32]).unwrap()
+                       .metadata(vec![43; 32]).unwrap()
                        .build()
                        .unwrap();
                assert_eq!(offer.metadata(), Some(&vec![43; 32]));
                assert_eq!(offer.as_tlv_stream().metadata, Some(&vec![43; 32]));
        }
 
+       #[test]
+       fn builds_offer_with_metadata_derived() {
+               let desc = "foo".to_string();
+               let node_id = recipient_pubkey();
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let offer = OfferBuilder
+                       ::deriving_signing_pubkey(desc, node_id, &expanded_key, &entropy, &secp_ctx)
+                       .amount_msats(1000)
+                       .build().unwrap();
+               assert_eq!(offer.signing_pubkey(), node_id);
+
+               let invoice_request = offer.request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert!(invoice_request.verify(&expanded_key, &secp_ctx).is_ok());
+
+               // Fails verification with altered offer field
+               let mut tlv_stream = offer.as_tlv_stream();
+               tlv_stream.amount = Some(100);
+
+               let mut encoded_offer = Vec::new();
+               tlv_stream.write(&mut encoded_offer).unwrap();
+
+               let invoice_request = Offer::try_from(encoded_offer).unwrap()
+                       .request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert!(invoice_request.verify(&expanded_key, &secp_ctx).is_err());
+
+               // Fails verification with altered metadata
+               let mut tlv_stream = offer.as_tlv_stream();
+               let metadata = tlv_stream.metadata.unwrap().iter().copied().rev().collect();
+               tlv_stream.metadata = Some(&metadata);
+
+               let mut encoded_offer = Vec::new();
+               tlv_stream.write(&mut encoded_offer).unwrap();
+
+               let invoice_request = Offer::try_from(encoded_offer).unwrap()
+                       .request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert!(invoice_request.verify(&expanded_key, &secp_ctx).is_err());
+       }
+
+       #[test]
+       fn builds_offer_with_derived_signing_pubkey() {
+               let desc = "foo".to_string();
+               let node_id = recipient_pubkey();
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let blinded_path = BlindedPath {
+                       introduction_node_id: pubkey(40),
+                       blinding_point: pubkey(41),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: pubkey(42), encrypted_payload: vec![0; 43] },
+                               BlindedHop { blinded_node_id: node_id, encrypted_payload: vec![0; 44] },
+                       ],
+               };
+
+               let offer = OfferBuilder
+                       ::deriving_signing_pubkey(desc, node_id, &expanded_key, &entropy, &secp_ctx)
+                       .amount_msats(1000)
+                       .path(blinded_path)
+                       .build().unwrap();
+               assert_ne!(offer.signing_pubkey(), node_id);
+
+               let invoice_request = offer.request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert!(invoice_request.verify(&expanded_key, &secp_ctx).is_ok());
+
+               // Fails verification with altered offer field
+               let mut tlv_stream = offer.as_tlv_stream();
+               tlv_stream.amount = Some(100);
+
+               let mut encoded_offer = Vec::new();
+               tlv_stream.write(&mut encoded_offer).unwrap();
+
+               let invoice_request = Offer::try_from(encoded_offer).unwrap()
+                       .request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert!(invoice_request.verify(&expanded_key, &secp_ctx).is_err());
+
+               // Fails verification with altered signing pubkey
+               let mut tlv_stream = offer.as_tlv_stream();
+               let signing_pubkey = pubkey(1);
+               tlv_stream.node_id = Some(&signing_pubkey);
+
+               let mut encoded_offer = Vec::new();
+               tlv_stream.write(&mut encoded_offer).unwrap();
+
+               let invoice_request = Offer::try_from(encoded_offer).unwrap()
+                       .request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert!(invoice_request.verify(&expanded_key, &secp_ctx).is_err());
+       }
+
        #[test]
        fn builds_offer_with_amount() {
                let bitcoin_amount = Amount::Bitcoin { amount_msats: 1000 };
index 6afd4d68fef385bb5b6b77b8375891e53878f6e7..3f1a9c887ec002315d56d766910b026e6d7efedc 100644 (file)
@@ -171,6 +171,8 @@ pub enum SemanticError {
        InvalidQuantity,
        /// A quantity or quantity bounds was provided but was not expected.
        UnexpectedQuantity,
+       /// Metadata could not be used to verify the offers message.
+       InvalidMetadata,
        /// Metadata was provided but was not expected.
        UnexpectedMetadata,
        /// Payer metadata was expected but was missing.
index 7e1da769edab6cec527c4de425bb7711fc50fa1f..bfc02b5dbcb9804f1fd08bdf689f541a12579ed9 100644 (file)
@@ -9,6 +9,7 @@
 
 //! Data structures and encoding for `invoice_request_metadata` records.
 
+use crate::offers::signer::Metadata;
 use crate::util::ser::WithoutLength;
 
 use crate::prelude::*;
@@ -17,9 +18,16 @@ use crate::prelude::*;
 /// [`InvoiceRequest::payer_id`].
 ///
 /// [`InvoiceRequest::payer_id`]: crate::offers::invoice_request::InvoiceRequest::payer_id
-#[derive(Clone, Debug, PartialEq)]
-pub(super) struct PayerContents(pub Vec<u8>);
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub(super) struct PayerContents(pub Metadata);
+
+/// TLV record type for [`InvoiceRequest::metadata`] and [`Refund::metadata`].
+///
+/// [`InvoiceRequest::metadata`]: crate::offers::invoice_request::InvoiceRequest::metadata
+/// [`Refund::metadata`]: crate::offers::refund::Refund::metadata
+pub(super) const PAYER_METADATA_TYPE: u64 = 0;
 
 tlv_stream!(PayerTlvStream, PayerTlvStreamRef, 0..1, {
-       (0, metadata: (Vec<u8>, WithoutLength)),
+       (PAYER_METADATA_TYPE, metadata: (Vec<u8>, WithoutLength)),
 });
index cc0388c0241b5f77a125ab0dabe4da8427504e8f..eac7a3754edf54444d3b87b6c3b31fd5ccf4f3dd 100644 (file)
 
 use bitcoin::blockdata::constants::ChainHash;
 use bitcoin::network::constants::Network;
-use bitcoin::secp256k1::PublicKey;
+use bitcoin::secp256k1::{PublicKey, Secp256k1, self};
 use core::convert::TryFrom;
+use core::ops::Deref;
 use core::str::FromStr;
 use core::time::Duration;
+use crate::chain::keysinterface::EntropySource;
 use crate::io;
 use crate::ln::PaymentHash;
 use crate::ln::features::InvoiceRequestFeatures;
+use crate::ln::inbound_payment::{ExpandedKey, IV_LEN, Nonce};
 use crate::ln::msgs::{DecodeError, MAX_VALUE_MSAT};
-use crate::offers::invoice::{BlindedPayInfo, InvoiceBuilder};
+use crate::offers::invoice::{BlindedPayInfo, DerivedSigningPubkey, ExplicitSigningPubkey, InvoiceBuilder};
 use crate::offers::invoice_request::{InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef};
 use crate::offers::offer::{OfferTlvStream, OfferTlvStreamRef};
 use crate::offers::parse::{Bech32Encode, ParseError, ParsedMessage, SemanticError};
 use crate::offers::payer::{PayerContents, PayerTlvStream, PayerTlvStreamRef};
+use crate::offers::signer::{Metadata, MetadataMaterial, self};
 use crate::onion_message::BlindedPath;
 use crate::util::ser::{SeekReadable, WithoutLength, Writeable, Writer};
 use crate::util::string::PrintableString;
@@ -95,16 +99,19 @@ use crate::prelude::*;
 #[cfg(feature = "std")]
 use std::time::SystemTime;
 
+pub(super) const IV_BYTES: &[u8; IV_LEN] = b"LDK Refund ~~~~~";
+
 /// Builds a [`Refund`] for the "offer for money" flow.
 ///
 /// See [module-level documentation] for usage.
 ///
 /// [module-level documentation]: self
-pub struct RefundBuilder {
+pub struct RefundBuilder<'a, T: secp256k1::Signing> {
        refund: RefundContents,
+       secp_ctx: Option<&'a Secp256k1<T>>,
 }
 
-impl RefundBuilder {
+impl<'a> RefundBuilder<'a, secp256k1::SignOnly> {
        /// Creates a new builder for a refund using the [`Refund::payer_id`] for the public node id to
        /// send to if no [`Refund::paths`] are set. Otherwise, it may be a transient pubkey.
        ///
@@ -117,13 +124,48 @@ impl RefundBuilder {
                        return Err(SemanticError::InvalidAmount);
                }
 
-               let refund = RefundContents {
-                       payer: PayerContents(metadata), description, absolute_expiry: None, issuer: None,
-                       paths: None, chain: None, amount_msats, features: InvoiceRequestFeatures::empty(),
-                       quantity: None, payer_id, payer_note: None,
-               };
+               let metadata = Metadata::Bytes(metadata);
+               Ok(Self {
+                       refund: RefundContents {
+                               payer: PayerContents(metadata), description, absolute_expiry: None, issuer: None,
+                               paths: None, chain: None, amount_msats, features: InvoiceRequestFeatures::empty(),
+                               quantity: None, payer_id, payer_note: None,
+                       },
+                       secp_ctx: None,
+               })
+       }
+}
 
-               Ok(RefundBuilder { refund })
+impl<'a, T: secp256k1::Signing> RefundBuilder<'a, T> {
+       /// Similar to [`RefundBuilder::new`] except, if [`RefundBuilder::path`] is called, the payer id
+       /// is derived from the given [`ExpandedKey`] and nonce. This provides sender privacy by using a
+       /// different payer id for each refund, assuming a different nonce is used.  Otherwise, the
+       /// provided `node_id` is used for the payer id.
+       ///
+       /// Also, sets the metadata when [`RefundBuilder::build`] is called such that it can be used to
+       /// verify that an [`InvoiceRequest`] was produced for the refund given an [`ExpandedKey`].
+       ///
+       /// [`InvoiceRequest`]: crate::offers::invoice_request::InvoiceRequest
+       /// [`ExpandedKey`]: crate::ln::inbound_payment::ExpandedKey
+       pub fn deriving_payer_id<ES: Deref>(
+               description: String, node_id: PublicKey, expanded_key: &ExpandedKey, entropy_source: ES,
+               secp_ctx: &'a Secp256k1<T>, amount_msats: u64
+       ) -> Result<Self, SemanticError> where ES::Target: EntropySource {
+               if amount_msats > MAX_VALUE_MSAT {
+                       return Err(SemanticError::InvalidAmount);
+               }
+
+               let nonce = Nonce::from_entropy_source(entropy_source);
+               let derivation_material = MetadataMaterial::new(nonce, expanded_key, IV_BYTES);
+               let metadata = Metadata::DerivedSigningPubkey(derivation_material);
+               Ok(Self {
+                       refund: RefundContents {
+                               payer: PayerContents(metadata), description, absolute_expiry: None, issuer: None,
+                               paths: None, chain: None, amount_msats, features: InvoiceRequestFeatures::empty(),
+                               quantity: None, payer_id: node_id, payer_note: None,
+                       },
+                       secp_ctx: Some(secp_ctx),
+               })
        }
 
        /// Sets the [`Refund::absolute_expiry`] as seconds since the Unix epoch. Any expiry that has
@@ -190,18 +232,38 @@ impl RefundBuilder {
                        self.refund.chain = None;
                }
 
+               // Create the metadata for stateless verification of an Invoice.
+               if self.refund.payer.0.has_derivation_material() {
+                       let mut metadata = core::mem::take(&mut self.refund.payer.0);
+
+                       if self.refund.paths.is_none() {
+                               metadata = metadata.without_keys();
+                       }
+
+                       let mut tlv_stream = self.refund.as_tlv_stream();
+                       tlv_stream.0.metadata = None;
+                       if metadata.derives_keys() {
+                               tlv_stream.2.payer_id = None;
+                       }
+
+                       let (derived_metadata, keys) = metadata.derive_from(tlv_stream, self.secp_ctx);
+                       metadata = derived_metadata;
+                       if let Some(keys) = keys {
+                               self.refund.payer_id = keys.public_key();
+                       }
+
+                       self.refund.payer.0 = metadata;
+               }
+
                let mut bytes = Vec::new();
                self.refund.write(&mut bytes).unwrap();
 
-               Ok(Refund {
-                       bytes,
-                       contents: self.refund,
-               })
+               Ok(Refund { bytes, contents: self.refund })
        }
 }
 
 #[cfg(test)]
-impl RefundBuilder {
+impl<'a, T: secp256k1::Signing> RefundBuilder<'a, T> {
        fn features_unchecked(mut self, features: InvoiceRequestFeatures) -> Self {
                self.refund.features = features;
                self
@@ -216,7 +278,8 @@ impl RefundBuilder {
 ///
 /// [`Invoice`]: crate::offers::invoice::Invoice
 /// [`Offer`]: crate::offers::offer::Offer
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub struct Refund {
        pub(super) bytes: Vec<u8>,
        pub(super) contents: RefundContents,
@@ -225,7 +288,8 @@ pub struct Refund {
 /// The contents of a [`Refund`], which may be shared with an [`Invoice`].
 ///
 /// [`Invoice`]: crate::offers::invoice::Invoice
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub(super) struct RefundContents {
        payer: PayerContents,
        // offer fields
@@ -246,7 +310,7 @@ impl Refund {
        /// A complete description of the purpose of the refund. Intended to be displayed to the user
        /// but with the caveat that it has not been verified in any way.
        pub fn description(&self) -> PrintableString {
-               PrintableString(&self.contents.description)
+               self.contents.description()
        }
 
        /// Duration since the Unix epoch when an invoice should no longer be sent.
@@ -279,7 +343,7 @@ impl Refund {
        ///
        /// [`payer_id`]: Self::payer_id
        pub fn metadata(&self) -> &[u8] {
-               &self.contents.payer.0
+               self.contents.metadata()
        }
 
        /// A chain that the refund is valid for.
@@ -317,19 +381,18 @@ impl Refund {
                self.contents.payer_note.as_ref().map(|payer_note| PrintableString(payer_note.as_str()))
        }
 
-       /// Creates an [`Invoice`] for the refund with the given required fields and using the
+       /// Creates an [`InvoiceBuilder`] for the refund with the given required fields and using the
        /// [`Duration`] since [`std::time::SystemTime::UNIX_EPOCH`] as the creation time.
        ///
        /// See [`Refund::respond_with_no_std`] for further details where the aforementioned creation
        /// time is used for the `created_at` parameter.
        ///
-       /// [`Invoice`]: crate::offers::invoice::Invoice
        /// [`Duration`]: core::time::Duration
        #[cfg(feature = "std")]
        pub fn respond_with(
                &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
                signing_pubkey: PublicKey,
-       ) -> Result<InvoiceBuilder, SemanticError> {
+       ) -> Result<InvoiceBuilder<ExplicitSigningPubkey>, SemanticError> {
                let created_at = std::time::SystemTime::now()
                        .duration_since(std::time::SystemTime::UNIX_EPOCH)
                        .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH");
@@ -337,7 +400,7 @@ impl Refund {
                self.respond_with_no_std(payment_paths, payment_hash, signing_pubkey, created_at)
        }
 
-       /// Creates an [`Invoice`] for the refund with the given required fields.
+       /// Creates an [`InvoiceBuilder`] for the refund with the given required fields.
        ///
        /// Unless [`InvoiceBuilder::relative_expiry`] is set, the invoice will expire two hours after
        /// `created_at`, which is used to set [`Invoice::created_at`]. Useful for `no-std` builds where
@@ -356,12 +419,11 @@ impl Refund {
        ///
        /// Errors if the request contains unknown required features.
        ///
-       /// [`Invoice`]: crate::offers::invoice::Invoice
        /// [`Invoice::created_at`]: crate::offers::invoice::Invoice::created_at
        pub fn respond_with_no_std(
                &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
                signing_pubkey: PublicKey, created_at: Duration
-       ) -> Result<InvoiceBuilder, SemanticError> {
+       ) -> Result<InvoiceBuilder<ExplicitSigningPubkey>, SemanticError> {
                if self.features().requires_unknown_bits() {
                        return Err(SemanticError::UnknownRequiredFeatures);
                }
@@ -369,6 +431,51 @@ impl Refund {
                InvoiceBuilder::for_refund(self, payment_paths, created_at, payment_hash, signing_pubkey)
        }
 
+       /// Creates an [`InvoiceBuilder`] for the refund using the given required fields and that uses
+       /// derived signing keys to sign the [`Invoice`].
+       ///
+       /// See [`Refund::respond_with`] for further details.
+       ///
+       /// [`Invoice`]: crate::offers::invoice::Invoice
+       #[cfg(feature = "std")]
+       pub fn respond_using_derived_keys<ES: Deref>(
+               &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
+               expanded_key: &ExpandedKey, entropy_source: ES
+       ) -> Result<InvoiceBuilder<DerivedSigningPubkey>, SemanticError>
+       where
+               ES::Target: EntropySource,
+       {
+               let created_at = std::time::SystemTime::now()
+                       .duration_since(std::time::SystemTime::UNIX_EPOCH)
+                       .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH");
+
+               self.respond_using_derived_keys_no_std(
+                       payment_paths, payment_hash, created_at, expanded_key, entropy_source
+               )
+       }
+
+       /// Creates an [`InvoiceBuilder`] for the refund using the given required fields and that uses
+       /// derived signing keys to sign the [`Invoice`].
+       ///
+       /// See [`Refund::respond_with_no_std`] for further details.
+       ///
+       /// [`Invoice`]: crate::offers::invoice::Invoice
+       pub fn respond_using_derived_keys_no_std<ES: Deref>(
+               &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
+               created_at: core::time::Duration, expanded_key: &ExpandedKey, entropy_source: ES
+       ) -> Result<InvoiceBuilder<DerivedSigningPubkey>, SemanticError>
+       where
+               ES::Target: EntropySource,
+       {
+               if self.features().requires_unknown_bits() {
+                       return Err(SemanticError::UnknownRequiredFeatures);
+               }
+
+               let nonce = Nonce::from_entropy_source(entropy_source);
+               let keys = signer::derive_keys(nonce, expanded_key);
+               InvoiceBuilder::for_refund_using_keys(self, payment_paths, created_at, payment_hash, keys)
+       }
+
        #[cfg(test)]
        fn as_tlv_stream(&self) -> RefundTlvStreamRef {
                self.contents.as_tlv_stream()
@@ -382,6 +489,10 @@ impl AsRef<[u8]> for Refund {
 }
 
 impl RefundContents {
+       pub fn description(&self) -> PrintableString {
+               PrintableString(&self.description)
+       }
+
        #[cfg(feature = "std")]
        pub(super) fn is_expired(&self) -> bool {
                match self.absolute_expiry {
@@ -393,6 +504,10 @@ impl RefundContents {
                }
        }
 
+       pub(super) fn metadata(&self) -> &[u8] {
+               self.payer.0.as_bytes().map(|bytes| bytes.as_slice()).unwrap_or(&[])
+       }
+
        pub(super) fn chain(&self) -> ChainHash {
                self.chain.unwrap_or_else(|| self.implied_chain())
        }
@@ -401,9 +516,17 @@ impl RefundContents {
                ChainHash::using_genesis_block(Network::Bitcoin)
        }
 
+       pub(super) fn derives_keys(&self) -> bool {
+               self.payer.0.derives_keys()
+       }
+
+       pub(super) fn payer_id(&self) -> PublicKey {
+               self.payer_id
+       }
+
        pub(super) fn as_tlv_stream(&self) -> RefundTlvStreamRef {
                let payer = PayerTlvStreamRef {
-                       metadata: Some(&self.payer.0),
+                       metadata: self.payer.0.as_bytes(),
                };
 
                let offer = OfferTlvStreamRef {
@@ -507,7 +630,7 @@ impl TryFrom<RefundTlvStream> for RefundContents {
 
                let payer = match payer_metadata {
                        None => return Err(SemanticError::MissingPayerMetadata),
-                       Some(metadata) => PayerContents(metadata),
+                       Some(metadata) => PayerContents(Metadata::Bytes(metadata)),
                };
 
                if metadata.is_some() {
@@ -575,33 +698,22 @@ mod tests {
 
        use bitcoin::blockdata::constants::ChainHash;
        use bitcoin::network::constants::Network;
-       use bitcoin::secp256k1::{KeyPair, PublicKey, Secp256k1, SecretKey};
+       use bitcoin::secp256k1::{KeyPair, Secp256k1, SecretKey};
        use core::convert::TryFrom;
        use core::time::Duration;
+       use crate::chain::keysinterface::KeyMaterial;
        use crate::ln::features::{InvoiceRequestFeatures, OfferFeatures};
+       use crate::ln::inbound_payment::ExpandedKey;
        use crate::ln::msgs::{DecodeError, MAX_VALUE_MSAT};
        use crate::offers::invoice_request::InvoiceRequestTlvStreamRef;
        use crate::offers::offer::OfferTlvStreamRef;
        use crate::offers::parse::{ParseError, SemanticError};
        use crate::offers::payer::PayerTlvStreamRef;
+       use crate::offers::test_utils::*;
        use crate::onion_message::{BlindedHop, BlindedPath};
        use crate::util::ser::{BigSize, Writeable};
        use crate::util::string::PrintableString;
 
-       fn payer_pubkey() -> PublicKey {
-               let secp_ctx = Secp256k1::new();
-               KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap()).public_key()
-       }
-
-       fn pubkey(byte: u8) -> PublicKey {
-               let secp_ctx = Secp256k1::new();
-               PublicKey::from_secret_key(&secp_ctx, &privkey(byte))
-       }
-
-       fn privkey(byte: u8) -> SecretKey {
-               SecretKey::from_slice(&[byte; 32]).unwrap()
-       }
-
        trait ToBytes {
                fn to_bytes(&self) -> Vec<u8>;
        }
@@ -677,6 +789,118 @@ mod tests {
                }
        }
 
+       #[test]
+       fn builds_refund_with_metadata_derived() {
+               let desc = "foo".to_string();
+               let node_id = payer_pubkey();
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let refund = RefundBuilder
+                       ::deriving_payer_id(desc, node_id, &expanded_key, &entropy, &secp_ctx, 1000)
+                       .unwrap()
+                       .build().unwrap();
+               assert_eq!(refund.payer_id(), node_id);
+
+               // Fails verification with altered fields
+               let invoice = refund
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(invoice.verify(&expanded_key, &secp_ctx));
+
+               let mut tlv_stream = refund.as_tlv_stream();
+               tlv_stream.2.amount = Some(2000);
+
+               let mut encoded_refund = Vec::new();
+               tlv_stream.write(&mut encoded_refund).unwrap();
+
+               let invoice = Refund::try_from(encoded_refund).unwrap()
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered metadata
+               let mut tlv_stream = refund.as_tlv_stream();
+               let metadata = tlv_stream.0.metadata.unwrap().iter().copied().rev().collect();
+               tlv_stream.0.metadata = Some(&metadata);
+
+               let mut encoded_refund = Vec::new();
+               tlv_stream.write(&mut encoded_refund).unwrap();
+
+               let invoice = Refund::try_from(encoded_refund).unwrap()
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+       }
+
+       #[test]
+       fn builds_refund_with_derived_payer_id() {
+               let desc = "foo".to_string();
+               let node_id = payer_pubkey();
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let blinded_path = BlindedPath {
+                       introduction_node_id: pubkey(40),
+                       blinding_point: pubkey(41),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: pubkey(43), encrypted_payload: vec![0; 43] },
+                               BlindedHop { blinded_node_id: node_id, encrypted_payload: vec![0; 44] },
+                       ],
+               };
+
+               let refund = RefundBuilder
+                       ::deriving_payer_id(desc, node_id, &expanded_key, &entropy, &secp_ctx, 1000)
+                       .unwrap()
+                       .path(blinded_path)
+                       .build().unwrap();
+               assert_ne!(refund.payer_id(), node_id);
+
+               let invoice = refund
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered fields
+               let mut tlv_stream = refund.as_tlv_stream();
+               tlv_stream.2.amount = Some(2000);
+
+               let mut encoded_refund = Vec::new();
+               tlv_stream.write(&mut encoded_refund).unwrap();
+
+               let invoice = Refund::try_from(encoded_refund).unwrap()
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered payer_id
+               let mut tlv_stream = refund.as_tlv_stream();
+               let payer_id = pubkey(1);
+               tlv_stream.2.payer_id = Some(&payer_id);
+
+               let mut encoded_refund = Vec::new();
+               tlv_stream.write(&mut encoded_refund).unwrap();
+
+               let invoice = Refund::try_from(encoded_refund).unwrap()
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+       }
+
        #[test]
        fn builds_refund_with_absolute_expiry() {
                let future_expiry = Duration::from_secs(u64::max_value());
@@ -822,6 +1046,18 @@ mod tests {
                assert_eq!(tlv_stream.payer_note, Some(&String::from("baz")));
        }
 
+       #[test]
+       fn fails_responding_with_unknown_required_features() {
+               match RefundBuilder::new("foo".into(), vec![1; 32], payer_pubkey(), 1000).unwrap()
+                       .features_unchecked(InvoiceRequestFeatures::unknown())
+                       .build().unwrap()
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+               {
+                       Ok(_) => panic!("expected error"),
+                       Err(e) => assert_eq!(e, SemanticError::UnknownRequiredFeatures),
+               }
+       }
+
        #[test]
        fn parses_refund_with_metadata() {
                let refund = RefundBuilder::new("foo".into(), vec![1; 32], payer_pubkey(), 1000).unwrap()
diff --git a/lightning/src/offers/signer.rs b/lightning/src/offers/signer.rs
new file mode 100644 (file)
index 0000000..8d5f98e
--- /dev/null
@@ -0,0 +1,231 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Utilities for signing offer messages and verifying metadata.
+
+use bitcoin::hashes::{Hash, HashEngine};
+use bitcoin::hashes::cmp::fixed_time_eq;
+use bitcoin::hashes::hmac::{Hmac, HmacEngine};
+use bitcoin::hashes::sha256::Hash as Sha256;
+use bitcoin::secp256k1::{KeyPair, PublicKey, Secp256k1, SecretKey, self};
+use core::convert::TryFrom;
+use core::fmt;
+use crate::ln::inbound_payment::{ExpandedKey, IV_LEN, Nonce};
+use crate::offers::merkle::TlvRecord;
+use crate::util::ser::Writeable;
+
+use crate::prelude::*;
+
+const DERIVED_METADATA_HMAC_INPUT: &[u8; 16] = &[1; 16];
+const DERIVED_METADATA_AND_KEYS_HMAC_INPUT: &[u8; 16] = &[2; 16];
+
+/// Message metadata which possibly is derived from [`MetadataMaterial`] such that it can be
+/// verified.
+#[derive(Clone)]
+pub(super) enum Metadata {
+       /// Metadata as parsed, supplied by the user, or derived from the message contents.
+       Bytes(Vec<u8>),
+
+       /// Metadata to be derived from message contents and given material.
+       Derived(MetadataMaterial),
+
+       /// Metadata and signing pubkey to be derived from message contents and given material.
+       DerivedSigningPubkey(MetadataMaterial),
+}
+
+impl Metadata {
+       pub fn as_bytes(&self) -> Option<&Vec<u8>> {
+               match self {
+                       Metadata::Bytes(bytes) => Some(bytes),
+                       Metadata::Derived(_) => None,
+                       Metadata::DerivedSigningPubkey(_) => None,
+               }
+       }
+
+       pub fn has_derivation_material(&self) -> bool {
+               match self {
+                       Metadata::Bytes(_) => false,
+                       Metadata::Derived(_) => true,
+                       Metadata::DerivedSigningPubkey(_) => true,
+               }
+       }
+
+       pub fn derives_keys(&self) -> bool {
+               match self {
+                       // Infer whether Metadata::derived_from was called on Metadata::DerivedSigningPubkey to
+                       // produce Metadata::Bytes. This is merely to determine which fields should be included
+                       // when verifying a message. It doesn't necessarily indicate that keys were in fact
+                       // derived, as wouldn't be the case if a Metadata::Bytes with length Nonce::LENGTH had
+                       // been set explicitly.
+                       Metadata::Bytes(bytes) => bytes.len() == Nonce::LENGTH,
+                       Metadata::Derived(_) => false,
+                       Metadata::DerivedSigningPubkey(_) => true,
+               }
+       }
+
+       pub fn without_keys(self) -> Self {
+               match self {
+                       Metadata::Bytes(_) => self,
+                       Metadata::Derived(_) => self,
+                       Metadata::DerivedSigningPubkey(material) => Metadata::Derived(material),
+               }
+       }
+
+       pub fn derive_from<W: Writeable, T: secp256k1::Signing>(
+               self, tlv_stream: W, secp_ctx: Option<&Secp256k1<T>>
+       ) -> (Self, Option<KeyPair>) {
+               match self {
+                       Metadata::Bytes(_) => (self, None),
+                       Metadata::Derived(mut metadata_material) => {
+                               tlv_stream.write(&mut metadata_material.hmac).unwrap();
+                               (Metadata::Bytes(metadata_material.derive_metadata()), None)
+                       },
+                       Metadata::DerivedSigningPubkey(mut metadata_material) => {
+                               tlv_stream.write(&mut metadata_material.hmac).unwrap();
+                               let secp_ctx = secp_ctx.unwrap();
+                               let (metadata, keys) = metadata_material.derive_metadata_and_keys(secp_ctx);
+                               (Metadata::Bytes(metadata), Some(keys))
+                       },
+               }
+       }
+}
+
+impl Default for Metadata {
+       fn default() -> Self {
+               Metadata::Bytes(vec![])
+       }
+}
+
+impl fmt::Debug for Metadata {
+       fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+               match self {
+                       Metadata::Bytes(bytes) => bytes.fmt(f),
+                       Metadata::Derived(_) => f.write_str("Derived"),
+                       Metadata::DerivedSigningPubkey(_) => f.write_str("DerivedSigningPubkey"),
+               }
+       }
+}
+
+#[cfg(test)]
+impl PartialEq for Metadata {
+       fn eq(&self, other: &Self) -> bool {
+               match self {
+                       Metadata::Bytes(bytes) => if let Metadata::Bytes(other_bytes) = other {
+                               bytes == other_bytes
+                       } else {
+                               false
+                       },
+                       Metadata::Derived(_) => false,
+                       Metadata::DerivedSigningPubkey(_) => false,
+               }
+       }
+}
+
+/// Material used to create metadata for a message.
+#[derive(Clone)]
+pub(super) struct MetadataMaterial {
+       nonce: Nonce,
+       hmac: HmacEngine<Sha256>,
+}
+
+impl MetadataMaterial {
+       pub fn new(nonce: Nonce, expanded_key: &ExpandedKey, iv_bytes: &[u8; IV_LEN]) -> Self {
+               Self {
+                       nonce,
+                       hmac: expanded_key.hmac_for_offer(nonce, iv_bytes),
+               }
+       }
+
+       fn derive_metadata(mut self) -> Vec<u8> {
+               self.hmac.input(DERIVED_METADATA_HMAC_INPUT);
+
+               let mut bytes = self.nonce.as_slice().to_vec();
+               bytes.extend_from_slice(&Hmac::from_engine(self.hmac).into_inner());
+               bytes
+       }
+
+       fn derive_metadata_and_keys<T: secp256k1::Signing>(
+               mut self, secp_ctx: &Secp256k1<T>
+       ) -> (Vec<u8>, KeyPair) {
+               self.hmac.input(DERIVED_METADATA_AND_KEYS_HMAC_INPUT);
+
+               let hmac = Hmac::from_engine(self.hmac);
+               let privkey = SecretKey::from_slice(hmac.as_inner()).unwrap();
+               let keys = KeyPair::from_secret_key(secp_ctx, &privkey);
+               (self.nonce.as_slice().to_vec(), keys)
+       }
+}
+
+pub(super) fn derive_keys(nonce: Nonce, expanded_key: &ExpandedKey) -> KeyPair {
+       const IV_BYTES: &[u8; IV_LEN] = b"LDK Invoice ~~~~";
+       let secp_ctx = Secp256k1::new();
+       let hmac = Hmac::from_engine(expanded_key.hmac_for_offer(nonce, IV_BYTES));
+       let privkey = SecretKey::from_slice(hmac.as_inner()).unwrap();
+       KeyPair::from_secret_key(&secp_ctx, &privkey)
+}
+
+/// Verifies data given in a TLV stream was used to produce the given metadata, consisting of:
+/// - a 128-bit [`Nonce`] and possibly
+/// - a [`Sha256`] hash of the nonce and the TLV records using the [`ExpandedKey`].
+///
+/// If the latter is not included in the metadata, the TLV stream is used to check if the given
+/// `signing_pubkey` can be derived from it.
+pub(super) fn verify_metadata<'a, T: secp256k1::Signing>(
+       metadata: &[u8], expanded_key: &ExpandedKey, iv_bytes: &[u8; IV_LEN],
+       signing_pubkey: PublicKey, tlv_stream: impl core::iter::Iterator<Item = TlvRecord<'a>>,
+       secp_ctx: &Secp256k1<T>
+) -> Result<Option<KeyPair>, ()> {
+       let hmac = hmac_for_message(metadata, expanded_key, iv_bytes, tlv_stream)?;
+
+       if metadata.len() == Nonce::LENGTH {
+               let derived_keys = KeyPair::from_secret_key(
+                       secp_ctx, &SecretKey::from_slice(hmac.as_inner()).unwrap()
+               );
+               if fixed_time_eq(&signing_pubkey.serialize(), &derived_keys.public_key().serialize()) {
+                       Ok(Some(derived_keys))
+               } else {
+                       Err(())
+               }
+       } else if metadata[Nonce::LENGTH..].len() == Sha256::LEN {
+               if fixed_time_eq(&metadata[Nonce::LENGTH..], &hmac.into_inner()) {
+                       Ok(None)
+               } else {
+                       Err(())
+               }
+       } else {
+               Err(())
+       }
+}
+
+fn hmac_for_message<'a>(
+       metadata: &[u8], expanded_key: &ExpandedKey, iv_bytes: &[u8; IV_LEN],
+       tlv_stream: impl core::iter::Iterator<Item = TlvRecord<'a>>
+) -> Result<Hmac<Sha256>, ()> {
+       if metadata.len() < Nonce::LENGTH {
+               return Err(());
+       }
+
+       let nonce = match Nonce::try_from(&metadata[..Nonce::LENGTH]) {
+               Ok(nonce) => nonce,
+               Err(_) => return Err(()),
+       };
+       let mut hmac = expanded_key.hmac_for_offer(nonce, iv_bytes);
+
+       for record in tlv_stream {
+               hmac.input(record.record_bytes);
+       }
+
+       if metadata.len() == Nonce::LENGTH {
+               hmac.input(DERIVED_METADATA_AND_KEYS_HMAC_INPUT);
+       } else {
+               hmac.input(DERIVED_METADATA_HMAC_INPUT);
+       }
+
+       Ok(Hmac::from_engine(hmac))
+}
diff --git a/lightning/src/offers/test_utils.rs b/lightning/src/offers/test_utils.rs
new file mode 100644 (file)
index 0000000..4366407
--- /dev/null
@@ -0,0 +1,119 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Utilities for testing BOLT 12 Offers interfaces
+
+use bitcoin::secp256k1::{KeyPair, Message, PublicKey, Secp256k1, SecretKey};
+use bitcoin::secp256k1::schnorr::Signature;
+use core::convert::Infallible;
+use core::time::Duration;
+use crate::chain::keysinterface::EntropySource;
+use crate::ln::PaymentHash;
+use crate::ln::features::BlindedHopFeatures;
+use crate::offers::invoice::BlindedPayInfo;
+use crate::onion_message::{BlindedHop, BlindedPath};
+
+pub(super) fn payer_keys() -> KeyPair {
+       let secp_ctx = Secp256k1::new();
+       KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap())
+}
+
+pub(super) fn payer_sign(digest: &Message) -> Result<Signature, Infallible> {
+       let secp_ctx = Secp256k1::new();
+       let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap());
+       Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys))
+}
+
+pub(super) fn payer_pubkey() -> PublicKey {
+       payer_keys().public_key()
+}
+
+pub(super) fn recipient_keys() -> KeyPair {
+       let secp_ctx = Secp256k1::new();
+       KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap())
+}
+
+pub(super) fn recipient_sign(digest: &Message) -> Result<Signature, Infallible> {
+       let secp_ctx = Secp256k1::new();
+       let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap());
+       Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys))
+}
+
+pub(super) fn recipient_pubkey() -> PublicKey {
+       recipient_keys().public_key()
+}
+
+pub(super) fn pubkey(byte: u8) -> PublicKey {
+       let secp_ctx = Secp256k1::new();
+       PublicKey::from_secret_key(&secp_ctx, &privkey(byte))
+}
+
+pub(super) fn privkey(byte: u8) -> SecretKey {
+       SecretKey::from_slice(&[byte; 32]).unwrap()
+}
+
+pub(super) fn payment_paths() -> Vec<(BlindedPath, BlindedPayInfo)> {
+       let paths = vec![
+               BlindedPath {
+                       introduction_node_id: pubkey(40),
+                       blinding_point: pubkey(41),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: pubkey(43), encrypted_payload: vec![0; 43] },
+                               BlindedHop { blinded_node_id: pubkey(44), encrypted_payload: vec![0; 44] },
+                       ],
+               },
+               BlindedPath {
+                       introduction_node_id: pubkey(40),
+                       blinding_point: pubkey(41),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: pubkey(45), encrypted_payload: vec![0; 45] },
+                               BlindedHop { blinded_node_id: pubkey(46), encrypted_payload: vec![0; 46] },
+                       ],
+               },
+       ];
+
+       let payinfo = vec![
+               BlindedPayInfo {
+                       fee_base_msat: 1,
+                       fee_proportional_millionths: 1_000,
+                       cltv_expiry_delta: 42,
+                       htlc_minimum_msat: 100,
+                       htlc_maximum_msat: 1_000_000_000_000,
+                       features: BlindedHopFeatures::empty(),
+               },
+               BlindedPayInfo {
+                       fee_base_msat: 1,
+                       fee_proportional_millionths: 1_000,
+                       cltv_expiry_delta: 42,
+                       htlc_minimum_msat: 100,
+                       htlc_maximum_msat: 1_000_000_000_000,
+                       features: BlindedHopFeatures::empty(),
+               },
+       ];
+
+       paths.into_iter().zip(payinfo.into_iter()).collect()
+}
+
+pub(super) fn payment_hash() -> PaymentHash {
+       PaymentHash([42; 32])
+}
+
+pub(super) fn now() -> Duration {
+       std::time::SystemTime::now()
+               .duration_since(std::time::SystemTime::UNIX_EPOCH)
+               .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH")
+}
+
+pub(super) struct FixedEntropy;
+
+impl EntropySource for FixedEntropy {
+       fn get_secure_random_bytes(&self) -> [u8; 32] {
+               [42; 32]
+       }
+}
index 2f2d33b29f7ea53c0a2dda084ac7a139583c6345..7352542605070c7812201d0a10f35f65eaa3b3e4 100644 (file)
@@ -3,6 +3,10 @@ use bitcoin::hashes::hmac::{Hmac, HmacEngine};
 use bitcoin::hashes::sha256::Hash as Sha256;
 use bitcoin::secp256k1::{Message, Secp256k1, SecretKey, ecdsa::Signature, Signing};
 
+use crate::chain::keysinterface::EntropySource;
+
+use core::ops::Deref;
+
 macro_rules! hkdf_extract_expand {
        ($salt: expr, $ikm: expr) => {{
                let mut hmac = HmacEngine::<Sha256>::new($salt);
@@ -20,13 +24,18 @@ macro_rules! hkdf_extract_expand {
                let (k1, k2, _) = hkdf_extract_expand!($salt, $ikm);
                (k1, k2)
        }};
-       ($salt: expr, $ikm: expr, 3) => {{
+       ($salt: expr, $ikm: expr, 4) => {{
                let (k1, k2, prk) = hkdf_extract_expand!($salt, $ikm);
 
                let mut hmac = HmacEngine::<Sha256>::new(&prk[..]);
                hmac.input(&k2);
                hmac.input(&[3; 1]);
-               (k1, k2, Hmac::from_engine(hmac).into_inner())
+               let k3 = Hmac::from_engine(hmac).into_inner();
+
+               let mut hmac = HmacEngine::<Sha256>::new(&prk[..]);
+               hmac.input(&k3);
+               hmac.input(&[4; 1]);
+               (k1, k2, k3, Hmac::from_engine(hmac).into_inner())
        }}
 }
 
@@ -34,8 +43,8 @@ pub fn hkdf_extract_expand_twice(salt: &[u8], ikm: &[u8]) -> ([u8; 32], [u8; 32]
        hkdf_extract_expand!(salt, ikm, 2)
 }
 
-pub fn hkdf_extract_expand_thrice(salt: &[u8], ikm: &[u8]) -> ([u8; 32], [u8; 32], [u8; 32]) {
-       hkdf_extract_expand!(salt, ikm, 3)
+pub fn hkdf_extract_expand_4x(salt: &[u8], ikm: &[u8]) -> ([u8; 32], [u8; 32], [u8; 32], [u8; 32]) {
+       hkdf_extract_expand!(salt, ikm, 4)
 }
 
 #[inline]
@@ -46,3 +55,21 @@ pub fn sign<C: Signing>(ctx: &Secp256k1<C>, msg: &Message, sk: &SecretKey) -> Si
        let sig = ctx.sign_ecdsa(msg, sk);
        sig
 }
+
+#[inline]
+pub fn sign_with_aux_rand<C: Signing, ES: Deref>(
+       ctx: &Secp256k1<C>, msg: &Message, sk: &SecretKey, entropy_source: &ES
+) -> Signature where ES::Target: EntropySource {
+       #[cfg(feature = "grind_signatures")]
+       let sig = loop {
+               let sig = ctx.sign_ecdsa_with_noncedata(msg, sk, &entropy_source.get_secure_random_bytes());
+               if sig.serialize_compact()[0] < 0x80 {
+                       break sig;
+               }
+       };
+       #[cfg(all(not(feature = "grind_signatures"), not(feature = "_test_vectors")))]
+       let sig = ctx.sign_ecdsa_with_noncedata(msg, sk, &entropy_source.get_secure_random_bytes());
+       #[cfg(all(not(feature = "grind_signatures"), feature = "_test_vectors"))]
+       let sig = sign(ctx, msg, sk);
+       sig
+}
index a34cb0cf323a337edc757579c761d1363948212b..679652b2da49402a2c76f2f0720407c0928e043b 100644 (file)
@@ -161,7 +161,7 @@ impl SignerProvider for OnlyReadsKeysInterface {
        fn derive_channel_signer(&self, _channel_value_satoshis: u64, _channel_keys_id: [u8; 32]) -> Self::Signer { unreachable!(); }
 
        fn read_chan_signer(&self, mut reader: &[u8]) -> Result<Self::Signer, msgs::DecodeError> {
-               let inner: InMemorySigner = Readable::read(&mut reader)?;
+               let inner: InMemorySigner = ReadableArgs::read(&mut reader, self)?;
                let state = Arc::new(Mutex::new(EnforcementState::new()));
 
                Ok(EnforcingSigner::new_with_revoked(
@@ -311,6 +311,17 @@ impl TestBroadcaster {
        pub fn new(blocks: Arc<Mutex<Vec<(Block, u32)>>>) -> TestBroadcaster {
                TestBroadcaster { txn_broadcasted: Mutex::new(Vec::new()), blocks }
        }
+
+       pub fn txn_broadcast(&self) -> Vec<Transaction> {
+               self.txn_broadcasted.lock().unwrap().split_off(0)
+       }
+
+       pub fn unique_txn_broadcast(&self) -> Vec<Transaction> {
+               let mut txn = self.txn_broadcasted.lock().unwrap().split_off(0);
+               let mut seen = HashSet::new();
+               txn.retain(|tx| seen.insert(tx.txid()));
+               txn
+       }
 }
 
 impl chaininterface::BroadcasterInterface for TestBroadcaster {
@@ -772,7 +783,7 @@ impl SignerProvider for TestKeysInterface {
        fn read_chan_signer(&self, buffer: &[u8]) -> Result<Self::Signer, msgs::DecodeError> {
                let mut reader = io::Cursor::new(buffer);
 
-               let inner: InMemorySigner = Readable::read(&mut reader)?;
+               let inner: InMemorySigner = ReadableArgs::read(&mut reader, self)?;
                let state = self.make_enforcement_state_cell(inner.commitment_seed);
 
                Ok(EnforcingSigner::new_with_revoked(